feat(d21): P4 — importer seeds module field_spec + writes field_values

ensureModule seeds each module field_spec (SMS/RTGS/MOBILEAPP declared
fields; ATM/WHATSAPP empty — their one datum stays in the D20 provider
column), back-filling only when a module spec is still empty so an
owner-customized spec is never clobbered. SMS/RTGS/MobileApp service data
now writes into client_module.field_values keyed to the spec; provider/
username/portal-password keep the D20 columns; unkeyed values still land in
details so nothing is lost. Tests assert seeded specs + keyed values; suite
363 green. Real dry run unchanged: 273 clients, 150 docs, 167 links.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/hq/src/import-apex-full.ts

@@ -2,6 +2,7 @@ import { fromRupees, fyOf, uuidv7 } from '@sims/domain'
 import { writeAudit } from './audit'
 import { encrypt } from './crypto'
 import type { DB } from './db'
+import { validateFieldSpec, type FieldDef } from './repos-modules'
 import { seedSeries } from './series'
 
 /**
@@ -235,6 +236,9 @@ interface PlannedAssignment {
   id: string; clientId: string; moduleCode: string
   status: AssignStatus
   provider: string | null; username: string | null; passwordEnc: string | null
+  /** D21: module-declared field values, keyed to the module's field_spec keys. */
+  fieldValues: Record<string, string>
+  /** Values with no matching declared field key — kept so nothing is lost. */
   details: { label: string; value: string }[]
   remark: string | null
   installedOn: string | null
@@ -321,6 +325,46 @@ const MODULE_NAMES: Record<string, string> = {
   SMS: 'Bulk SMS', RTGS: 'RTGS', WHATSAPP: 'WhatsApp', MOBILEAPP: 'Mobile App', ATM: 'ATM',
 }
 
+/**
+ * D21 module-defined field schema. When the importer get-or-creates each of the five
+ * modules it seeds this field_spec, declaring ONLY the module-specific extra fields —
+ * provider / username / portal-password stay in the D20 client_module columns and are
+ * NOT declared here. The per-service values then land in client_module.field_values
+ * keyed to these keys (lower_snake_case, unique — validated at load below), instead of
+ * the free-form details list. Anything with no matching key still goes to details so no
+ * data is lost.
+ */
+const FIELD_SPECS: Record<string, FieldDef[]> = {
+  SMS: [
+    { key: 'sms_balance', label: 'SMS balance', type: 'text' },
+    { key: 'reseller', label: 'Reseller', type: 'text' },
+    { key: 'added_features', label: 'Added features', type: 'text' },
+    { key: 'active_alerts', label: 'Active alerts', type: 'text' },
+    { key: 'phone', label: 'Phone', type: 'text' },
+    { key: 'installation_date', label: 'Installation date', type: 'text' },
+  ],
+  RTGS: [
+    { key: 'ip', label: 'IP', type: 'text' },
+    { key: 'ip_code', label: 'IP code', type: 'text' },
+    { key: 'e_collection_code', label: 'E-collection code', type: 'text' },
+    { key: 'abb', label: 'ABB', type: 'text' },
+    { key: 'partner_bank', label: 'Partner bank', type: 'text' },
+    { key: 'ifsc_2025', label: 'IFSC 2025', type: 'text' },
+    { key: 'generate_qr', label: 'Generate QR', type: 'text' },
+    { key: 'sbi_inward_report', label: 'SBI inward report', type: 'text' },
+    { key: 'passbook_program', label: 'Passbook program', type: 'text' },
+    { key: 'notification', label: 'Notification', type: 'text' },
+    { key: 'misc_rtgs_outward', label: 'Misc RTGS outward', type: 'text' },
+    { key: 'phone', label: 'Phone', type: 'text' },
+    { key: 'installed_on_note', label: 'Installed on note', type: 'text' },
+  ],
+  ATM: [], // provider stays in the D20 column — no extra fields
+  WHATSAPP: [], // flag-only in the export — no per-service field data
+  MOBILEAPP: [{ key: 'contact', label: 'Contact', type: 'text' }],
+}
+// Fail fast at load if any spec is malformed (unique lower_snake_case keys, known types).
+for (const spec of Object.values(FIELD_SPECS)) validateFieldSpec(spec)
+
 const yesIsh = (v: string): boolean => v.toUpperCase().startsWith('Y')
 const looksLikePhone = (v: string): boolean => /^[\d\s+()-]{7,}$/.test(v)
 
@@ -370,6 +414,12 @@ function setDetail(a: PlannedAssignment, label: string, value: string): void {
   else a.details.push({ label, value })
 }
 
+/** Set one module-declared field value (D21) — skip junk/empty; key is a field_spec key. */
+function setField(a: PlannedAssignment, key: string, value: string): void {
+  if (junkDetail(value)) return
+  a.fieldValues[key] = value.trim()
+}
+
 /** Count of non-empty cells — "richer row wins" for duplicate dimension rows (policy 9). */
 function richness(r: Record<string, string>): number {
   return Object.values(r).filter((v) => v !== '').length
@@ -426,7 +476,8 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise<Plan
     if (a === undefined) {
       a = {
         id: uuidv7(), clientId, moduleCode, status,
-        provider: null, username: null, passwordEnc: null, details: [], remark: null,
+        provider: null, username: null, passwordEnc: null,
+        fieldValues: {}, details: [], remark: null,
         installedOn: null, serviceTouched: false,
       }
       assignmentByKey.set(key, a)
@@ -491,7 +542,7 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise<Plan
     }
     if (r.MOBILE_APP !== '' && r.MOBILE_APP.toUpperCase() !== 'NO') {
       const a = ensureAssignment(pc.id, 'MOBILEAPP', 'live')
-      if (!yesIsh(r.MOBILE_APP) && looksLikePhone(r.MOBILE_APP)) setDetail(a, 'Contact', r.MOBILE_APP)
+      if (!yesIsh(r.MOBILE_APP) && looksLikePhone(r.MOBILE_APP)) setField(a, 'contact', r.MOBILE_APP)
     }
   }
   if (pwNoKey > 0) {
@@ -809,14 +860,19 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise<Plan
       a.username = orNull(r.USERNAME)
       a.passwordEnc = passwordEnc
       if (!noData(r.REMARK)) a.remark = r.REMARK
-      a.installedOn = parseApexDate(r.INSTALLATION_DATE) ?? a.installedOn
+      // D21: SMS module-specific values land under the SMS field_spec keys (field_values);
+      // provider/username/portal password keep the D20 columns/password_enc path.
+      const smsInstall = parseApexDate(r.INSTALLATION_DATE)
+      a.installedOn = smsInstall ?? a.installedOn // D20 typed go-live date (roster/card)
+      if (smsInstall !== null) setField(a, 'installation_date', smsInstall)
       // SMS_BALANCE was repurposed as a flag ('YES' on 74 rows) — only numbers land.
-      if (/^\d+(\.\d+)?$/.test(r.SMS_BALANCE)) setDetail(a, 'SMS balance', r.SMS_BALANCE)
+      if (/^\d+(\.\d+)?$/.test(r.SMS_BALANCE)) setField(a, 'sms_balance', r.SMS_BALANCE)
-      setDetail(a, 'Reseller', r.RESELLER)
+      setField(a, 'reseller', r.RESELLER)
-      setDetail(a, 'Features', r.ADDED_FEATURES)
+      setField(a, 'added_features', r.ADDED_FEATURES)
+      setField(a, 'phone', r.PHONE)
+      // No declared SMS field for these — keep them in details so nothing is lost.
       setDetail(a, 'Updation', r.UPDATION)
       if (isPaid(r.PAYMENT)) setDetail(a, 'Payment', 'paid')
-      setDetail(a, 'Phone', r.PHONE)
       a.serviceTouched = true
     }
     if (pwRowsNoKey > 0) {
@@ -846,7 +902,8 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise<Plan
     for (const [clientId, entry] of alertsByClient) {
       // Live alert config proves a live install even where the clients-list flag said NO.
       const a = ensureAssignment(clientId, 'SMS', 'live')
-      setDetail(a, 'Active alerts', entry.heads.join(', '))
+      setField(a, 'active_alerts', entry.heads.join(', '))
+      // Per-alert deviation notes have no declared SMS field — keep them in details.
       if (entry.notes.length > 0) setDetail(a, 'Alert notes', entry.notes.join('; '))
       a.serviceTouched = true
     }
@@ -885,22 +942,24 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise<Plan
         report.rtgs_clients.notes.push(`row ${rows[0]!.row}: RTGS='${r.RTGS}' gives no module signal and no assignment exists — row skipped`)
         continue
       }
-      setDetail(a, 'IP', r.ADDRESS_IP)
+      // D21: RTGS module-specific values land under the RTGS field_spec keys.
-      setDetail(a, 'IP code', r.IP_CODE)
+      setField(a, 'ip', r.ADDRESS_IP)
-      setDetail(a, 'E-collection code', r.E_COLLECTION_CODE)
+      setField(a, 'ip_code', r.IP_CODE)
-      setDetail(a, 'ABB', r.ABB)
+      setField(a, 'e_collection_code', r.E_COLLECTION_CODE)
-      setDetail(a, 'Partner bank', r.INTEGRATED_BANK_PARTNERS)
+      setField(a, 'abb', r.ABB)
-      setDetail(a, 'QR', r.GENERATE_QR)
+      setField(a, 'partner_bank', r.INTEGRATED_BANK_PARTNERS)
-      setDetail(a, 'SBI inward report', r.SBI_INWARD_REPORT)
+      setField(a, 'generate_qr', r.GENERATE_QR)
-      setDetail(a, 'Passbook program', r.PASSBOOK_PROGRAM)
+      setField(a, 'sbi_inward_report', r.SBI_INWARD_REPORT)
-      setDetail(a, 'IFSC 2025', r.IFSC_2025)
+      setField(a, 'passbook_program', r.PASSBOOK_PROGRAM)
-      setDetail(a, 'Notification', r.NOTIFICATION)
+      setField(a, 'ifsc_2025', r.IFSC_2025)
-      setDetail(a, 'Misc RTGS outward', r.MISC_RTGS_OUTWARD)
+      setField(a, 'notification', r.NOTIFICATION)
-      setDetail(a, 'Phone', r.PHONE_NO)
+      setField(a, 'misc_rtgs_outward', r.MISC_RTGS_OUTWARD)
-      // INSTALLATION_DATE is the go-live date → the typed installed_on column (audit
+      setField(a, 'phone', r.PHONE_NO)
-      // verdict); INSTALLED_ON is server-location text ('Database Server') → detail.
+      // INSTALLATION_DATE is the go-live date → the typed installed_on column (D20
+      // verdict); INSTALLED_ON is server-location text ('Database Server') → its own field.
       a.installedOn = parseApexDate(r.INSTALLATION_DATE) ?? a.installedOn
-      setDetail(a, 'Installed on', r.INSTALLED_ON)
+      setField(a, 'installed_on_note', r.INSTALLED_ON)
+      // No declared RTGS field for payment — keep it in details so nothing is lost.
       if (isPaid(r.PAYMENT)) setDetail(a, 'Payment', 'paid')
       if (!noData(r.REMARK)) a.remark = r.REMARK
       a.serviceTouched = true
@@ -1018,13 +1077,22 @@ export interface FullCommitResult {
 const ALL_KINDS_JSON = JSON.stringify(['one_time', 'monthly', 'yearly', 'usage'])
 
 async function ensureModule(db: DB, code: string): Promise<string> {
-  const row = await db.get<{ id: string }>(`SELECT id FROM module WHERE code=?`, code)
+  const spec = JSON.stringify(FIELD_SPECS[code] ?? [])
-  if (row !== undefined) return row.id
+  const row = await db.get<{ id: string; field_spec: string }>(
+    `SELECT id, field_spec FROM module WHERE code=?`, code)
+  if (row !== undefined) {
+    // Additive backfill: seed our field_spec only if the module has none yet —
+    // never clobber a spec an owner has since customized.
+    if ((row.field_spec ?? '[]') === '[]' && spec !== '[]') {
+      await db.run(`UPDATE module SET field_spec=? WHERE id=?`, spec, row.id)
+    }
+    return row.id
+  }
   const id = uuidv7()
   await db.run(
-    `INSERT INTO module (id, code, name, sac, allowed_kinds, multi_subscription, quote_content)
+    `INSERT INTO module (id, code, name, sac, allowed_kinds, multi_subscription, quote_content, field_spec)
-     VALUES (?, ?, ?, '998313', ?, 0, '[]')`,
+     VALUES (?, ?, ?, '998313', ?, 0, '[]', ?)`,
-    id, code, MODULE_NAMES[code] ?? code, ALL_KINDS_JSON,
+    id, code, MODULE_NAMES[code] ?? code, ALL_KINDS_JSON, spec,
   )
   return id
 }
@@ -1071,16 +1139,18 @@ export async function commitApexFull(
       if (existingCm === undefined) {
         await db.run(
           `INSERT INTO client_module (id, client_id, module_id, status, kind, edition, active,
-             installed_on, provider, username, password_enc, details, remark)
+             installed_on, provider, username, password_enc, details, remark, field_values)
-           VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?)`,
+           VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?, ?)`,
           a.id, a.clientId, moduleId, a.status, a.installedOn,
           a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark,
+          JSON.stringify(a.fieldValues),
         )
       } else if (a.serviceTouched) {
         // Client pre-dated this import with a live assignment: service data wins.
         await db.run(
-          `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=?, installed_on=? WHERE id=?`,
+          `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=?, installed_on=?, field_values=? WHERE id=?`,
-          a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, a.installedOn, existingCm.id,
+          a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, a.installedOn,
+          JSON.stringify(a.fieldValues), existingCm.id,
         )
       }
     }

apps/hq/test/import-apex-full.test.ts

@@ -201,8 +201,10 @@ const cmFor = async (db: DB, clientCode: string, moduleCode: string) =>
   db.get<{
     status: string; kind: string; provider: string | null; username: string | null
     password_enc: string | null; details: string; remark: string | null; installed_on: string | null
+    field_values: string
   }>(
-    `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark, cm.installed_on
+    `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark,
+            cm.installed_on, cm.field_values
      FROM client_module cm JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id
      WHERE c.code=? AND m.code=?`, clientCode, moduleCode)
 
@@ -280,13 +282,29 @@ describe('APEX full importer v2 (real-export shapes)', () => {
     expect(atm.status).toBe('live')
     expect(atm.provider).toBe('EWIRE')
     const mob = (await cmFor(db, '103', 'MOBILEAPP'))!
-    expect(JSON.parse(mob.details)).toContainEqual({ label: 'Contact', value: '8301932166' })
+    expect(JSON.parse(mob.field_values)).toEqual({ contact: '8301932166' }) // D21 declared field
+    expect(JSON.parse(mob.details)).toEqual([])
 
     // Status vocabulary: 103's SMS came from GENERAL ENQUIRY (quoted) and the latest
     // project row (NEW PROJECT) upgraded it to installing — never to live.
     const sms103 = (await cmFor(db, '103', 'SMS'))!
     expect(sms103.status).toBe('installing')
 
+    // D21: each get-or-created module carries its declared field_spec (module-specific
+    // fields only — provider/username/portal password stay in the D20 columns).
+    const specKeys = async (code: string): Promise<string[]> => {
+      const row = (await db.get<{ field_spec: string }>(`SELECT field_spec FROM module WHERE code=?`, code))!
+      return (JSON.parse(row.field_spec) as { key: string }[]).map((f) => f.key)
+    }
+    expect(await specKeys('SMS')).toEqual(
+      ['sms_balance', 'reseller', 'added_features', 'active_alerts', 'phone', 'installation_date'])
+    expect(await specKeys('RTGS')).toEqual([
+      'ip', 'ip_code', 'e_collection_code', 'abb', 'partner_bank', 'ifsc_2025', 'generate_qr',
+      'sbi_inward_report', 'passbook_program', 'notification', 'misc_rtgs_outward', 'phone', 'installed_on_note'])
+    expect(await specKeys('ATM')).toEqual([]) // provider stays in the D20 column
+    expect(await specKeys('WHATSAPP')).toEqual([])
+    expect(await specKeys('MOBILEAPP')).toEqual(['contact'])
+
     // SMS service data on 102 (richer duplicate row won; junk 'WHATSAPP' row dropped).
     const sms102 = (await cmFor(db, '102', 'SMS'))!
     expect(sms102.status).toBe('live')
@@ -294,23 +312,32 @@ describe('APEX full importer v2 (real-export shapes)', () => {
     expect(sms102.username).toBe('betauser')
     expect(decrypt(sms102.password_enc!, KEY)).toBe('smsPass9')
     expect(sms102.remark).toBe('ok')
-    expect(sms102.installed_on).toBe('2026-03-25') // typed column, not a detail
+    expect(sms102.installed_on).toBe('2026-03-25') // typed column, not a field/detail
+    // D21: module-specific values land under the SMS field_spec keys (field_values).
+    const smsFields = JSON.parse(sms102.field_values) as Record<string, string>
+    expect(smsFields).toEqual({
+      reseller: 'RSL', phone: '0474222', installation_date: '2026-03-25',
+      active_alerts: 'ATM ALERT (PKG_SMS_JOB), BALANCE ALERT',
+    })
+    expect(smsFields.sms_balance).toBeUndefined() // 'YES' is a flag, not a balance
+    // Values with no declared field key stay in details so nothing is lost.
     const smsDetails = JSON.parse(sms102.details) as { label: string; value: string }[]
-    expect(smsDetails.find((d) => d.label === 'SMS balance')).toBeUndefined() // 'YES' is a flag, not a balance
     expect(smsDetails).toContainEqual({ label: 'Payment', value: 'paid' })
-    expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT (PKG_SMS_JOB), BALANCE ALERT' })
     expect(smsDetails).toContainEqual({ label: 'Alert notes', value: 'BALANCE ALERT: emp only' })
 
     // RTGS: duplicate engagements merged richer-first; INSTALLATION_DATE → installed_on;
     // INSTALLED_ON server text → detail; Nil payment did not overwrite PAID.
     const rtgs102 = (await cmFor(db, '102', 'RTGS'))!
     expect(rtgs102.status).toBe('live')
-    expect(rtgs102.installed_on).toBe('2024-01-05')
+    expect(rtgs102.installed_on).toBe('2024-01-05') // typed go-live date column
+    // D21: RTGS values land under the RTGS field_spec keys.
+    const rtgsFields = JSON.parse(rtgs102.field_values) as Record<string, string>
+    expect(rtgsFields).toEqual({
+      ip: '10.0.0.5', ip_code: 'IPC1', abb: 'BETA', partner_bank: 'SBI', generate_qr: 'YES',
+      phone: '333', installed_on_note: 'Database Server', e_collection_code: 'EC77', // merged from sparse row
+    })
     const rtgsDetails = JSON.parse(rtgs102.details) as { label: string; value: string }[]
-    expect(rtgsDetails).toContainEqual({ label: 'IP', value: '10.0.0.5' })
+    expect(rtgsDetails).toEqual([{ label: 'Payment', value: 'paid' }]) // only the unkeyed value
-    expect(rtgsDetails).toContainEqual({ label: 'Installed on', value: 'Database Server' })
-    expect(rtgsDetails).toContainEqual({ label: 'E-collection code', value: 'EC77' }) // merged from sparse row
-    expect(rtgsDetails).toContainEqual({ label: 'Payment', value: 'paid' })
 
     // Bills, era (a): grouped lines → ONE document; GST from the pseudo-line; ONE payment.
     const inv = (await db.get<{