From 51023f9ec4a69b69461d394c1d9f7d8f2002707b Mon Sep 17 00:00:00 2001 From: Thomas Joise Date: Fri, 17 Jul 2026 20:48:16 +0530 Subject: [PATCH] =?UTF-8?q?feat(d21):=20P4=20=E2=80=94=20importer=20seeds?= =?UTF-8?q?=20module=20field=5Fspec=20+=20writes=20field=5Fvalues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureModule seeds each module field_spec (SMS/RTGS/MOBILEAPP declared fields; ATM/WHATSAPP empty — their one datum stays in the D20 provider column), back-filling only when a module spec is still empty so an owner-customized spec is never clobbered. SMS/RTGS/MobileApp service data now writes into client_module.field_values keyed to the spec; provider/ username/portal-password keep the D20 columns; unkeyed values still land in details so nothing is lost. Tests assert seeded specs + keyed values; suite 363 green. Real dry run unchanged: 273 clients, 150 docs, 167 links. Co-Authored-By: Claude Fable 5 --- apps/hq/src/import-apex-full.ts | 134 ++++++++++++++++++++------ apps/hq/test/import-apex-full.test.ts | 47 +++++++-- 2 files changed, 139 insertions(+), 42 deletions(-) diff --git a/apps/hq/src/import-apex-full.ts b/apps/hq/src/import-apex-full.ts index 283c523..39bdaca 100644 --- a/apps/hq/src/import-apex-full.ts +++ b/apps/hq/src/import-apex-full.ts @@ -2,6 +2,7 @@ import { fromRupees, fyOf, uuidv7 } from '@sims/domain' import { writeAudit } from './audit' import { encrypt } from './crypto' import type { DB } from './db' +import { validateFieldSpec, type FieldDef } from './repos-modules' import { seedSeries } from './series' /** @@ -235,6 +236,9 @@ interface PlannedAssignment { id: string; clientId: string; moduleCode: string status: AssignStatus provider: string | null; username: string | null; passwordEnc: string | null + /** D21: module-declared field values, keyed to the module's field_spec keys. */ + fieldValues: Record + /** Values with no matching declared field key — kept so nothing is lost. */ details: { label: string; value: string }[] remark: string | null installedOn: string | null @@ -321,6 +325,46 @@ const MODULE_NAMES: Record = { SMS: 'Bulk SMS', RTGS: 'RTGS', WHATSAPP: 'WhatsApp', MOBILEAPP: 'Mobile App', ATM: 'ATM', } +/** + * D21 module-defined field schema. When the importer get-or-creates each of the five + * modules it seeds this field_spec, declaring ONLY the module-specific extra fields — + * provider / username / portal-password stay in the D20 client_module columns and are + * NOT declared here. The per-service values then land in client_module.field_values + * keyed to these keys (lower_snake_case, unique — validated at load below), instead of + * the free-form details list. Anything with no matching key still goes to details so no + * data is lost. + */ +const FIELD_SPECS: Record = { + SMS: [ + { key: 'sms_balance', label: 'SMS balance', type: 'text' }, + { key: 'reseller', label: 'Reseller', type: 'text' }, + { key: 'added_features', label: 'Added features', type: 'text' }, + { key: 'active_alerts', label: 'Active alerts', type: 'text' }, + { key: 'phone', label: 'Phone', type: 'text' }, + { key: 'installation_date', label: 'Installation date', type: 'text' }, + ], + RTGS: [ + { key: 'ip', label: 'IP', type: 'text' }, + { key: 'ip_code', label: 'IP code', type: 'text' }, + { key: 'e_collection_code', label: 'E-collection code', type: 'text' }, + { key: 'abb', label: 'ABB', type: 'text' }, + { key: 'partner_bank', label: 'Partner bank', type: 'text' }, + { key: 'ifsc_2025', label: 'IFSC 2025', type: 'text' }, + { key: 'generate_qr', label: 'Generate QR', type: 'text' }, + { key: 'sbi_inward_report', label: 'SBI inward report', type: 'text' }, + { key: 'passbook_program', label: 'Passbook program', type: 'text' }, + { key: 'notification', label: 'Notification', type: 'text' }, + { key: 'misc_rtgs_outward', label: 'Misc RTGS outward', type: 'text' }, + { key: 'phone', label: 'Phone', type: 'text' }, + { key: 'installed_on_note', label: 'Installed on note', type: 'text' }, + ], + ATM: [], // provider stays in the D20 column — no extra fields + WHATSAPP: [], // flag-only in the export — no per-service field data + MOBILEAPP: [{ key: 'contact', label: 'Contact', type: 'text' }], +} +// Fail fast at load if any spec is malformed (unique lower_snake_case keys, known types). +for (const spec of Object.values(FIELD_SPECS)) validateFieldSpec(spec) + const yesIsh = (v: string): boolean => v.toUpperCase().startsWith('Y') const looksLikePhone = (v: string): boolean => /^[\d\s+()-]{7,}$/.test(v) @@ -370,6 +414,12 @@ function setDetail(a: PlannedAssignment, label: string, value: string): void { else a.details.push({ label, value }) } +/** Set one module-declared field value (D21) — skip junk/empty; key is a field_spec key. */ +function setField(a: PlannedAssignment, key: string, value: string): void { + if (junkDetail(value)) return + a.fieldValues[key] = value.trim() +} + /** Count of non-empty cells — "richer row wins" for duplicate dimension rows (policy 9). */ function richness(r: Record): number { return Object.values(r).filter((v) => v !== '').length @@ -426,7 +476,8 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise 0) { @@ -809,14 +860,19 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise 0) { @@ -846,7 +902,8 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise 0) setDetail(a, 'Alert notes', entry.notes.join('; ')) a.serviceTouched = true } @@ -885,22 +942,24 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise { - const row = await db.get<{ id: string }>(`SELECT id FROM module WHERE code=?`, code) - if (row !== undefined) return row.id + const spec = JSON.stringify(FIELD_SPECS[code] ?? []) + const row = await db.get<{ id: string; field_spec: string }>( + `SELECT id, field_spec FROM module WHERE code=?`, code) + if (row !== undefined) { + // Additive backfill: seed our field_spec only if the module has none yet — + // never clobber a spec an owner has since customized. + if ((row.field_spec ?? '[]') === '[]' && spec !== '[]') { + await db.run(`UPDATE module SET field_spec=? WHERE id=?`, spec, row.id) + } + return row.id + } const id = uuidv7() await db.run( - `INSERT INTO module (id, code, name, sac, allowed_kinds, multi_subscription, quote_content) - VALUES (?, ?, ?, '998313', ?, 0, '[]')`, - id, code, MODULE_NAMES[code] ?? code, ALL_KINDS_JSON, + `INSERT INTO module (id, code, name, sac, allowed_kinds, multi_subscription, quote_content, field_spec) + VALUES (?, ?, ?, '998313', ?, 0, '[]', ?)`, + id, code, MODULE_NAMES[code] ?? code, ALL_KINDS_JSON, spec, ) return id } @@ -1071,16 +1139,18 @@ export async function commitApexFull( if (existingCm === undefined) { await db.run( `INSERT INTO client_module (id, client_id, module_id, status, kind, edition, active, - installed_on, provider, username, password_enc, details, remark) - VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?)`, + installed_on, provider, username, password_enc, details, remark, field_values) + VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?, ?)`, a.id, a.clientId, moduleId, a.status, a.installedOn, a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, + JSON.stringify(a.fieldValues), ) } else if (a.serviceTouched) { // Client pre-dated this import with a live assignment: service data wins. await db.run( - `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=?, installed_on=? WHERE id=?`, - a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, a.installedOn, existingCm.id, + `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=?, installed_on=?, field_values=? WHERE id=?`, + a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, a.installedOn, + JSON.stringify(a.fieldValues), existingCm.id, ) } } diff --git a/apps/hq/test/import-apex-full.test.ts b/apps/hq/test/import-apex-full.test.ts index 946c270..45d6cfc 100644 --- a/apps/hq/test/import-apex-full.test.ts +++ b/apps/hq/test/import-apex-full.test.ts @@ -201,8 +201,10 @@ const cmFor = async (db: DB, clientCode: string, moduleCode: string) => db.get<{ status: string; kind: string; provider: string | null; username: string | null password_enc: string | null; details: string; remark: string | null; installed_on: string | null + field_values: string }>( - `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark, cm.installed_on + `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark, + cm.installed_on, cm.field_values FROM client_module cm JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id WHERE c.code=? AND m.code=?`, clientCode, moduleCode) @@ -280,13 +282,29 @@ describe('APEX full importer v2 (real-export shapes)', () => { expect(atm.status).toBe('live') expect(atm.provider).toBe('EWIRE') const mob = (await cmFor(db, '103', 'MOBILEAPP'))! - expect(JSON.parse(mob.details)).toContainEqual({ label: 'Contact', value: '8301932166' }) + expect(JSON.parse(mob.field_values)).toEqual({ contact: '8301932166' }) // D21 declared field + expect(JSON.parse(mob.details)).toEqual([]) // Status vocabulary: 103's SMS came from GENERAL ENQUIRY (quoted) and the latest // project row (NEW PROJECT) upgraded it to installing — never to live. const sms103 = (await cmFor(db, '103', 'SMS'))! expect(sms103.status).toBe('installing') + // D21: each get-or-created module carries its declared field_spec (module-specific + // fields only — provider/username/portal password stay in the D20 columns). + const specKeys = async (code: string): Promise => { + const row = (await db.get<{ field_spec: string }>(`SELECT field_spec FROM module WHERE code=?`, code))! + return (JSON.parse(row.field_spec) as { key: string }[]).map((f) => f.key) + } + expect(await specKeys('SMS')).toEqual( + ['sms_balance', 'reseller', 'added_features', 'active_alerts', 'phone', 'installation_date']) + expect(await specKeys('RTGS')).toEqual([ + 'ip', 'ip_code', 'e_collection_code', 'abb', 'partner_bank', 'ifsc_2025', 'generate_qr', + 'sbi_inward_report', 'passbook_program', 'notification', 'misc_rtgs_outward', 'phone', 'installed_on_note']) + expect(await specKeys('ATM')).toEqual([]) // provider stays in the D20 column + expect(await specKeys('WHATSAPP')).toEqual([]) + expect(await specKeys('MOBILEAPP')).toEqual(['contact']) + // SMS service data on 102 (richer duplicate row won; junk 'WHATSAPP' row dropped). const sms102 = (await cmFor(db, '102', 'SMS'))! expect(sms102.status).toBe('live') @@ -294,23 +312,32 @@ describe('APEX full importer v2 (real-export shapes)', () => { expect(sms102.username).toBe('betauser') expect(decrypt(sms102.password_enc!, KEY)).toBe('smsPass9') expect(sms102.remark).toBe('ok') - expect(sms102.installed_on).toBe('2026-03-25') // typed column, not a detail + expect(sms102.installed_on).toBe('2026-03-25') // typed column, not a field/detail + // D21: module-specific values land under the SMS field_spec keys (field_values). + const smsFields = JSON.parse(sms102.field_values) as Record + expect(smsFields).toEqual({ + reseller: 'RSL', phone: '0474222', installation_date: '2026-03-25', + active_alerts: 'ATM ALERT (PKG_SMS_JOB), BALANCE ALERT', + }) + expect(smsFields.sms_balance).toBeUndefined() // 'YES' is a flag, not a balance + // Values with no declared field key stay in details so nothing is lost. const smsDetails = JSON.parse(sms102.details) as { label: string; value: string }[] - expect(smsDetails.find((d) => d.label === 'SMS balance')).toBeUndefined() // 'YES' is a flag, not a balance expect(smsDetails).toContainEqual({ label: 'Payment', value: 'paid' }) - expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT (PKG_SMS_JOB), BALANCE ALERT' }) expect(smsDetails).toContainEqual({ label: 'Alert notes', value: 'BALANCE ALERT: emp only' }) // RTGS: duplicate engagements merged richer-first; INSTALLATION_DATE → installed_on; // INSTALLED_ON server text → detail; Nil payment did not overwrite PAID. const rtgs102 = (await cmFor(db, '102', 'RTGS'))! expect(rtgs102.status).toBe('live') - expect(rtgs102.installed_on).toBe('2024-01-05') + expect(rtgs102.installed_on).toBe('2024-01-05') // typed go-live date column + // D21: RTGS values land under the RTGS field_spec keys. + const rtgsFields = JSON.parse(rtgs102.field_values) as Record + expect(rtgsFields).toEqual({ + ip: '10.0.0.5', ip_code: 'IPC1', abb: 'BETA', partner_bank: 'SBI', generate_qr: 'YES', + phone: '333', installed_on_note: 'Database Server', e_collection_code: 'EC77', // merged from sparse row + }) const rtgsDetails = JSON.parse(rtgs102.details) as { label: string; value: string }[] - expect(rtgsDetails).toContainEqual({ label: 'IP', value: '10.0.0.5' }) - expect(rtgsDetails).toContainEqual({ label: 'Installed on', value: 'Database Server' }) - expect(rtgsDetails).toContainEqual({ label: 'E-collection code', value: 'EC77' }) // merged from sparse row - expect(rtgsDetails).toContainEqual({ label: 'Payment', value: 'paid' }) + expect(rtgsDetails).toEqual([{ label: 'Payment', value: 'paid' }]) // only the unkeyed value // Bills, era (a): grouped lines → ONE document; GST from the pseudo-line; ONE payment. const inv = (await db.get<{