From 1328a9e898757c27fe579d534f43d0cbacaf87e2 Mon Sep 17 00:00:00 2001 From: Thomas Joise Date: Fri, 17 Jul 2026 20:09:27 +0530 Subject: [PATCH] feat(d20): importer v2 rework from the 46-agent real-data audit Bills are LINE ITEMS not documents: group by INVOICE_NO across three GST eras (GST pseudo-line / 2026 columns-only / pre-GST zero-tax). 46 raw ticket statuses -> 5 via ordered pattern map; OLD_ID re-key chains collapse so superseded rows do not double-import. YES/NO flags respect real vocabulary (GENERAL ENQUIRY != yes); ATM cell value becomes the module provider. SID 329 test client + orphan refs skip (counted notes, distinct from commit-blocking problems). District normalize, NO DATA scrub, payment normalize, richer-row merge for duplicate service rows, shared-number-space series seed. Dry run over the real export: 273 clients, 150 documents = Rs 33,16,800.24, 72 payments, 1,826 tickets, 667 interactions, INVOICE seed 10252 (matches APEX next_val). Suite 358 green. Co-Authored-By: Claude Fable 5 --- apps/hq/scripts/import-apex-full.ts | 30 +- apps/hq/src/import-apex-full.ts | 919 ++++++++++++++++++-------- apps/hq/test/import-apex-full.test.ts | 513 ++++++++------ 3 files changed, 994 insertions(+), 468 deletions(-) diff --git a/apps/hq/scripts/import-apex-full.ts b/apps/hq/scripts/import-apex-full.ts index 126ea7d..74c530b 100644 --- a/apps/hq/scripts/import-apex-full.ts +++ b/apps/hq/scripts/import-apex-full.ts @@ -1,14 +1,16 @@ /** * Full-APEX importer CLI (D20): `npm run import:full -- --dir [--commit]` * Reads the nine workbook CSVs from (all optional except clients.csv), - * prints the per-section stage report, and — only with --commit and a clean - * report — applies everything in one transaction. Without --commit it is a dry - * run that writes nothing. DB path comes from HQ_DATA_DIR (run from the repo - * root otherwise — the default resolves ./data from CWD); passwords in the - * export need HQ_SECRET_KEY set or their rows are refused. + * prints the per-section stage report (problems block commit; notes are the + * deliberate skips/merges), and — only with --commit and a problem-free report — + * applies everything in one transaction. Without --commit it is a dry run that + * writes nothing and needs no HQ_SECRET_KEY. DB path comes from HQ_DATA_DIR (run + * from the repo root otherwise — the default resolves ./data from CWD). NEVER + * prints credential values — the report carries counts and row numbers only. */ import fs from 'node:fs' import path from 'node:path' +import { formatINR } from '@sims/domain' import { openDb } from '../src/db' import { seedIfEmpty } from '../src/seed' import { APEX_FILES, commitApexFull, stageApexFull } from '../src/import-apex-full' @@ -38,20 +40,26 @@ void (async () => { await seedIfEmpty(db) const keyHex = process.env['HQ_SECRET_KEY'] ?? '' - const report = await stageApexFull(db, files, keyHex) + const { sections, summary } = await stageApexFull(db, files, keyHex) let problemTotal = 0 for (const section of APEX_FILES) { - const r = report[section] - if (r.staged === 0 && r.problems.length === 0) continue - console.log(`${section}: ${r.staged} staged, ${r.problems.length} problem(s)`) - for (const p of r.problems) console.log(` - ${p}`) + const r = sections[section] + if (r.staged === 0 && r.problems.length === 0 && r.notes.length === 0) continue + console.log(`${section}: ${r.staged} staged, ${r.problems.length} problem(s), ${r.notes.length} note(s)`) + for (const p of r.problems) console.log(` ! ${p}`) + for (const n of r.notes) console.log(` - ${n}`) problemTotal += r.problems.length } + console.log('---') + console.log(`Plan: ${summary.clients} clients, ${summary.client_modules} module assignments, ${summary.branches} branches,`) + console.log(` ${summary.documents} documents totalling ${formatINR(summary.totalPayablePaise)} payable, ${summary.payments} payments,`) + console.log(` ${summary.tickets} tickets, ${summary.interactions} interactions;`) + console.log(` INVOICE series seed for ${summary.currentFy}: ${summary.seedTail ?? 'none (no current-FY docs)'}`) if (!args.includes('--commit')) { console.log('Dry run — nothing written. Re-run with --commit to apply.') } else if (problemTotal > 0) { - console.error(`Refusing to commit: ${problemTotal} problem row(s) above — fix the CSVs and re-run.`) + console.error(`Refusing to commit: ${problemTotal} problem(s) above — fix the CSVs and re-run.`) process.exitCode = 1 } else { const counts = await commitApexFull(db, 'system', files, keyHex) diff --git a/apps/hq/src/import-apex-full.ts b/apps/hq/src/import-apex-full.ts index 7b486e6..283c523 100644 --- a/apps/hq/src/import-apex-full.ts +++ b/apps/hq/src/import-apex-full.ts @@ -5,15 +5,39 @@ import type { DB } from './db' import { seedSeries } from './series' /** - * Full-APEX importer v2 (D20 — docs/superpowers/specs/2026-07-17-apex-parity-design.md §4). - * Follows v1's discipline: stage → per-row problems → refuse commit while any problem - * exists. Unlike v1 there are no staging tables — `stageApexFull` is a pure validation - * pass that writes nothing; `commitApexFull` re-runs the same validation and applies - * everything inside ONE transaction, auditing one summary row per section (per-row - * client/document audits are deliberately skipped — 5k+ rows would drown the log). - * Rupee amounts convert to integer paise via `fromRupees` at this edge; plaintext - * passwords (client DB + SMS portal) are AES-256-GCM encrypted or the row is refused. - * v1 (`import-apex.ts`) stays untouched for the simple owner-only web flow. + * Full-APEX importer v2 (D20 — docs/superpowers/specs/2026-07-17-apex-parity-design.md §4), + * reworked against the profiled REAL export (46-agent audit, 2026-07-17). Follows v1's + * discipline: stage → per-row problems → refuse commit while problems exist — with one + * addition: deliberate, founder-approved auto-resolutions (test-client skips, orphan refs, + * duplicate-row merges, chain collapses) are counted `notes`, NOT commit-blocking problems, + * because the real export contains them and the import must still land. + * + * The big structural facts this version encodes (from the audit): + * - bills.csv is LINE grain: 269 rows = 151 documents; GST rides as pseudo-lines + * (DESCRIPTION='GST' / PER_QTY '18%') or in GST_AMOUNT columns depending on era. + * - tickets.csv is a day-workbench: 4,567 rows = ~1,857 threads chained by OLD_ID; + * superseded carry-forward copies must collapse or the open queue is ~13x inflated. + * - sms/rtgs service files carry pipeline-status vocabulary (GENERAL ENQUIRY / NEW + * PROJECT / WAITING…), not YES/NO flags. + * + * `stageApexFull` is a pure validation pass that writes nothing; `commitApexFull` + * re-runs the same validation and applies everything inside ONE transaction, auditing + * one summary row per section. Rupees→paise via `fromRupees` at this edge; plaintext + * passwords are AES-256-GCM encrypted or the commit refuses. v1 stays untouched. + * + * Explicit skips (audit verdicts, recorded here as the deliberate calls): + * - tickets.MAIL_BLOB: exported as '[unsupported data type]' — content lost in the + * export; MAIL='Y' is preserved as a description marker so the fact survives. + * - bills TDS split: short-payments (e.g. the 1% TDS on SMS/2026/10211) land as a + * partial allocation with tds_paise=0 — the true TDS split is not derivable. + * - bills QTY/PER_QTY: not used for amounts (transposed on 4 rows; AMT is reliable). + * - bills INVOICE_AMOUNT: not used — its meaning flips base→gross in the 2026 era. + * - Cross-bill lump-sum payments (3 events stamped on 2 bills each): accepted; + * per-doc allocation is clamped to that doc's payable, which caps the damage. + * - sms_clients.ERR (ORA-29273 noise) and TID; tickets ASSIGNED_TO (numeric APEX + * staff ids with no HQ mapping) and BRID (blank on all rows); projects NEW_FORM + * (APEX form flag) and CREATED_BY/UPDATED_BY (provenance survives only via the + * '[assigned: …]' note folded from ASSIGNED_TO). */ // ---------- tiny CSV parser (copied from v1, which keeps it private) ---------- @@ -147,11 +171,12 @@ function validDate(y: number, month: number, day: number): string | null { } /** - * Best-effort date-part extraction for the formats seen in the APEX export: - * ISO ('2025-08-18 00:00:00'), DD-MON-YYYY ('10-NOV-2023 10:57:18'), and - * X-Y-ZZZZ / X/Y/ZZZZ where whichever of X/Y is >12 must be the day - * ('08/29/2024' → MM/DD, '13-05-2024' → DD-MM); ambiguous → assume DD-MM - * (the dominant APEX habit). Returns 'YYYY-MM-DD' or null. + * Best-effort date-part extraction for the formats profiled in the APEX export: + * ISO ('2025-08-18 00:00:00'), DD-MON-YYYY ('10-NOV-2023 10:57:18'), and X-Y-ZZZZ. + * Profiling proved the SEPARATOR decides day/month order in the project files + * (dash = day-first '16-11-2023', slash = month-first '09/19/2024', with a handful + * of contradictions like '12-13-2023') — so: dash tries DD-MM then falls back to + * MM-DD, slash tries MM/DD then falls back to DD/MM. Returns 'YYYY-MM-DD' or null. */ export function parseApexDate(raw: string): string | null { const s = raw.trim() @@ -163,21 +188,42 @@ export function parseApexDate(raw: string): string | null { const month = MONTHS[m[2]!.toUpperCase()] return month === undefined ? null : validDate(Number(m[3]!), month, Number(m[1]!)) } - m = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})/.exec(s) + m = /^(\d{1,2})([/-])(\d{1,2})[/-](\d{4})/.exec(s) if (m !== null) { const a = Number(m[1]!) - const b = Number(m[2]!) - const [day, month] = a > 12 ? [a, b] : b > 12 ? [b, a] : [a, b] - return validDate(Number(m[3]!), month!, day!) + const b = Number(m[3]!) + const y = Number(m[4]!) + return m[2] === '/' + ? (validDate(y, a, b) ?? validDate(y, b, a)) // slash: MM/DD, fall back DD/MM + : (validDate(y, b, a) ?? validDate(y, a, b)) // dash: DD-MM, fall back MM-DD } return null } // ---------- report + plan ---------- -export interface SectionReport { staged: number; problems: string[] } +/** + * `problems` block commit (malformed data an operator must fix). `notes` are the + * counted, deliberate auto-resolutions (test-client skips, orphan refs, merges, + * chain collapses) — the real export contains them, so they must not block. + */ +export interface SectionReport { staged: number; problems: string[]; notes: string[] } export type FullImportReport = Record +export interface StageSummary { + clients: number; client_modules: number; branches: number + documents: number; totalPayablePaise: number; payments: number + tickets: number; interactions: number + currentFy: string + /** Max numeric tail among current-FY docs of ANY type (QT/PI/INV share one APEX space). */ + seedTail: number | null +} + +export interface FullStageResult { sections: FullImportReport; summary: StageSummary } + +type AssignStatus = 'quoted' | 'installing' | 'live' +const STATUS_RANK: Record = { quoted: 0, installing: 1, live: 2 } + interface PlannedClient { id: string; code: string; name: string; address: string district: string | null; sector: string | null; os: string | null; anydesk: string | null @@ -187,10 +233,12 @@ interface PlannedClient { interface PlannedAssignment { id: string; clientId: string; moduleCode: string + status: AssignStatus provider: string | null; username: string | null; passwordEnc: string | null details: { label: string; value: string }[] remark: string | null - /** True once sms/rtgs service data landed — gates updates onto pre-existing rows. */ + installedOn: string | null + /** True once service data landed — gates updates onto pre-existing rows. */ serviceTouched: boolean } @@ -200,12 +248,18 @@ interface PlannedDoc { id: string; docType: 'INVOICE' | 'PROFORMA' | 'QUOTATION'; docNo: string; fy: string clientId: string; docDate: string; status: string taxable: number; cgst: number; sgst: number; igst: number; roundOff: number; total: number - payment: { amount: number; receivedOn: string } | null + payments: { amount: number; receivedOn: string }[] + /** Joined real line DESCRIPTIONs + NOTES memos (policy 1: land in payload.notes). */ + notes: string | null + /** As-issued bill-to snapshot — name/address/mail differ from today's client on some bills. */ + billTo: { name?: string; address?: string; email?: string } | null } +type TicketStatus = 'open' | 'in_progress' | 'waiting' | 'closed' | 'dropped' + interface PlannedTicket { id: string; clientId: string; moduleCode: string | null; kind: string; description: string - status: 'open' | 'in_progress' | 'waiting' | 'closed' | 'dropped' + status: TicketStatus onlineOffline: string | null; openedOn: string; closedOn: string | null; createdAt: string } @@ -223,8 +277,9 @@ interface Plan { docs: PlannedDoc[] tickets: PlannedTicket[] interactions: PlannedInteraction[] + /** Applied-row counts per project section (interactions + tickets created from it). */ + projectCounts: { sms_projects: number; rtgs_projects: number } currentFy: string - /** Max numeric tail among current-FY invoice numbers, for the series seed. */ seedTail: number | null } @@ -241,51 +296,98 @@ function rupeesToPaise(raw: string): number | null { return Number.isFinite(n) ? fromRupees(n) : null } -/** '' and the APEX placeholder 'NO DATA' both mean "nothing here". */ -const noData = (v: string): boolean => v === '' || v.toUpperCase() === 'NO DATA' +/** '' and the APEX placeholders ('NO DATA', 'No Data', 'NO DATA FOUND') mean "nothing here". */ +const noData = (v: string): boolean => + v === '' || v.toUpperCase() === 'NO DATA' || v.toUpperCase() === 'NO DATA FOUND' + +/** Junk placeholder values that must not become labeled details. */ +const junkDetail = (v: string): boolean => + noData(v) || ['N/A', 'NA', 'NIL'].includes(v.toUpperCase()) + +/** Payment normalize (policy 8): PAID any case → paid; N/UNPAID/PENDING/Nil/NA/blank → not. */ +const isPaid = (v: string): boolean => v.toUpperCase() === 'PAID' + +/** DISTRICT normalize (policy 8): UPPER trimmed + the known misspelling. */ +function normDistrict(v: string): string | null { + const d = v.toUpperCase() + if (d === '') return null + return d === 'PATHANAMTHITA' ? 'PATHANAMTHITTA' : d +} + +/** APEX-internal test client (SID 329 = 'WQEQE'/'SIMS') — skipped everywhere, never landed. */ +const TEST_SIDS = new Set(['329']) const MODULE_NAMES: Record = { SMS: 'Bulk SMS', RTGS: 'RTGS', WHATSAPP: 'WhatsApp', MOBILEAPP: 'Mobile App', ATM: 'ATM', } -const FLAG_MODULES: { col: 'SMS' | 'RTGS' | 'WHATSAPP' | 'MOBILE_APP' | 'ATM'; code: string }[] = [ - { col: 'SMS', code: 'SMS' }, - { col: 'RTGS', code: 'RTGS' }, - { col: 'WHATSAPP', code: 'WHATSAPP' }, - { col: 'MOBILE_APP', code: 'MOBILEAPP' }, - { col: 'ATM', code: 'ATM' }, -] +const yesIsh = (v: string): boolean => v.toUpperCase().startsWith('Y') +const looksLikePhone = (v: string): boolean => /^[\d\s+()-]{7,}$/.test(v) + +/** + * Pipeline-status vocabulary carried by sms_clients.SMS / rtgs_clients.RTGS (policy 5): + * PAID/YES/COMPLETED → live; NEW PROJECT → installing; WAITING…/GENERAL ENQUIRY → quoted; + * CANCELLED/NO/blank/anything else → no module from this signal. + */ +function moduleSignal(raw: string): AssignStatus | null { + const v = raw.toUpperCase() + if (v === '') return null + if (yesIsh(v) || v === 'PAID' || v === 'COMPLETED') return 'live' + if (v === 'NEW PROJECT') return 'installing' + if (v.startsWith('WAITING') || v === 'GENERAL ENQUIRY' || v === 'GENERAL ENQUIERY') return 'quoted' + return null +} + +/** Project rows may only UPGRADE an existing assignment (policy 5, second clause). */ +function projectUpgrade(raw: string): AssignStatus | null { + const v = raw.toUpperCase() + if (v === 'COMPLETED') return 'live' + if (v === 'NEW PROJECT' || v.startsWith('WAITING')) return 'installing' + return null +} -function mapTicketStatus(status: string, dropped: string): PlannedTicket['status'] { +/** + * Ticket status pattern mapping over the 46 raw APEX values (policy 2 — IN THIS ORDER): + * contains CLOSED → closed; contains DROP or TICKET_DROPED='Y' → dropped; contains + * WAITING → waiting; contains TRANSFER (PENDING_TRANSfERED, 2,836 rows) → in_progress; + * contains PENDING → open; else open. + */ +function mapTicketStatus(status: string, dropped: string): TicketStatus { const s = status.toUpperCase() - if (s === 'CLOSED') return 'closed' - if (s === 'OPEN') return 'open' + if (s.includes('CLOSED')) return 'closed' + if (s.includes('DROP') || dropped.toUpperCase() === 'Y') return 'dropped' if (s.includes('WAITING')) return 'waiting' - if (dropped.toUpperCase() === 'Y') return 'dropped' + if (s.includes('TRANSFER')) return 'in_progress' + if (s.includes('PENDING')) return 'open' return 'open' } -/** Set-or-replace one labeled detail (skip empty values, replace an existing label). */ +/** Set-or-replace one labeled detail (skip junk/empty values, replace an existing label). */ function setDetail(a: PlannedAssignment, label: string, value: string): void { - if (value === '') return + if (junkDetail(value)) return const existing = a.details.find((d) => d.label === label) if (existing !== undefined) existing.value = value else a.details.push({ label, value }) } +/** Count of non-empty cells — "richer row wins" for duplicate dimension rows (policy 9). */ +function richness(r: Record): number { + return Object.values(r).filter((v) => v !== '').length +} + /** * The single validation + mapping pass shared by stage and commit. Reads the DB - * (existing codes/doc numbers/state codes) but NEVER writes. Rows with problems - * are reported and excluded from the plan; commit refuses while any exist. + * (existing codes/doc numbers/state codes) but NEVER writes. */ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise { const ourState = await companyStateCode(db) const report = Object.fromEntries( - APEX_FILES.map((s) => [s, { staged: wb[s].length, problems: [] as string[] }]), + APEX_FILES.map((s) => [s, { staged: wb[s].length, problems: [] as string[], notes: [] as string[] }]), ) as FullImportReport const plan: Plan = { report, ourState, clients: [], assignments: [], branches: [], docs: [], tickets: [], - interactions: [], currentFy: fyOf(new Date().toISOString().slice(0, 10)), seedTail: null, + interactions: [], projectCounts: { sms_projects: 0, rtgs_projects: 0 }, + currentFy: fyOf(new Date().toISOString().slice(0, 10)), seedTail: null, } const todayIso = new Date().toISOString().slice(0, 10) const now = new Date().toISOString() @@ -295,7 +397,7 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise [c.code, c])) const existingByName = new Map() - for (const c of existing) if (!existingByName.has(c.name)) existingByName.set(c.name, c.id) + for (const c of existing) if (!existingByName.has(c.name.toUpperCase())) existingByName.set(c.name.toUpperCase(), c.id) const existingDocNos = new Set( (await db.all<{ doc_no: string }>(`SELECT doc_no FROM document WHERE doc_no IS NOT NULL`)) .map((r) => r.doc_no), @@ -309,23 +411,35 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise + planByName.get(name.toUpperCase()) ?? existingByName.get(name.toUpperCase()) + /** Skip-note text for an unresolvable/test client ref (policy 7: skip, never stub). */ + const refSkip = (col: string, code: string): string => + TEST_SIDS.has(code) + ? `${col}=${code} is the APEX test client — skipped` + : `unknown client ${col}=${code} — skipped (never stubbed)` const assignmentByKey = new Map() - const ensureAssignment = (clientId: string, moduleCode: string): PlannedAssignment => { + const ensureAssignment = (clientId: string, moduleCode: string, status: AssignStatus): PlannedAssignment => { const key = `${clientId}|${moduleCode}` let a = assignmentByKey.get(key) if (a === undefined) { a = { - id: uuidv7(), clientId, moduleCode, + id: uuidv7(), clientId, moduleCode, status, provider: null, username: null, passwordEnc: null, details: [], remark: null, - serviceTouched: false, + installedOn: null, serviceTouched: false, } assignmentByKey.set(key, a) + } else if (STATUS_RANK[status] > STATUS_RANK[a.status]) { + a.status = status // signals only ever upgrade, never downgrade } return a } + const findAssignment = (clientId: string, moduleCode: string): PlannedAssignment | undefined => + assignmentByKey.get(`${clientId}|${moduleCode}`) // ----- clients (SID_NO is the cross-file join key → client.code) ----- + let pwNoKey = 0 for (let i = 0; i < wb.clients.length; i++) { const r = wb.clients[i]! const probs: string[] = [] @@ -335,20 +449,21 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise 0) { report.clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) continue } + const phone = noData(r.PHONE) ? '' : r.PHONE + const mail = noData(r.MAIL) ? '' : r.MAIL const contacts: PlannedClient['contacts'] = [] - if (r.PHONE !== '' || r.MAIL !== '') { + if (phone !== '' || mail !== '') { contacts.push({ name: r.BANK, - ...(r.PHONE !== '' ? { phone: r.PHONE } : {}), - ...(r.MAIL !== '' ? { email: r.MAIL } : {}), + ...(phone !== '' ? { phone } : {}), + ...(mail !== '' ? { email: mail } : {}), }) } if (r.SECRETARY_CONTACT_DETAILS !== '') { @@ -356,259 +471,538 @@ async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise 0) { + report.clients.problems.push( + `${pwNoKey} row(s) carry a plaintext DB_PASSWORD but HQ_SECRET_KEY is not set — commit refuses until the key is provided (values are never stored unencrypted)`, + ) } - // ----- branches (MID → client.code) ----- + // ----- branches (MID → client.code; byte-identical duplicate rows deduped) ----- + const seenBranchKeys = new Set() for (let i = 0; i < wb.branches.length; i++) { const r = wb.branches[i]! - const probs: string[] = [] const c = r.MID === '' ? undefined : resolveClient(r.MID) - if (r.MID === '') probs.push('missing MID') - else if (c === undefined) probs.push(`unknown client MID=${r.MID}`) - if (r.BRANCH === '') probs.push('missing BRANCH') - if (probs.length > 0) { - report.branches.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + if (r.MID === '' || c === undefined) { + report.branches.notes.push(`row ${i + 1}: ${r.MID === '' ? 'blank MID — skipped' : refSkip('MID', r.MID)}`) + continue + } + if (r.BRANCH === '') { report.branches.problems.push(`row ${i + 1}: missing BRANCH`); continue } + const key = `${r.MID}|${r.BRANCH}|${r.BRCODE}` + if (seenBranchKeys.has(key)) { + report.branches.notes.push(`row ${i + 1}: duplicate branch row for MID=${r.MID} ('${r.BRANCH}') — deduped`) continue } - plan.branches.push({ id: uuidv7(), clientId: c!.id, name: r.BRANCH, code: orNull(r.BRCODE) }) + seenBranchKeys.add(key) + plan.branches.push({ id: uuidv7(), clientId: c.id, name: r.BRANCH, code: orNull(r.BRCODE) }) } - // ----- bills (documents + payments; legacy numbers kept, series seeded past them) ----- + // ----- bills: LINE grain → group by INVOICE_NO (269 rows = 151 documents) ----- + // GST pseudo-lines (DESCRIPTION='GST' or PER_QTY like '18%') carry the tax as AMT. + // Three eras (policy 1): (a) GST line present → tax = GST-line AMT sum; (b) no GST + // line but GST_AMOUNT populated (2026 era, where INVOICE_AMOUNT flips meaning to + // gross — never trust it; AMT is the base) → tax = GST_AMOUNT, total = TOTAL_AMOUNT; + // (c) no GST anywhere → tax 0 (pre-GST/DLT history; history docs bypass computeBill + // by design). total fallback = taxable + tax. const TYPE_MAP: Record = { 'INVOICE': 'INVOICE', 'PROFORMA INVOICE': 'PROFORMA', 'QUOTATION': 'QUOTATION', } - const seenDocNos = new Set() + interface BillLine { row: number; r: ApexBillRow; isGst: boolean } + const billGroups = new Map() for (let i = 0; i < wb.bills.length; i++) { const r = wb.bills[i]! + const rowNo = i + 1 + if (r.BANK_ID !== '' && TEST_SIDS.has(r.BANK_ID)) { + report.bills.notes.push(`row ${rowNo}: ${refSkip('BANK_ID', r.BANK_ID)}${r.INVOICE_NO !== '' ? ` (doc ${r.INVOICE_NO})` : ''}`) + continue + } + if (r.INVOICE_NO === '') { report.bills.problems.push(`row ${rowNo}: missing INVOICE_NO`); continue } + const isGst = r.DESCRIPTION.toUpperCase() === 'GST' || r.PER_QTY.includes('%') + const group = billGroups.get(r.INVOICE_NO) ?? [] + group.push({ row: rowNo, r, isGst }) + billGroups.set(r.INVOICE_NO, group) + } + for (const [docNo, lines] of billGroups) { const probs: string[] = [] - const c = r.BANK_ID === '' ? undefined : resolveClient(r.BANK_ID) - if (r.BANK_ID === '') probs.push('missing BANK_ID') - else if (c === undefined) probs.push(`unknown client BANK_ID=${r.BANK_ID}`) - const docType = TYPE_MAP[r.INVOICE_TYPE.toUpperCase()] - if (docType === undefined) probs.push(`unknown INVOICE_TYPE: ${r.INVOICE_TYPE}`) - const docNo = r.INVOICE_NO - if (docNo === '') probs.push('missing INVOICE_NO') - else if (seenDocNos.has(docNo) || existingDocNos.has(docNo)) probs.push(`duplicate doc_no: ${docNo}`) - if (docNo !== '') seenDocNos.add(docNo) - const docDate = parseApexDate(r.BILL_DATE) - if (docDate === null) probs.push(`bad BILL_DATE: ${r.BILL_DATE}`) - const taxable = rupeesToPaise(r.INVOICE_AMOUNT) - if (taxable === null) probs.push(`bad INVOICE_AMOUNT: ${r.INVOICE_AMOUNT}`) - const tax = r.GST_AMOUNT === '' ? 0 : rupeesToPaise(r.GST_AMOUNT) - if (tax === null) probs.push(`bad GST_AMOUNT: ${r.GST_AMOUNT}`) - let total: number | null = null - if (taxable !== null && tax !== null) { - total = r.TOTAL_AMOUNT === '' ? taxable + tax : rupeesToPaise(r.TOTAL_AMOUNT) - if (total === null) probs.push(`bad TOTAL_AMOUNT: ${r.TOTAL_AMOUNT}`) + const first = lines[0]!.r + const rowsRef = lines.map((l) => l.row).join(',') + // Cross-line consistency: BANK_ID / type / date must agree within a document. + for (const [label, vals] of [ + ['BANK_ID', new Set(lines.map((l) => l.r.BANK_ID))], + ['INVOICE_TYPE', new Set(lines.map((l) => l.r.INVOICE_TYPE.toUpperCase()))], + ['BILL_DATE', new Set(lines.map((l) => l.r.BILL_DATE.slice(0, 10)))], + ] as const) { + if (vals.size > 1) probs.push(`inconsistent ${label} across lines`) + } + const c = first.BANK_ID === '' ? undefined : resolveClient(first.BANK_ID) + if (first.BANK_ID === '' || c === undefined) { + report.bills.notes.push(`doc ${docNo} (rows ${rowsRef}): ${first.BANK_ID === '' ? 'blank BANK_ID — skipped' : refSkip('BANK_ID', first.BANK_ID)}`) + continue + } + const docType = TYPE_MAP[first.INVOICE_TYPE.toUpperCase()] + if (docType === undefined) probs.push(`unknown INVOICE_TYPE: ${first.INVOICE_TYPE}`) + if (existingDocNos.has(docNo)) probs.push(`doc_no already exists in HQ: ${docNo}`) + const docDate = parseApexDate(first.BILL_DATE) + if (docDate === null) probs.push(`bad BILL_DATE: ${first.BILL_DATE}`) + // Base lines, with the exact-duplicate-line dedupe (SMS/2026/10208 pattern: + // same SRNO + same AMT, blank/identical description → drop one, note it). + const baseLines: BillLine[] = [] + for (const l of lines.filter((x) => !x.isGst)) { + const dup = baseLines.find((k) => + k.r.SRNO === l.r.SRNO && k.r.AMT === l.r.AMT + && (l.r.DESCRIPTION === '' || l.r.DESCRIPTION === k.r.DESCRIPTION)) + if (dup !== undefined) { + report.bills.notes.push(`doc ${docNo}: exact-duplicate line (row ${l.row}, SRNO ${l.r.SRNO || '?'}) dropped`) + continue + } + baseLines.push(l) + } + if (baseLines.length === 0) probs.push('no base (non-GST) line') + let taxable = 0 + for (const l of baseLines) { + const amt = rupeesToPaise(l.r.AMT) + if (amt === null) { probs.push(`row ${l.row}: bad AMT: ${l.r.AMT}`); continue } + taxable += amt + } + let tax = 0 + const gstLines = lines.filter((x) => x.isGst) + if (gstLines.length > 0) { + for (const l of gstLines) { + const amt = rupeesToPaise(l.r.AMT) + if (amt === null) { probs.push(`row ${l.row}: bad GST-line AMT: ${l.r.AMT}`); continue } + tax += amt + } + } else { + const gstCol = lines.map((l) => l.r.GST_AMOUNT).find((v) => v !== '') + if (gstCol !== undefined) { + const t = rupeesToPaise(gstCol) + if (t === null) probs.push(`bad GST_AMOUNT: ${gstCol}`) + else tax = t + } + } + const totalCol = lines.map((l) => l.r.TOTAL_AMOUNT).find((v) => v !== '') + let total = taxable + tax + if (totalCol !== undefined) { + const t = rupeesToPaise(totalCol) + if (t === null) probs.push(`bad TOTAL_AMOUNT: ${totalCol}`) + else total = t } if (probs.length > 0) { - report.bills.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + report.bills.problems.push(...probs.map((p) => `doc ${docNo} (rows ${rowsRef}): ${p}`)) continue } - // GST split by place of supply (v1 pattern): intra-state → CGST+SGST, else IGST. - const intra = c!.state === ourState - const cgst = intra ? Math.floor(tax! / 2) : 0 - const sgst = intra ? tax! - cgst : 0 - const igst = intra ? 0 : tax! - const status = docType === 'INVOICE' - ? (r.PAYMENT_STATUS.toUpperCase() === 'PAID' ? 'paid' : 'sent') - : 'sent' - const payAmt = rupeesToPaise(r.PAYMENT_AMOUNT) - const payDate = parseApexDate(r.PAYMENT_DATE) - const payment = docType === 'INVOICE' && payAmt !== null && payAmt > 0 && payDate !== null - ? { amount: Math.min(payAmt, total!), receivedOn: payDate } - : null + // GST split by place of supply (v1 pattern). APEX has no client state column, so + // every imported client carries the company state → all docs split CGST+SGST. + const intra = c.state === ourState + const cgst = intra ? Math.floor(tax / 2) : 0 + const sgst = intra ? tax - cgst : 0 + const igst = intra ? 0 : tax + const paid = lines.some((l) => isPaid(l.r.PAYMENT_STATUS)) + const status = docType === 'INVOICE' ? (paid ? 'paid' : 'sent') : 'sent' + // Payments: distinct (date, amount) pairs across the doc's lines — header fields + // repeat per line (same pair twice = ONE payment), while genuinely different + // pairs are real split payments (SMS/2025/10133: 17,500 + 3,150). Allocation + // clamps to the remaining payable (policy 8) so overpayments and cross-bill + // lump-sum stamps cannot inflate a document past settled. + const payments: PlannedDoc['payments'] = [] + if (docType === 'INVOICE') { + const pairs = new Map() + for (const l of lines) { + const amt = rupeesToPaise(l.r.PAYMENT_AMOUNT) + const date = parseApexDate(l.r.PAYMENT_DATE) + if (amt !== null && amt > 0 && date !== null) pairs.set(`${date}|${amt}`, { amount: amt, receivedOn: date }) + } + let remaining = total + for (const p of [...pairs.values()].sort((x, y) => x.receivedOn.localeCompare(y.receivedOn))) { + const alloc = Math.min(p.amount, remaining) + if (alloc <= 0) { + report.bills.notes.push(`doc ${docNo}: extra payment pair on ${p.receivedOn} exceeds payable — skipped`) + continue + } + payments.push({ amount: alloc, receivedOn: p.receivedOn }) + remaining -= alloc + } + if (paid && payments.length === 0) { + report.bills.notes.push(`doc ${docNo}: marked PAID with no payment date/amount — status 'paid', no payment row`) + } + } + // Real line descriptions + NOTES memos survive in payload.notes (audit: 67 rows + // carry reconciliation memos with no other destination). + const noteBits: string[] = [] + for (const l of baseLines) if (l.r.DESCRIPTION !== '' && !noteBits.includes(l.r.DESCRIPTION)) noteBits.push(l.r.DESCRIPTION) + for (const l of lines) if (l.r.NOTES !== '' && !noteBits.includes(l.r.NOTES)) noteBits.push(l.r.NOTES) + const billTo: PlannedDoc['billTo'] = + first.BANK_NAME !== '' || first.ADDRESS !== '' || !noData(first.MAIL) + ? { + ...(first.BANK_NAME !== '' ? { name: first.BANK_NAME } : {}), + ...(first.ADDRESS !== '' ? { address: first.ADDRESS } : {}), + ...(!noData(first.MAIL) ? { email: first.MAIL } : {}), + } + : null plan.docs.push({ - id: uuidv7(), docType: docType!, docNo, fy: fyOf(docDate!), clientId: c!.id, + id: uuidv7(), docType: docType!, docNo, fy: fyOf(docDate!), clientId: c.id, docDate: docDate!, status, - taxable: taxable!, cgst, sgst, igst, roundOff: total! - taxable! - tax!, total: total!, - payment, + taxable, cgst, sgst, igst, roundOff: total - taxable - tax, total, + payments, notes: noteBits.length > 0 ? noteBits.join('\n') : null, billTo, }) } - // Series seed (v1 logic): continue INVOICE numbering after the last current-FY - // legacy number — prefixes differ, so only the numeric tail matters. + if (plan.docs.length > 0) { + report.bills.notes.push( + `${plan.docs.length} document(s) split GST as CGST+SGST — APEX has no client state; company state ${ourState} assumed for all`, + ) + } + // Series seed (policy 10): QT/PI/INV share ONE 'SMS/YYYY/NNNNN' space in APEX, so + // the INVOICE series continues after the max numeric tail across ALL current-FY docs. const tails = plan.docs - .filter((d) => d.docType === 'INVOICE' && d.fy === plan.currentFy) + .filter((d) => d.fy === plan.currentFy) .map((d) => /(\d+)$/.exec(d.docNo)?.[1]) .filter((t): t is string => t !== undefined) .map((t) => Number(t)) plan.seedTail = tails.length > 0 ? Math.max(...tails) : null - // ----- tickets (BANK_ID → code; blank BANK_ID falls back to exact BANK_NAME) ----- - for (let i = 0; i < wb.tickets.length; i++) { - const r = wb.tickets[i]! - const probs: string[] = [] - let clientId: string | undefined - if (r.BANK_ID !== '') { - const c = resolveClient(r.BANK_ID) - if (c === undefined) probs.push(`unknown client BANK_ID=${r.BANK_ID}`) - else clientId = c.id - } else if (r.BANK_NAME !== '') { - clientId = planByName.get(r.BANK_NAME) ?? existingByName.get(r.BANK_NAME) - if (clientId === undefined) probs.push(`no client named '${r.BANK_NAME}'`) - } else probs.push('missing BANK_ID and BANK_NAME') - const openedRaw = r.ASSIGN_DATE !== '' ? r.ASSIGN_DATE : (r.DAY_DT !== '' ? r.DAY_DT : r.CREATED) - const openedOn = parseApexDate(openedRaw) - if (openedOn === null) probs.push(`no parseable open date (ASSIGN_DATE='${r.ASSIGN_DATE}', DAY_DT='${r.DAY_DT}', CREATED='${r.CREATED}')`) - if (probs.length > 0) { - report.tickets.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) - continue + // ----- tickets: collapse OLD_ID chains (4,567 rows = ~1,857 threads) ----- + // APEX's day-workbench re-created pending tickets daily; superseded carry-forward + // copies chain via OLD_ID (policy 3). One ticket per chain: terminal row wins + // status, earliest CREATED wins opened_on. Per-row import would inflate the open + // queue ~13x (2,909 phantom open tickets vs ~219 real). + { + const byId = new Map() + for (const r of wb.tickets) if (r.ID !== '') byId.set(r.ID, r) + const chainKeyOf = (r: ApexTicketRow): string => { + let cur = r + const seen = new Set([cur.ID]) + while (cur.OLD_ID !== '' && cur.OLD_ID !== cur.ID) { + if (seen.has(cur.OLD_ID)) return cur.OLD_ID + const next = byId.get(cur.OLD_ID) + if (next === undefined) return cur.OLD_ID // dangling root — key by the referenced id + seen.add(cur.OLD_ID) + cur = next + } + return cur.ID + } + const chains = new Map() + for (let i = 0; i < wb.tickets.length; i++) { + const r = wb.tickets[i]! + const key = r.ID === '' ? `blank#${i}` : chainKeyOf(r) + const members = chains.get(key) ?? [] + members.push({ row: i + 1, r }) + chains.set(key, members) + } + let superseded = 0 + for (const members of chains.values()) { + members.sort((a, b) => a.r.CREATED.localeCompare(b.r.CREATED)) + superseded += members.length - 1 + const terminals = members.filter((m) => { + const s = mapTicketStatus(m.r.STATUS, m.r.TICKET_DROPED) + return s === 'closed' || s === 'dropped' + }) + const rep = terminals.length > 0 ? terminals[terminals.length - 1]! : members[members.length - 1]! + const firstM = members[0]! + // Client: first chain member with a resolvable BANK_ID, else exact BANK_NAME. + let clientId: string | undefined + let skipNote: string | undefined + for (const m of members) { + if (m.r.BANK_ID !== '') { + const c = resolveClient(m.r.BANK_ID) + if (c !== undefined) { clientId = c.id; break } + skipNote = refSkip('BANK_ID', m.r.BANK_ID) + } else if (m.r.BANK_NAME !== '') { + const byName = resolveByName(m.r.BANK_NAME) + if (byName !== undefined) { clientId = byName; break } + skipNote = `no client named '${m.r.BANK_NAME}' — skipped` + } + } + if (clientId === undefined) { + report.tickets.notes.push(`rows ${members.map((m) => m.row).join(',')}: ${skipNote ?? 'no BANK_ID/BANK_NAME on any chain row — skipped'}`) + continue + } + const openedRaw = firstM.r.CREATED !== '' ? firstM.r.CREATED + : (firstM.r.ASSIGN_DATE !== '' ? firstM.r.ASSIGN_DATE : firstM.r.DAY_DT) + const openedOn = parseApexDate(openedRaw) + if (openedOn === null) { + report.tickets.problems.push(`row ${firstM.row}: no parseable open date (CREATED='${firstM.r.CREATED}', ASSIGN_DATE='${firstM.r.ASSIGN_DATE}', DAY_DT='${firstM.r.DAY_DT}')`) + continue + } + const status = mapTicketStatus(rep.r.STATUS, rep.r.TICKET_DROPED) + // MOBILE_NO is the caller callback number (the ticket table has no phone column + // — policy 3 appends it to the description); MAIL='Y' rows had an attached mail + // whose BLOB the export lost — the marker preserves that it existed (in APEX). + let description = rep.r.DESCRIPTION + const mobile = members.map((m) => m.r.MOBILE_NO).find((v) => v !== '') + if (mobile !== undefined) description += `\n[contact: ${mobile}]` + if (members.some((m) => m.r.MAIL.toUpperCase() === 'Y')) description += '\n[mail attachment retained in APEX]' + const oo = rep.r.ONLINE_OFFLINE.toUpperCase() + plan.tickets.push({ + id: uuidv7(), clientId, + moduleCode: orNull(rep.r.GROUP), kind: rep.r.TYPE, description: description.trim(), + status, + onlineOffline: oo === 'ONLINE' || oo === 'OFFLINE' ? oo.toLowerCase() : null, + openedOn, + closedOn: status === 'closed' || status === 'dropped' ? parseApexDate(rep.r.UPDATED) : null, + createdAt: firstM.r.CREATED !== '' ? firstM.r.CREATED : now, + // BRID is blank on all 4,567 rows and APEX numeric staff ids have no HQ + // mapping — branch_id and assigned_to stay NULL by design. + }) + } + if (superseded > 0) { + report.tickets.notes.push(`collapsed ${superseded} superseded OLD_ID chain rows into ${plan.tickets.length} ticket thread(s)`) } - const status = mapTicketStatus(r.STATUS, r.TICKET_DROPED) - const oo = r.ONLINE_OFFLINE.toUpperCase() - plan.tickets.push({ - id: uuidv7(), clientId: clientId!, - moduleCode: orNull(r.GROUP), kind: r.TYPE, description: r.DESCRIPTION, status, - onlineOffline: oo === 'ONLINE' || oo === 'OFFLINE' ? oo.toLowerCase() : null, - openedOn: openedOn!, - closedOn: status === 'closed' || status === 'dropped' ? parseApexDate(r.UPDATED) : null, - createdAt: r.CREATED !== '' ? r.CREATED : now, - // BRID has no reliable branch join and APEX staff ids don't map to HQ - // staff — branch_id and assigned_to stay NULL by design. - }) } // ----- SMS service data (sms_clients: CID → code → the client's SMS module) ----- - const seenSmsCids = new Set() - for (let i = 0; i < wb.sms_clients.length; i++) { - const r = wb.sms_clients[i]! - const probs: string[] = [] - const c = r.CID === '' ? undefined : resolveClient(r.CID) - if (r.CID === '') probs.push('missing CID') - else if (c === undefined) probs.push(`unknown client CID=${r.CID}`) - else if (seenSmsCids.has(r.CID)) probs.push(`duplicate CID: ${r.CID}`) - if (r.CID !== '') seenSmsCids.add(r.CID) - let passwordEnc: string | null = null - if (r.PASSWORD !== '') { - if (keyHex === '') { - probs.push('PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted') - } else passwordEnc = encrypt(r.PASSWORD, keyHex) + // Duplicate CID rows: richer row wins, junk twin dropped (policy 9 — CID 33's + // 'WHATSAPP' enquiry row). The SMS column carries pipeline vocabulary (policy 5). + { + const byCid = new Map() + let pwRowsNoKey = 0 + for (let i = 0; i < wb.sms_clients.length; i++) { + const r = wb.sms_clients[i]! + if (r.CID === '') { report.sms_clients.problems.push(`row ${i + 1}: missing CID`); continue } + const list = byCid.get(r.CID) ?? [] + list.push({ row: i + 1, r }) + byCid.set(r.CID, list) } - if (probs.length > 0) { - report.sms_clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) - continue + for (const [cid, rows] of byCid) { + rows.sort((a, b) => richness(b.r) - richness(a.r)) + const kept = rows[0]! + for (const dropped of rows.slice(1)) { + report.sms_clients.notes.push(`row ${dropped.row}: duplicate CID=${cid} — dropped in favour of richer row ${kept.row}`) + } + const r = kept.r + const c = TEST_SIDS.has(cid) ? undefined : resolveClient(cid) + if (c === undefined) { report.sms_clients.notes.push(`row ${kept.row}: ${refSkip('CID', cid)}`); continue } + let passwordEnc: string | null = null + if (r.PASSWORD !== '') { + if (keyHex === '') pwRowsNoKey++ + else passwordEnc = encrypt(r.PASSWORD, keyHex) + } + const signal = moduleSignal(r.SMS) + let a = findAssignment(c.id, 'SMS') + if (signal !== null) a = ensureAssignment(c.id, 'SMS', signal) + if (a === undefined) { + report.sms_clients.notes.push(`row ${kept.row}: SMS='${r.SMS}' gives no module signal and no assignment exists — row skipped`) + continue + } + a.provider = orNull(r.SMS_PROVIDER) + a.username = orNull(r.USERNAME) + a.passwordEnc = passwordEnc + if (!noData(r.REMARK)) a.remark = r.REMARK + a.installedOn = parseApexDate(r.INSTALLATION_DATE) ?? a.installedOn + // SMS_BALANCE was repurposed as a flag ('YES' on 74 rows) — only numbers land. + if (/^\d+(\.\d+)?$/.test(r.SMS_BALANCE)) setDetail(a, 'SMS balance', r.SMS_BALANCE) + setDetail(a, 'Reseller', r.RESELLER) + setDetail(a, 'Features', r.ADDED_FEATURES) + setDetail(a, 'Updation', r.UPDATION) + if (isPaid(r.PAYMENT)) setDetail(a, 'Payment', 'paid') + setDetail(a, 'Phone', r.PHONE) + a.serviceTouched = true + } + if (pwRowsNoKey > 0) { + report.sms_clients.problems.push( + `${pwRowsNoKey} row(s) carry a plaintext portal PASSWORD but HQ_SECRET_KEY is not set — commit refuses until the key is provided (values are never stored unencrypted)`, + ) } - // Flag said NO but this table has a row → data wins, create the assignment. - const a = ensureAssignment(c!.id, 'SMS') - a.provider = orNull(r.SMS_PROVIDER) - a.username = orNull(r.USERNAME) - a.passwordEnc = passwordEnc - a.remark = orNull(r.REMARK) - setDetail(a, 'SMS balance', r.SMS_BALANCE) - setDetail(a, 'Reseller', r.RESELLER) - setDetail(a, 'Features', r.ADDED_FEATURES) - setDetail(a, 'Updation', r.UPDATION) - setDetail(a, 'Payment', r.PAYMENT) - setDetail(a, 'Phone', r.PHONE) - setDetail(a, 'Installed', r.INSTALLATION_DATE) - a.serviceTouched = true } - // ----- SMS alert types (sms_modules: group by SID_NO, HEAD where ACTIV='Y') ----- - const alertsByClient = new Map() - for (let i = 0; i < wb.sms_modules.length; i++) { - const r = wb.sms_modules[i]! - const c = r.SID_NO === '' ? undefined : resolveClient(r.SID_NO) - if (c === undefined) { - report.sms_modules.problems.push(`row ${i + 1}: ${r.SID_NO === '' ? 'missing SID_NO' : `unknown client SID_NO=${r.SID_NO}`}`) - continue + // ----- SMS alert types (sms_modules: HEAD where ACTIV='Y', + the PKG codepath and + // per-alert REMARK deviation notes — policy 6 says neither may be dropped) ----- + { + const alertsByClient = new Map() + for (let i = 0; i < wb.sms_modules.length; i++) { + const r = wb.sms_modules[i]! + const c = r.SID_NO === '' || TEST_SIDS.has(r.SID_NO) ? undefined : resolveClient(r.SID_NO) + if (c === undefined) { + report.sms_modules.notes.push(`row ${i + 1}: ${r.SID_NO === '' ? 'blank SID_NO — skipped' : refSkip('SID_NO', r.SID_NO)}`) + continue + } + if (r.ACTIV.toUpperCase() !== 'Y' || r.HEAD === '') continue + const entry = alertsByClient.get(c.id) ?? { heads: [], notes: [] } + entry.heads.push(r.PKG !== '' ? `${r.HEAD} (${r.PKG})` : r.HEAD) + if (r.REMARK !== '') entry.notes.push(`${r.HEAD}: ${r.REMARK}`) + alertsByClient.set(c.id, entry) } - if (r.ACTIV.toUpperCase() === 'Y' && r.HEAD !== '') { - const list = alertsByClient.get(c.id) ?? [] - list.push(r.HEAD) - alertsByClient.set(c.id, list) + for (const [clientId, entry] of alertsByClient) { + // Live alert config proves a live install even where the clients-list flag said NO. + const a = ensureAssignment(clientId, 'SMS', 'live') + setDetail(a, 'Active alerts', entry.heads.join(', ')) + if (entry.notes.length > 0) setDetail(a, 'Alert notes', entry.notes.join('; ')) + a.serviceTouched = true } } - for (const [clientId, heads] of alertsByClient) { - const a = ensureAssignment(clientId, 'SMS') - setDetail(a, 'Active alerts', heads.join(', ')) - a.serviceTouched = true - } - // ----- RTGS service data (rtgs_clients: SID_NO → code → the RTGS module) ----- - const seenRtgsSids = new Set() - for (let i = 0; i < wb.rtgs_clients.length; i++) { - const r = wb.rtgs_clients[i]! - const probs: string[] = [] - const c = r.SID_NO === '' ? undefined : resolveClient(r.SID_NO) - if (r.SID_NO === '') probs.push('missing SID_NO') - else if (c === undefined) probs.push(`unknown client SID_NO=${r.SID_NO}`) - else if (seenRtgsSids.has(r.SID_NO)) probs.push(`duplicate SID_NO: ${r.SID_NO}`) - if (r.SID_NO !== '') seenRtgsSids.add(r.SID_NO) - if (probs.length > 0) { - report.rtgs_clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) - continue + // ----- RTGS service data (rtgs_clients: duplicate SID_NO engagements merged) ----- + { + const bySid = new Map() + for (let i = 0; i < wb.rtgs_clients.length; i++) { + const r = wb.rtgs_clients[i]! + if (r.SID_NO === '') { report.rtgs_clients.problems.push(`row ${i + 1}: missing SID_NO`); continue } + const list = bySid.get(r.SID_NO) ?? [] + list.push({ row: i + 1, r }) + bySid.set(r.SID_NO, list) + } + for (const [sid, rows] of bySid) { + rows.sort((a, b) => richness(b.r) - richness(a.r)) + let r = rows[0]!.r + if (rows.length > 1) { + // Merge duplicate engagements field-wise, richer row preferred (policy 9). + const merged = { ...r } + for (const other of rows.slice(1)) { + for (const k of RTGS_CLIENT_H) if (merged[k] === '') merged[k] = other.r[k] + } + r = merged + report.rtgs_clients.notes.push( + `rows ${rows.map((x) => x.row).join(',')}: duplicate SID_NO=${sid} merged (non-empty fields, richer row preferred)`, + ) + } + const c = TEST_SIDS.has(sid) ? undefined : resolveClient(sid) + if (c === undefined) { report.rtgs_clients.notes.push(`row ${rows[0]!.row}: ${refSkip('SID_NO', sid)}`); continue } + const signal = moduleSignal(r.RTGS) + let a = findAssignment(c.id, 'RTGS') + if (signal !== null) a = ensureAssignment(c.id, 'RTGS', signal) + if (a === undefined) { + report.rtgs_clients.notes.push(`row ${rows[0]!.row}: RTGS='${r.RTGS}' gives no module signal and no assignment exists — row skipped`) + continue + } + setDetail(a, 'IP', r.ADDRESS_IP) + setDetail(a, 'IP code', r.IP_CODE) + setDetail(a, 'E-collection code', r.E_COLLECTION_CODE) + setDetail(a, 'ABB', r.ABB) + setDetail(a, 'Partner bank', r.INTEGRATED_BANK_PARTNERS) + setDetail(a, 'QR', r.GENERATE_QR) + setDetail(a, 'SBI inward report', r.SBI_INWARD_REPORT) + setDetail(a, 'Passbook program', r.PASSBOOK_PROGRAM) + setDetail(a, 'IFSC 2025', r.IFSC_2025) + setDetail(a, 'Notification', r.NOTIFICATION) + setDetail(a, 'Misc RTGS outward', r.MISC_RTGS_OUTWARD) + setDetail(a, 'Phone', r.PHONE_NO) + // INSTALLATION_DATE is the go-live date → the typed installed_on column (audit + // verdict); INSTALLED_ON is server-location text ('Database Server') → detail. + a.installedOn = parseApexDate(r.INSTALLATION_DATE) ?? a.installedOn + setDetail(a, 'Installed on', r.INSTALLED_ON) + if (isPaid(r.PAYMENT)) setDetail(a, 'Payment', 'paid') + if (!noData(r.REMARK)) a.remark = r.REMARK + a.serviceTouched = true } - const a = ensureAssignment(c!.id, 'RTGS') - setDetail(a, 'IP', r.ADDRESS_IP) - setDetail(a, 'IP code', r.IP_CODE) - setDetail(a, 'E-collection code', r.E_COLLECTION_CODE) - setDetail(a, 'ABB', r.ABB) - setDetail(a, 'Partner bank', r.INTEGRATED_BANK_PARTNERS) - setDetail(a, 'QR', r.GENERATE_QR) - setDetail(a, 'SBI inward report', r.SBI_INWARD_REPORT) - setDetail(a, 'Passbook program', r.PASSBOOK_PROGRAM) - setDetail(a, 'IFSC 2025', r.IFSC_2025) - setDetail(a, 'Notification', r.NOTIFICATION) - setDetail(a, 'Misc RTGS outward', r.MISC_RTGS_OUTWARD) - setDetail(a, 'Phone', r.PHONE_NO) - setDetail(a, 'Installed', r.INSTALLED_ON !== '' ? r.INSTALLED_ON : r.INSTALLATION_DATE) - setDetail(a, 'Payment', r.PAYMENT) - a.remark = orNull(r.REMARK) - a.serviceTouched = true } - // ----- project lists → interaction rows ----- - const planProjects = ( - rows: { CID: string; DT_INFO: string; TICKET_STATUS: string; DESCRIPTION: string }[], - tag: string, section: 'sms_projects' | 'rtgs_projects', - ): void => { - for (let i = 0; i < rows.length; i++) { - const r = rows[i]! - const c = r.CID === '' ? undefined : resolveClient(r.CID) - if (c === undefined) { - report[section].problems.push(`row ${i + 1}: ${r.CID === '' ? 'missing CID' : `unknown client CID=${r.CID}`}`) - continue + // ----- project lists → interactions (history) + tickets (live work) ----- + // Audit verdict (projects TICKET_STATUS, major): non-closed rows WITHOUT a TID are + // the open SMS/RTGS work queue and are NOT recovered by the ticket import — they + // land as ticket rows; everything else lands as interaction history (rows WITH a + // TID echo tickets the ticket import already lands). The latest project row per + // client+module may upgrade the assignment status (policy 5). + { + const upgrades = new Map() + const planProjects = ( + rows: { CID: string; DT_INFO: string; TICKET_STATUS: string; DESCRIPTION: string; PHONE: string; ASSIGNED_TO: string; TID: string; RNO: string; moduleValue: string }[], + moduleCode: 'SMS' | 'RTGS', section: 'sms_projects' | 'rtgs_projects', + ): void => { + const seenRows = new Set() + for (let i = 0; i < rows.length; i++) { + const r = rows[i]! + const rowNo = i + 1 + if (r.CID === '') { report[section].notes.push(`row ${rowNo}: blank CID — skipped`); continue } + const c = TEST_SIDS.has(r.CID) ? undefined : resolveClient(r.CID) + if (c === undefined) { report[section].notes.push(`row ${rowNo}: ${refSkip('CID', r.CID)}`); continue } + const dupKey = [r.CID, r.RNO, r.DT_INFO, r.TICKET_STATUS, r.DESCRIPTION].join('|') + if (seenRows.has(dupKey)) { + report[section].notes.push(`row ${rowNo}: byte-identical duplicate project row — deduped`) + continue + } + seenRows.add(dupKey) + const parsed = parseApexDate(r.DT_INFO) + const onDate = parsed ?? todayIso + const up = projectUpgrade(r.moduleValue) + if (up !== null) { + const key = `${c.id}|${moduleCode}` + const prev = upgrades.get(key) + if (prev === undefined || onDate.localeCompare(prev.onDate) >= 0) upgrades.set(key, { onDate, status: up }) + } + const markers: string[] = [] + if (r.ASSIGNED_TO !== '' && r.ASSIGNED_TO !== '1') markers.push(`[assigned: ${r.ASSIGNED_TO}]`) + if (r.PHONE !== '') markers.push(`[contact: ${r.PHONE}]`) + if (parsed === null && r.DT_INFO !== '') markers.push(`(DT_INFO unparsed: ${r.DT_INFO})`) + const status = mapTicketStatus(r.TICKET_STATUS, '') + if (r.TID === '' && status !== 'closed' && status !== 'dropped') { + // Open project work with no APEX ticket behind it — lands in the live queue. + plan.tickets.push({ + id: uuidv7(), clientId: c.id, moduleCode, kind: 'PROJECT', + description: [r.DESCRIPTION, ...markers].filter((s) => s !== '').join('\n').trim(), + status, onlineOffline: null, openedOn: onDate, closedOn: null, createdAt: now, + }) + plan.projectCounts[section]++ + continue + } + const bits = [r.TICKET_STATUS, r.DESCRIPTION].filter((s) => s !== '') + const tag = `[${moduleCode} project]` + let notes = bits.length > 0 ? `${tag} ${bits.join(' — ')}` : tag + if (markers.length > 0) notes += ` ${markers.join(' ')}` + plan.interactions.push({ id: uuidv7(), clientId: c.id, onDate, notes, section }) + plan.projectCounts[section]++ } - const parsed = parseApexDate(r.DT_INFO) - const bits = [r.TICKET_STATUS, r.DESCRIPTION].filter((s) => s !== '') - let notes = bits.length > 0 ? `${tag} ${bits.join(' — ')}` : tag - if (parsed === null && r.DT_INFO !== '') notes += ` (DT_INFO unparsed: ${r.DT_INFO})` - plan.interactions.push({ - id: uuidv7(), clientId: c.id, onDate: parsed ?? todayIso, notes, section, - }) + } + planProjects(wb.sms_projects.map((r) => ({ ...r, moduleValue: r.SMS })), 'SMS', 'sms_projects') + planProjects(wb.rtgs_projects.map((r) => ({ ...r, moduleValue: r.RTGS })), 'RTGS', 'rtgs_projects') + for (const [key, u] of upgrades) { + const a = assignmentByKey.get(key) + if (a !== undefined && STATUS_RANK[u.status] > STATUS_RANK[a.status]) a.status = u.status } } - planProjects(wb.sms_projects, '[SMS project]', 'sms_projects') - planProjects(wb.rtgs_projects, '[RTGS project]', 'rtgs_projects') plan.assignments = [...assignmentByKey.values()] return plan } +function summarize(plan: Plan): StageSummary { + return { + clients: plan.clients.length, + client_modules: plan.assignments.length, + branches: plan.branches.length, + documents: plan.docs.length, + totalPayablePaise: plan.docs.reduce((s, d) => s + d.total, 0), + payments: plan.docs.reduce((s, d) => s + d.payments.length, 0), + tickets: plan.tickets.length, + interactions: plan.interactions.length, + currentFy: plan.currentFy, + seedTail: plan.seedTail, + } +} + // ---------- stage (pure validation — writes nothing) ---------- /** - * Validation pass over the whole workbook: per-section staged counts and per-row - * problems. Reads the DB for join/duplicate checks but writes nothing permanent - * (nothing at all, in fact — there are no v2 staging tables). + * Validation pass over the whole workbook: per-section staged counts, blocking + * problems, deliberate skip/merge notes, and a plan summary. Reads the DB for + * join/duplicate checks but writes nothing. A dry run needs no keyHex — plaintext + * passwords then surface as an aggregated blocking problem (commit needs the key). */ export async function stageApexFull( db: DB, files: Record, keyHex = '', -): Promise { +): Promise { const plan = await buildPlan(db, parseWorkbookCsvs(files), keyHex) - return plan.report + return { sections: plan.report, summary: summarize(plan) } } // ---------- commit (same validation, ONE transaction) ---------- @@ -636,9 +1030,9 @@ async function ensureModule(db: DB, code: string): Promise { } /** - * Re-run the validation and, only if EVERY section is clean, apply the whole - * workbook in one transaction. Auditing is one summary row per section (the - * deliberate volume call — 5k tickets would drown per-row audits). + * Re-run the validation and, only if EVERY section is problem-free (notes are the + * deliberate skips and do not block), apply the whole workbook in one transaction. + * Auditing is one summary row per section — 4.5k per-row audits would drown the log. */ export async function commitApexFull( db: DB, userId: string, files: Record, keyHex = '', @@ -647,7 +1041,7 @@ export async function commitApexFull( const plan = await buildPlan(db, wb, keyHex) const allProblems = APEX_FILES.flatMap((s) => plan.report[s].problems.map((p) => `${s} ${p}`)) if (allProblems.length > 0) { - throw new Error(`${allProblems.length} problem row(s) block the import — fix the CSVs and re-stage:\n${allProblems.join('\n')}`) + throw new Error(`${allProblems.length} problem(s) block the import — fix the CSVs and re-stage:\n${allProblems.join('\n')}`) } const now = new Date().toISOString() return db.transaction(async (): Promise => { @@ -677,16 +1071,16 @@ export async function commitApexFull( if (existingCm === undefined) { await db.run( `INSERT INTO client_module (id, client_id, module_id, status, kind, edition, active, - provider, username, password_enc, details, remark) - VALUES (?, ?, ?, 'live', 'yearly', 'standard', 1, ?, ?, ?, ?, ?)`, - a.id, a.clientId, moduleId, + installed_on, provider, username, password_enc, details, remark) + VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?)`, + a.id, a.clientId, moduleId, a.status, a.installedOn, a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, ) } else if (a.serviceTouched) { // Client pre-dated this import with a live assignment: service data wins. await db.run( - `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=? WHERE id=?`, - a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, existingCm.id, + `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=?, installed_on=? WHERE id=?`, + a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, a.installedOn, existingCm.id, ) } } @@ -699,7 +1093,8 @@ export async function commitApexFull( ) } - // Documents (history-only: empty payload lines, legacy numbers) + payments. + // Documents (history-only: empty payload lines, legacy numbers; payload carries + // the line/NOTES memos and the as-issued bill-to snapshot) + payments. let payments = 0 for (const d of plan.docs) { const totals = { @@ -707,6 +1102,11 @@ export async function commitApexFull( cgstPaise: d.cgst, sgstPaise: d.sgst, igstPaise: d.igst, cessPaise: 0, roundOffPaise: d.roundOff, payablePaise: d.total, savingsVsMrpPaise: 0, } + const payload = { + lines: [], totals, + ...(d.notes !== null ? { notes: d.notes } : {}), + ...(d.billTo !== null ? { billTo: d.billTo } : {}), + } await db.run( `INSERT INTO document (id, doc_type, doc_no, fy, client_id, doc_date, status, ref_doc_id, taxable_paise, cgst_paise, sgst_paise, igst_paise, round_off_paise, payable_paise, @@ -714,26 +1114,27 @@ export async function commitApexFull( VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, 'apex', ?, ?)`, d.id, d.docType, d.docNo, d.fy, d.clientId, d.docDate, d.status, d.taxable, d.cgst, d.sgst, d.igst, d.roundOff, d.total, - JSON.stringify({ lines: [], totals }), userId, now, + JSON.stringify(payload), userId, now, ) - if (d.payment !== null) { + for (const p of d.payments) { const paymentId = uuidv7() await db.run( `INSERT INTO payment (id, client_id, received_on, mode, reference, amount_paise, tds_paise, created_by, created_at) VALUES (?, ?, ?, 'bank', 'APEX import', ?, 0, ?, ?)`, - paymentId, d.clientId, d.payment.receivedOn, d.payment.amount, userId, now, + paymentId, d.clientId, p.receivedOn, p.amount, userId, now, ) await db.run( `INSERT INTO payment_allocation (id, payment_id, document_id, amount_paise) VALUES (?, ?, ?, ?)`, - uuidv7(), paymentId, d.id, d.payment.amount, + uuidv7(), paymentId, d.id, p.amount, ) payments++ } } - // Series seed: continue INVOICE numbering after the last current-FY legacy number. + // Series seed: QT/PI/INV shared one APEX number space — continue the INVOICE + // series after the max current-FY tail across ALL imported docs (policy 10). let seeded: FullCommitResult['seeded'] = null if (plan.seedTail !== null) { await seedSeries(db, 'INVOICE', plan.currentFy, plan.seedTail) @@ -742,7 +1143,7 @@ export async function commitApexFull( { lastSeq: plan.seedTail }) } - // Tickets. + // Tickets (collapsed chains + open project work). for (const t of plan.tickets) { await db.run( `INSERT INTO ticket (id, client_id, branch_id, module_code, kind, description, status, @@ -753,7 +1154,7 @@ export async function commitApexFull( ) } - // Project lists → interaction rows under a get-or-create 'project' type. + // Project history → interaction rows under a get-or-create 'project' type. if (plan.interactions.length > 0) { // Portability quirk: ON CONFLICT upsert is SQLite/Postgres dialect (standard SQL has MERGE). await db.run( @@ -780,8 +1181,8 @@ export async function commitApexFull( sms_clients: plan.report.sms_clients.staged, sms_modules: plan.report.sms_modules.staged, rtgs_clients: plan.report.rtgs_clients.staged, - sms_projects: plan.interactions.filter((x) => x.section === 'sms_projects').length, - rtgs_projects: plan.interactions.filter((x) => x.section === 'rtgs_projects').length, + sms_projects: plan.projectCounts.sms_projects, + rtgs_projects: plan.projectCounts.rtgs_projects, seeded, } // One summary audit row per section that carried data — same txn as the writes. diff --git a/apps/hq/test/import-apex-full.test.ts b/apps/hq/test/import-apex-full.test.ts index 27b2f9c..946c270 100644 --- a/apps/hq/test/import-apex-full.test.ts +++ b/apps/hq/test/import-apex-full.test.ts @@ -1,4 +1,5 @@ -// apps/hq/test/import-apex-full.test.ts — D20 full-APEX importer v2 (stage → problems → one-txn commit) +// apps/hq/test/import-apex-full.test.ts — D20 full-APEX importer v2, reworked to the +// REAL export shape (bills = line grain, OLD_ID ticket chains, status vocabulary). import { describe, it, expect } from 'vitest' import { fyOf } from '@sims/domain' import { openDb, type DB } from '../src/db' @@ -25,75 +26,167 @@ const RTGS_CLIENT_HDR = 'SID_NO,BANK_NAME,ADDRESS_IP,PHONE_NO,RTGS,IP_CODE,E_COL const SMS_PROJECT_HDR = 'CID,BANK,DISTRICT,DT_INFO,RNO,SMS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,NEW_FORM,CREATED_BY,UPDATED_BY,TID' const RTGS_PROJECT_HDR = 'CID,BANK,DT_INFO,RNO,RTGS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,CREATED_BY,UPDATED_BY,TID' -/** A small but complete workbook exercising every section. */ +/** A small but complete workbook exercising the real-export shapes per section. */ function fullFiles(): Record { return { clients: csv(CLIENT_HDR, [ { SID_NO: '101', BANK: 'Acme Bank', ADDRESS: 'Kochi HQ', PHONE: '9876543210', MAIL: 'acme@bank.in', DISTRICT: 'Ernakulam', SMS: 'YES', RTGS: 'NO', WHATSAPP: 'Y', - MOBILE_APP: 'NO', DB_PASSWORD: 's3cret-db', ATM: 'NO', ANYDESK: '123 456 789', + DB_PASSWORD: 's3cret-db', ATM: 'NO', ANYDESK: '123 456 789', SECRETARY_CONTACT_DETAILS: 'Secretary John 9999', OS: 'Windows 10', SECTOR: 'Co-op', }, - { - SID_NO: '102', BANK: 'Beta Bank', DISTRICT: 'Kollam', SMS: 'NO', RTGS: 'YES', - WHATSAPP: 'NO', MOBILE_APP: 'NO', DB_PASSWORD: 'NO DATA', ATM: 'NO', - ANYDESK: 'NO DATA', SECTOR: 'Service', + { // 'NO DATA' placeholders filtered case-insensitively; district misspelling fixed + SID_NO: '102', BANK: 'Beta Bank', PHONE: 'NO DATA', DISTRICT: 'Pathanamthita', + SMS: 'NO', RTGS: 'YES', DB_PASSWORD: 'NO DATA', ANYDESK: 'No Data', SECTOR: 'Service', + }, + { // flag pollution: GENERAL ENQUIRY is NOT yes; ATM holds a provider name; + // MOBILE_APP holds a phone number + SID_NO: '103', BANK: 'Gamma Bank', SMS: 'GENERAL ENQUIRY', RTGS: 'NO', + ATM: 'EWIRE', MOBILE_APP: '8301932166', }, ]), branches: csv(BRANCH_HDR, [ - { ID: '1', BRANCH: 'Main Branch', MID: '101', BANK: 'Acme Bank', BRCODE: 'BR001' }, + { ID: '1', BRANCH: 'HEAD OFFICE', MID: '101', BANK: 'Acme Bank', BRCODE: '1' }, + { ID: '2', BRANCH: 'HEAD OFFICE', MID: '101', BANK: 'Acme Bank', BRCODE: '1' }, // dup → dedupe + { ID: '3', BRANCH: 'HEAD OFFICE', MID: '999', BANK: 'Ghost Bank', BRCODE: '1' }, // orphan → note ]), bills: csv(BILL_HDR, [ - { // paid current-FY invoice — rupees→paise, payment+allocation, series seed - BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '101', INVOICE_NO: 'SMS/2026/0042', - INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '1000', GST_AMOUNT: '180', TOTAL_AMOUNT: '1180', + // Era (a): base line + GST pseudo-line; payment fields repeated per line = ONE payment. + { + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '101', BANK_NAME: 'ACME BANK OLD NAME', + ADDRESS: 'Old Addr', MAIL: 'old@acme.in', INVOICE_NO: 'SMS/2026/10042', SRNO: '1', + DESCRIPTION: 'Bulk SMS 1L', AMT: '1000', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '1000', PAYMENT_STATUS: 'PAID', PAYMENT_DATE: `${TODAY} 00:00:00`, PAYMENT_AMOUNT: '1180', + NOTES: 'DLT renewal amount', TOTAL_AMOUNT: '1180', + }, + { + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '101', INVOICE_NO: 'SMS/2026/10042', SRNO: '2', + DESCRIPTION: 'GST', PER_QTY: '18%', AMT: '180', INVOICE_TYPE: 'INVOICE', + INVOICE_AMOUNT: '1000', PAYMENT_STATUS: 'PAID', PAYMENT_DATE: `${TODAY} 00:00:00`, + PAYMENT_AMOUNT: '1180', TOTAL_AMOUNT: '1180', + }, + // Era (b): no GST line, GST in columns; INVOICE_AMOUNT is GROSS here — AMT is the base. + { + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '102', INVOICE_NO: 'SMS/2026/10234', SRNO: '1', + DESCRIPTION: 'Software', AMT: '12500', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '14750', + PAYMENT_STATUS: 'N', GST_RATE: '18', GST_AMOUNT: '2250', TOTAL_AMOUNT: '14750', + }, + // Era (c): pre-GST history — no GST anywhere, tax 0, total = taxable. + { + BILL_DATE: '2024-11-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'SMS/2024/10010', SRNO: '1', + DESCRIPTION: 'AMC renewal', AMT: '500', INVOICE_TYPE: 'PROFORMA INVOICE', INVOICE_AMOUNT: '500', + }, + // Exact-duplicate line (10208 pattern): same SRNO+AMT, blank description → dropped. + { + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '102', INVOICE_NO: 'SMS/2026/10208', SRNO: '1', + DESCRIPTION: 'Renewal', AMT: '100', INVOICE_TYPE: 'INVOICE', PAYMENT_STATUS: 'N', + }, + { + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '102', INVOICE_NO: 'SMS/2026/10208', SRNO: '1', + DESCRIPTION: '', AMT: '100', INVOICE_TYPE: 'INVOICE', PAYMENT_STATUS: 'N', + }, + // Genuine per-line split payment (10133 pattern): two distinct (date, amount) pairs. + { + BILL_DATE: '2025-06-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'SMS/2025/10133', SRNO: '1', + DESCRIPTION: 'Websms', AMT: '175', INVOICE_TYPE: 'INVOICE', PAYMENT_STATUS: 'PAID', + PAYMENT_DATE: '2025-06-10 00:00:00', PAYMENT_AMOUNT: '175', + }, + { + BILL_DATE: '2025-06-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'SMS/2025/10133', SRNO: '2', + DESCRIPTION: 'Addon', AMT: '31.5', INVOICE_TYPE: 'INVOICE', PAYMENT_STATUS: 'PAID', + PAYMENT_DATE: '2025-06-20 00:00:00', PAYMENT_AMOUNT: '31.5', }, - { // proforma with '' tax ('→0) and '' total ('→taxable+tax); never gets a payment - BILL_DATE: '2025-08-18 00:00:00', BANK_ID: '102', INVOICE_NO: 'PI/2025/007', - INVOICE_TYPE: 'PROFORMA INVOICE', INVOICE_AMOUNT: '500', GST_AMOUNT: '', TOTAL_AMOUNT: '', + // PAID with no payment data (10140 pattern): status paid, NO payment row, note. + { + BILL_DATE: '2025-07-01 00:00:00', BANK_ID: '102', INVOICE_NO: 'SMS/2025/10140', SRNO: '1', + DESCRIPTION: 'Small job', AMT: '59', INVOICE_TYPE: 'INVOICE', PAYMENT_STATUS: 'PAID', }, ]), tickets: csv(TICKET_HDR, [ + // One OLD_ID chain: two superseded PENDING_TRANSfERED copies + terminal CLOSED row. + { + ID: '1', OLD_ID: '1', BANK_ID: '101', STATUS: 'PENDING_TRANSfERED', + CREATED: '2025-01-01 09:00:00', ASSIGN_DATE: '2025-01-01 00:00:00', + DESCRIPTION: 'Printer issue', MOBILE_NO: '9400000001', + }, { - ID: '1', BANK_ID: '101', TYPE: 'MODIFICATION', GROUP: 'SMS', - ASSIGN_DATE: '2025-02-28 00:00:00', DESCRIPTION: 'Fix template', STATUS: 'CLOSED', - CREATED: '2025-02-28 09:00:00', UPDATED: '2025-03-02 10:00:00', ONLINE_OFFLINE: 'ONLINE', + ID: '2', OLD_ID: '1', BANK_ID: '101', STATUS: 'PENDING_TRANSfERED', + CREATED: '2025-01-02 09:00:00', DESCRIPTION: 'Printer issue', + }, + { + ID: '3', OLD_ID: '2', BANK_ID: '101', STATUS: 'CLOSED', CREATED: '2025-01-03 09:00:00', + UPDATED: '2025-01-04 10:00:00', DESCRIPTION: 'Printer issue - fixed', MAIL: 'Y', + TYPE: 'MODIFICATION', GROUP: 'SMS', ONLINE_OFFLINE: 'ONLINE', + }, + { // blank BANK_ID → exact BANK_NAME match; WAITING* → waiting + ID: '10', OLD_ID: '10', BANK_NAME: 'Acme Bank', STATUS: 'WAITING FOR CONFIRMATION', + CREATED: '2025-03-01 08:00:00', DESCRIPTION: 'Waiting one', + }, + { // 'DROPPED BY …' free text + flag → dropped, closed_on from UPDATED + ID: '11', OLD_ID: '11', BANK_ID: '102', STATUS: 'DROPPED BY DEVIKA-01-01-2025', + TICKET_DROPED: 'Y', CREATED: '2025-04-01 09:00:00', UPDATED: '2025-04-02 00:00:00', + DESCRIPTION: 'Dropped one', }, - { // blank BANK_ID → exact BANK_NAME match; DD-MON-YYYY open date; WAITING* status - ID: '2', BANK_NAME: 'Acme Bank', DAY_DT: '01-MAR-2025', - STATUS: 'WAITING FOR CONFIRMATION', DESCRIPTION: 'Waiting one', + { // orphan client ref → skipped with a note, never a blocking problem + ID: '12', OLD_ID: '12', BANK_ID: '999', STATUS: 'OPEN', + CREATED: '2025-05-01 00:00:00', DESCRIPTION: 'Orphan one', }, - { // TICKET_DROPED='Y' with a non-terminal status → dropped, closed_on from UPDATED - ID: '3', BANK_ID: '102', STATUS: '', TICKET_DROPED: 'Y', DESCRIPTION: 'Dropped one', - CREATED: '2025-04-01 09:00:00', UPDATED: '2025-04-02 00:00:00', + { // 'PENDING AS ON ' → open + ID: '13', OLD_ID: '13', BANK_ID: '102', STATUS: 'PENDING AS ON 20-02-2026', + CREATED: '2025-06-01 07:00:00', DESCRIPTION: 'Pending one', }, ]), sms_clients: csv(SMS_CLIENT_HDR, [ - { // CID 102's SMS flag said NO — this row wins and creates the assignment - RNO: '1', CID: '102', BANK: 'Beta Bank', USERNAME: 'betauser', PASSWORD: 'smsPass9', - SMS_PROVIDER: 'AltSMS', PHONE: '0474222', SMS_BALANCE: '5000', REMARK: 'ok', - RESELLER: 'RSL', PAYMENT: 'PAID 2025', INSTALLATION_DATE: '10-NOV-2023', + { // junk duplicate (CID 33 pattern) — fewer fields, dropped in favour of the richer row + RNO: '1', CID: '102', BANK: 'Beta Bank', SMS: 'GENERAL ENQUIRY', + USERNAME: 'WHATSAPP', PASSWORD: 'WHATSAPP', + }, + { + RNO: '2', CID: '102', BANK: 'Beta Bank', SMS: 'YES', USERNAME: 'betauser', + PASSWORD: 'smsPass9', SMS_PROVIDER: 'AltSMS', PHONE: '0474222', SMS_BALANCE: 'YES', + REMARK: 'ok', RESELLER: 'RSL', PAYMENT: 'PAID', INSTALLATION_DATE: '2026-03-25 00:00:00', + }, + { // pipeline vocabulary: GENERAL ENQUIRY → assignment exists at 'quoted' + RNO: '3', CID: '103', BANK: 'Gamma Bank', SMS: 'GENERAL ENQUIRY', }, ]), sms_modules: csv(SMS_MODULE_HDR, [ - { RNO: '1', SID_NO: '102', HEAD: 'ATM ALERT', ACTIV: 'Y' }, - { RNO: '2', SID_NO: '102', HEAD: 'BALANCE ALERT', ACTIV: 'Y' }, - { RNO: '3', SID_NO: '102', HEAD: 'OLD ALERT', ACTIV: 'N' }, + { RNO: '1', SID_NO: '102', HEAD: 'ATM ALERT', ACTIV: 'Y', PKG: 'PKG_SMS_JOB' }, + { RNO: '2', SID_NO: '102', HEAD: 'BALANCE ALERT', ACTIV: 'Y', REMARK: 'emp only' }, + { RNO: '3', SID_NO: '102', HEAD: 'OLD ALERT', ACTIV: 'N', PKG: 'PKG_SMS_JOB' }, ]), rtgs_clients: csv(RTGS_CLIENT_HDR, [ - { + { // richer engagement row SID_NO: '102', BANK_NAME: 'Beta Bank', ADDRESS_IP: '10.0.0.5', PHONE_NO: '333', - IP_CODE: 'IPC1', ABB: 'BETA', INTEGRATED_BANK_PARTNERS: 'SBI', - INSTALLED_ON: '2024-01-05 00:00:00', GENERATE_QR: 'YES', PAYMENT: 'PAID', + RTGS: 'YES', IP_CODE: 'IPC1', ABB: 'BETA', INTEGRATED_BANK_PARTNERS: 'SBI', + INSTALLATION_DATE: '2024-01-05 00:00:00', INSTALLED_ON: 'Database Server', + GENERATE_QR: 'YES', PAYMENT: 'PAID', + }, + { // sparse duplicate engagement (SID 84/91/205 pattern) → merged, richer preferred + SID_NO: '102', BANK_NAME: 'Beta Bank', RTGS: 'NEW PROJECT', PAYMENT: 'Nil', + E_COLLECTION_CODE: 'EC77', }, ]), sms_projects: csv(SMS_PROJECT_HDR, [ - { CID: '101', DT_INFO: '13-05-2024', TICKET_STATUS: 'COMPLETED', DESCRIPTION: 'Install SMS' }, + { // TID-bearing closed row → interaction history (ticket import owns the TID) + CID: '101', DT_INFO: '13-05-2024', SMS: 'COMPLETED', TICKET_STATUS: 'CLOSED', + DESCRIPTION: 'Install SMS', ASSIGNED_TO: 'DEVIKA', TID: '54055184000000000000000000000000000001', + }, + { // TID-less OPEN row → lands as a ticket (the live queue must survive cutover) + CID: '103', DT_INFO: '08/29/2024', SMS: 'YES', TICKET_STATUS: 'OPEN', + DESCRIPTION: 'Onboard SMS', PHONE: '9037779234', + }, + { // NEW PROJECT vocabulary upgrades 103's quoted SMS assignment to 'installing' + CID: '103', DT_INFO: '12-13-2023', SMS: 'NEW PROJECT', TICKET_STATUS: 'CLOSED', + DESCRIPTION: 'Site survey', TID: '54055184000000000000000000000000000002', + }, ]), rtgs_projects: csv(RTGS_PROJECT_HDR, [ - { CID: '102', DT_INFO: 'not-a-date', TICKET_STATUS: 'PENDING', DESCRIPTION: 'RTGS setup' }, + { // TID-less PENDING_TRANSfERED → in_progress ticket; unparseable date noted + CID: '102', DT_INFO: 'not-a-date', RTGS: 'YES', TICKET_STATUS: 'PENDING_TRANSfERED', + DESCRIPTION: 'RTGS setup', + }, ]), } } @@ -104,183 +197,211 @@ async function freshDb(): Promise { return db } +const cmFor = async (db: DB, clientCode: string, moduleCode: string) => + db.get<{ + status: string; kind: string; provider: string | null; username: string | null + password_enc: string | null; details: string; remark: string | null; installed_on: string | null + }>( + `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark, cm.installed_on + FROM client_module cm JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id + WHERE c.code=? AND m.code=?`, clientCode, moduleCode) + describe('parseApexDate', () => { - it('handles every format seen in the APEX export', () => { + it('handles every format profiled in the APEX export (separator decides day/month)', () => { expect(parseApexDate('2025-08-18 00:00:00')).toBe('2025-08-18') // ISO datetime expect(parseApexDate('10-NOV-2023 10:57:18')).toBe('2023-11-10') // DD-MON-YYYY - expect(parseApexDate('08/29/2024')).toBe('2024-08-29') // second >12 → MM/DD - expect(parseApexDate('13-05-2024')).toBe('2024-05-13') // first >12 → DD-MM - expect(parseApexDate('10-11-2023')).toBe('2023-11-10') // ambiguous → assume DD-MM + expect(parseApexDate('08/29/2024')).toBe('2024-08-29') // slash → MM/DD + expect(parseApexDate('09/03/2024')).toBe('2024-09-03') // slash ambiguous → MM/DD + expect(parseApexDate('13-05-2024')).toBe('2024-05-13') // dash → DD-MM + expect(parseApexDate('10-11-2023')).toBe('2023-11-10') // dash ambiguous → DD-MM + expect(parseApexDate('12-13-2023')).toBe('2023-12-13') // dash with month>12 → falls back MM-DD + expect(parseApexDate('06-12-2024 ')).toBe('2024-12-06') // trailing whitespace trimmed expect(parseApexDate('')).toBeNull() expect(parseApexDate('not-a-date')).toBeNull() expect(parseApexDate('99-99-2024')).toBeNull() }) }) -describe('APEX full importer v2', () => { - it('stage: clean workbook → per-section counts, zero problems, and NO writes', async () => { +describe('APEX full importer v2 (real-export shapes)', () => { + it('stage: clean workbook → zero problems, deliberate skips as notes, and NO writes', async () => { const db = await freshDb() - const report = await stageApexFull(db, fullFiles(), KEY) - expect(report.clients).toEqual({ staged: 2, problems: [] }) - expect(report.branches).toEqual({ staged: 1, problems: [] }) - expect(report.bills).toEqual({ staged: 2, problems: [] }) - expect(report.tickets).toEqual({ staged: 3, problems: [] }) - expect(report.sms_clients).toEqual({ staged: 1, problems: [] }) - expect(report.sms_modules).toEqual({ staged: 3, problems: [] }) - expect(report.rtgs_clients).toEqual({ staged: 1, problems: [] }) - expect(report.sms_projects).toEqual({ staged: 1, problems: [] }) - expect(report.rtgs_projects).toEqual({ staged: 1, problems: [] }) - // Pure validation: no clients, no documents, no import audit rows. + const { sections, summary } = await stageApexFull(db, fullFiles(), KEY) + for (const s of Object.values(sections)) expect(s.problems).toEqual([]) + // Deliberate auto-resolutions surface as notes, not problems. + expect(sections.branches.notes.some((n) => n.includes('MID=999'))).toBe(true) + expect(sections.branches.notes.some((n) => n.includes('duplicate branch row'))).toBe(true) + expect(sections.bills.notes.some((n) => n.includes('exact-duplicate line'))).toBe(true) + expect(sections.bills.notes.some((n) => n.includes('marked PAID with no payment'))).toBe(true) + expect(sections.bills.notes.some((n) => n.includes('CGST+SGST'))).toBe(true) + expect(sections.tickets.notes.some((n) => n.includes('BANK_ID=999'))).toBe(true) + expect(sections.tickets.notes.some((n) => n.includes('collapsed 2 superseded'))).toBe(true) + expect(sections.sms_clients.notes.some((n) => n.includes('duplicate CID=102'))).toBe(true) + expect(sections.rtgs_clients.notes.some((n) => n.includes('duplicate SID_NO=102 merged'))).toBe(true) + expect(summary).toMatchObject({ + clients: 3, client_modules: 7, branches: 1, + documents: 6, totalPayablePaise: 1679550, payments: 3, + tickets: 6, interactions: 2, currentFy: FY, seedTail: 10234, + }) + // Pure validation: nothing written at all. expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) - expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ticket`))!.n).toBe(0) expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM audit_log WHERE action='import'`))!.n).toBe(0) }) - it('stage: unknown join keys are per-row problems with row numbers', async () => { - const db = await freshDb() - const report = await stageApexFull(db, { - clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), - branches: csv(BRANCH_HDR, [{ ID: '9', BRANCH: 'Ghost', MID: '999' }]), - tickets: csv(TICKET_HDR, [{ ID: '9', BANK_ID: '999', STATUS: 'OPEN', DAY_DT: '2025-01-01 00:00:00' }]), - sms_projects: csv(SMS_PROJECT_HDR, [{ CID: '999', DT_INFO: '2025-01-01' }]), - }, KEY) - expect(report.branches.problems).toEqual(['row 1: unknown client MID=999']) - expect(report.tickets.problems).toEqual(['row 1: unknown client BANK_ID=999']) - expect(report.sms_projects.problems).toEqual(['row 1: unknown client CID=999']) - }) - it('commit: full happy path lands every section in one transaction', async () => { const db = await freshDb() const res = await commitApexFull(db, 'u-test', fullFiles(), KEY) - expect(res.clients).toBe(2) - expect(res.client_modules).toBe(4) // 101: SMS+WHATSAPP (flags); 102: RTGS (flag) + SMS (data wins) - expect(res.branches).toBe(1) - expect(res.bills).toBe(2) - expect(res.payments).toBe(1) - expect(res.tickets).toBe(3) - expect(res.sms_projects).toBe(1) - expect(res.rtgs_projects).toBe(1) - expect(res.seeded).toEqual({ fy: FY, lastSeq: 42 }) + expect(res).toMatchObject({ + clients: 3, client_modules: 7, branches: 1, bills: 6, payments: 3, tickets: 6, + sms_projects: 3, rtgs_projects: 1, seeded: { fy: FY, lastSeq: 10234 }, + }) - // Clients: code = SID_NO, contacts carry bank + secretary, support fields ride along. + // Clients: hygiene (district normalize, NO DATA filters) + contacts + encrypted password. const c101 = (await db.get<{ - id: string; name: string; address: string; district: string; sector: string - os: string; anydesk: string; contacts: string; status: string; source: string - state_code: string; db_password_enc: string - }>(`SELECT * FROM client WHERE code='101'`))! - expect(c101).toMatchObject({ - name: 'Acme Bank', address: 'Kochi HQ', district: 'Ernakulam', sector: 'Co-op', - os: 'Windows 10', anydesk: '123 456 789', status: 'active', source: 'apex', state_code: '32', - }) + id: string; district: string; contacts: string; db_password_enc: string; source: string + }>(`SELECT id, district, contacts, db_password_enc, source FROM client WHERE code='101'`))! + expect(c101.district).toBe('ERNAKULAM') + expect(c101.source).toBe('apex') expect(JSON.parse(c101.contacts)).toEqual([ { name: 'Acme Bank', phone: '9876543210', email: 'acme@bank.in' }, { name: 'Secretary John 9999', role: 'secretary' }, ]) - // DB password encrypted at rest, decryptable with the key. expect(decrypt(c101.db_password_enc, KEY)).toBe('s3cret-db') - const c102 = (await db.get<{ anydesk: string | null; db_password_enc: string | null }>( - `SELECT anydesk, db_password_enc FROM client WHERE code='102'`))! - expect(c102).toEqual({ anydesk: null, db_password_enc: null }) // 'NO DATA' → nothing stored + const c102 = (await db.get<{ anydesk: string | null; db_password_enc: string | null; district: string; contacts: string }>( + `SELECT anydesk, db_password_enc, district, contacts FROM client WHERE code='102'`))! + expect(c102.anydesk).toBeNull() // 'No Data' → nothing stored + expect(c102.db_password_enc).toBeNull() + expect(c102.district).toBe('PATHANAMTHITTA') // misspelling normalized + expect(JSON.parse(c102.contacts)).toEqual([]) // PHONE 'NO DATA' filtered - // Flags → live yearly assignments on get-or-created modules. - const modules = await db.all<{ code: string; name: string }>( - `SELECT code, name FROM module WHERE code IN ('SMS','RTGS','WHATSAPP') ORDER BY code`) - expect(modules).toEqual([ - { code: 'RTGS', name: 'RTGS' }, { code: 'SMS', name: 'Bulk SMS' }, - { code: 'WHATSAPP', name: 'WhatsApp' }, - ]) - const codes101 = await db.all<{ code: string; status: string; kind: string }>( - `SELECT m.code, cm.status, cm.kind FROM client_module cm - JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id - WHERE c.code='101' ORDER BY m.code`) - expect(codes101).toEqual([ - { code: 'SMS', status: 'live', kind: 'yearly' }, - { code: 'WHATSAPP', status: 'live', kind: 'yearly' }, - ]) + // Flags per policy 4: GENERAL ENQUIRY is not a module; ATM value → provider; + // MOBILE_APP phone number → Contact detail. + const atm = (await cmFor(db, '103', 'ATM'))! + expect(atm.status).toBe('live') + expect(atm.provider).toBe('EWIRE') + const mob = (await cmFor(db, '103', 'MOBILEAPP'))! + expect(JSON.parse(mob.details)).toContainEqual({ label: 'Contact', value: '8301932166' }) - // Branch joined by MID. - const branch = await db.get(`SELECT b.name, b.code, b.active FROM client_branch b - JOIN client c ON c.id=b.client_id WHERE c.code='101'`) - expect(branch).toEqual({ name: 'Main Branch', code: 'BR001', active: 1 }) + // Status vocabulary: 103's SMS came from GENERAL ENQUIRY (quoted) and the latest + // project row (NEW PROJECT) upgraded it to installing — never to live. + const sms103 = (await cmFor(db, '103', 'SMS'))! + expect(sms103.status).toBe('installing') - // Bills: rupees→paise, intra-state split, payment + allocation, series seed. - const inv = (await db.get<{ - id: string; doc_type: string; fy: string; status: string; taxable_paise: number - cgst_paise: number; sgst_paise: number; igst_paise: number; round_off_paise: number - payable_paise: number; source: string; payload: string - }>(`SELECT * FROM document WHERE doc_no='SMS/2026/0042'`))! - expect(inv).toMatchObject({ - doc_type: 'INVOICE', fy: FY, status: 'paid', taxable_paise: 100000, - cgst_paise: 9000, sgst_paise: 9000, igst_paise: 0, round_off_paise: 0, - payable_paise: 118000, source: 'apex', - }) - expect(JSON.parse(inv.payload)).toMatchObject({ lines: [], totals: { payablePaise: 118000 } }) - const pi = await db.get(`SELECT doc_type, status, taxable_paise, payable_paise, cgst_paise - FROM document WHERE doc_no='PI/2025/007'`) - expect(pi).toEqual({ - doc_type: 'PROFORMA', status: 'sent', taxable_paise: 50000, payable_paise: 50000, cgst_paise: 0, - }) - const pay = (await db.get<{ id: string; mode: string; reference: string; amount_paise: number; received_on: string }>( - `SELECT * FROM payment`))! - expect(pay).toMatchObject({ - mode: 'bank', reference: 'APEX import', amount_paise: 118000, received_on: TODAY, - }) - const alloc = await db.get(`SELECT payment_id, document_id, amount_paise FROM payment_allocation`) - expect(alloc).toEqual({ payment_id: pay.id, document_id: inv.id, amount_paise: 118000 }) - const series = await db.get(`SELECT next_seq FROM doc_series WHERE doc_type='INVOICE' AND fy=?`, FY) - expect(series).toEqual({ next_seq: 43 }) // continues after the legacy 0042 tail - - // Tickets: status mapping + join fallbacks. - const t1 = await db.get(`SELECT status, opened_on, closed_on, online_offline, module_code, source - FROM ticket WHERE description='Fix template'`) - expect(t1).toEqual({ - status: 'closed', opened_on: '2025-02-28', closed_on: '2025-03-02', - online_offline: 'online', module_code: 'SMS', source: 'apex', - }) - const t2 = (await db.get<{ status: string; opened_on: string; client_id: string }>( - `SELECT status, opened_on, client_id FROM ticket WHERE description='Waiting one'`))! - expect(t2.status).toBe('waiting') - expect(t2.opened_on).toBe('2025-03-01') // DD-MON-YYYY DAY_DT - expect(t2.client_id).toBe(c101.id) // blank BANK_ID resolved by exact BANK_NAME - const t3 = await db.get(`SELECT status, opened_on, closed_on FROM ticket WHERE description='Dropped one'`) - expect(t3).toEqual({ status: 'dropped', opened_on: '2025-04-01', closed_on: '2025-04-02' }) - - // SMS service data landed on 102's (data-created) SMS module. - const sms102 = (await db.get<{ provider: string; username: string; password_enc: string; details: string; remark: string }>( - `SELECT cm.provider, cm.username, cm.password_enc, cm.details, cm.remark FROM client_module cm - JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id - WHERE c.code='102' AND m.code='SMS'`))! + // SMS service data on 102 (richer duplicate row won; junk 'WHATSAPP' row dropped). + const sms102 = (await cmFor(db, '102', 'SMS'))! + expect(sms102.status).toBe('live') expect(sms102.provider).toBe('AltSMS') expect(sms102.username).toBe('betauser') - expect(decrypt(sms102.password_enc, KEY)).toBe('smsPass9') + expect(decrypt(sms102.password_enc!, KEY)).toBe('smsPass9') expect(sms102.remark).toBe('ok') + expect(sms102.installed_on).toBe('2026-03-25') // typed column, not a detail const smsDetails = JSON.parse(sms102.details) as { label: string; value: string }[] - expect(smsDetails).toContainEqual({ label: 'SMS balance', value: '5000' }) - expect(smsDetails).toContainEqual({ label: 'Installed', value: '10-NOV-2023' }) - expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT, BALANCE ALERT' }) + expect(smsDetails.find((d) => d.label === 'SMS balance')).toBeUndefined() // 'YES' is a flag, not a balance + expect(smsDetails).toContainEqual({ label: 'Payment', value: 'paid' }) + expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT (PKG_SMS_JOB), BALANCE ALERT' }) + expect(smsDetails).toContainEqual({ label: 'Alert notes', value: 'BALANCE ALERT: emp only' }) - // RTGS service data on 102's RTGS module. - const rtgs102 = (await db.get<{ details: string }>( - `SELECT cm.details FROM client_module cm - JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id - WHERE c.code='102' AND m.code='RTGS'`))! + // RTGS: duplicate engagements merged richer-first; INSTALLATION_DATE → installed_on; + // INSTALLED_ON server text → detail; Nil payment did not overwrite PAID. + const rtgs102 = (await cmFor(db, '102', 'RTGS'))! + expect(rtgs102.status).toBe('live') + expect(rtgs102.installed_on).toBe('2024-01-05') const rtgsDetails = JSON.parse(rtgs102.details) as { label: string; value: string }[] expect(rtgsDetails).toContainEqual({ label: 'IP', value: '10.0.0.5' }) - expect(rtgsDetails).toContainEqual({ label: 'Installed', value: '2024-01-05 00:00:00' }) - expect(rtgsDetails).toContainEqual({ label: 'QR', value: 'YES' }) + expect(rtgsDetails).toContainEqual({ label: 'Installed on', value: 'Database Server' }) + expect(rtgsDetails).toContainEqual({ label: 'E-collection code', value: 'EC77' }) // merged from sparse row + expect(rtgsDetails).toContainEqual({ label: 'Payment', value: 'paid' }) - // Projects → interaction rows; unparseable DT_INFO falls back to today + a note. + // Bills, era (a): grouped lines → ONE document; GST from the pseudo-line; ONE payment. + const inv = (await db.get<{ + id: string; taxable_paise: number; cgst_paise: number; sgst_paise: number + payable_paise: number; status: string; payload: string; fy: string + }>(`SELECT id, taxable_paise, cgst_paise, sgst_paise, payable_paise, status, payload, fy + FROM document WHERE doc_no='SMS/2026/10042'`))! + expect(inv).toMatchObject({ + taxable_paise: 100000, cgst_paise: 9000, sgst_paise: 9000, payable_paise: 118000, + status: 'paid', fy: FY, + }) + const payload = JSON.parse(inv.payload) as { notes?: string; billTo?: object; totals: { payablePaise: number } } + expect(payload.notes).toBe('Bulk SMS 1L\nDLT renewal amount') + expect(payload.billTo).toEqual({ name: 'ACME BANK OLD NAME', address: 'Old Addr', email: 'old@acme.in' }) + // Era (b): taxable keyed on AMT (12500), never INVOICE_AMOUNT (14750 gross). + const eraB = await db.get(`SELECT taxable_paise, cgst_paise, sgst_paise, payable_paise, status + FROM document WHERE doc_no='SMS/2026/10234'`) + expect(eraB).toEqual({ + taxable_paise: 1250000, cgst_paise: 112500, sgst_paise: 112500, payable_paise: 1475000, status: 'sent', + }) + // Era (c): pre-GST history, zero tax. + const eraC = await db.get(`SELECT doc_type, taxable_paise, cgst_paise, payable_paise + FROM document WHERE doc_no='SMS/2024/10010'`) + expect(eraC).toEqual({ doc_type: 'PROFORMA', taxable_paise: 50000, cgst_paise: 0, payable_paise: 50000 }) + // Duplicate line dropped: taxable is one line, not two. + const dup = await db.get(`SELECT taxable_paise, payable_paise FROM document WHERE doc_no='SMS/2026/10208'`) + expect(dup).toEqual({ taxable_paise: 10000, payable_paise: 10000 }) + // Split payment: two payment rows + allocations, fully settling the doc. + const splitDoc = (await db.get<{ id: string; payable_paise: number }>( + `SELECT id, payable_paise FROM document WHERE doc_no='SMS/2025/10133'`))! + const allocs = await db.all<{ amount_paise: number }>( + `SELECT amount_paise FROM payment_allocation WHERE document_id=? ORDER BY amount_paise`, splitDoc.id) + expect(allocs).toEqual([{ amount_paise: 3150 }, { amount_paise: 17500 }]) + expect(splitDoc.payable_paise).toBe(20650) + // PAID with no data: status paid, zero payment rows. + const noPay = (await db.get<{ id: string; status: string }>( + `SELECT id, status FROM document WHERE doc_no='SMS/2025/10140'`))! + expect(noPay.status).toBe('paid') + expect((await db.get<{ n: number }>( + `SELECT COUNT(*) AS n FROM payment_allocation WHERE document_id=?`, noPay.id))!.n).toBe(0) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM payment`))!.n).toBe(3) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(6) + // Series: shared APEX number space → INVOICE series seeds from max tail of ALL + // current-FY docs (10042 / 10208 / 10234 → 10234). + const series = await db.get(`SELECT next_seq FROM doc_series WHERE doc_type='INVOICE' AND fy=?`, FY) + expect(series).toEqual({ next_seq: 10235 }) + + // Tickets: the OLD_ID chain collapsed to ONE closed ticket with markers. + const chain = (await db.get<{ + status: string; opened_on: string; closed_on: string; description: string + kind: string; module_code: string; online_offline: string; created_at: string + }>(`SELECT status, opened_on, closed_on, description, kind, module_code, online_offline, created_at + FROM ticket WHERE description LIKE 'Printer issue - fixed%'`))! + expect(chain.status).toBe('closed') + expect(chain.opened_on).toBe('2025-01-01') // earliest CREATED in the chain + expect(chain.closed_on).toBe('2025-01-04') // terminal row's UPDATED + expect(chain.kind).toBe('MODIFICATION') + expect(chain.module_code).toBe('SMS') + expect(chain.online_offline).toBe('online') + expect(chain.created_at).toBe('2025-01-01 09:00:00') + expect(chain.description).toBe('Printer issue - fixed\n[contact: 9400000001]\n[mail attachment retained in APEX]') + // Status mapping: WAITING* / DROP* / PENDING AS ON. + const waiting = (await db.get<{ status: string; client_id: string }>( + `SELECT status, client_id FROM ticket WHERE description='Waiting one'`))! + expect(waiting.status).toBe('waiting') + expect(waiting.client_id).toBe(c101.id) // blank BANK_ID resolved by exact BANK_NAME + const droppedT = await db.get(`SELECT status, closed_on FROM ticket WHERE description='Dropped one'`) + expect(droppedT).toEqual({ status: 'dropped', closed_on: '2025-04-02' }) + const pending = await db.get(`SELECT status FROM ticket WHERE description='Pending one'`) + expect(pending).toEqual({ status: 'open' }) // PENDING AS ON → open + expect(await db.get(`SELECT id FROM ticket WHERE description='Orphan one'`)).toBeUndefined() + // Open project rows without a TID landed as tickets (the live queue survives). + const projT = (await db.get<{ status: string; kind: string; module_code: string; opened_on: string }>( + `SELECT status, kind, module_code, opened_on FROM ticket WHERE description LIKE 'Onboard SMS%'`))! + expect(projT).toMatchObject({ status: 'open', kind: 'PROJECT', module_code: 'SMS', opened_on: '2024-08-29' }) + const projR = (await db.get<{ status: string; module_code: string; description: string }>( + `SELECT status, module_code, description FROM ticket WHERE description LIKE 'RTGS setup%'`))! + expect(projR.status).toBe('in_progress') // PENDING_TRANSfERED → in_progress + expect(projR.module_code).toBe('RTGS') + expect(projR.description).toContain('(DT_INFO unparsed: not-a-date)') + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ticket`))!.n).toBe(6) + + // TID-bearing / closed project rows became interaction history with markers. const sp = (await db.get<{ on_date: string; notes: string; type_code: string }>( - `SELECT on_date, notes, type_code FROM interaction WHERE notes LIKE '[SMS project]%'`))! + `SELECT on_date, notes, type_code FROM interaction WHERE notes LIKE '%Install SMS%'`))! expect(sp.type_code).toBe('project') expect(sp.on_date).toBe('2024-05-13') - expect(sp.notes).toBe('[SMS project] COMPLETED — Install SMS') - const rp = (await db.get<{ on_date: string; notes: string }>( - `SELECT on_date, notes FROM interaction WHERE notes LIKE '[RTGS project]%'`))! - expect(rp.on_date).toBe(TODAY) - expect(rp.notes).toContain('DT_INFO unparsed: not-a-date') + expect(sp.notes).toBe('[SMS project] CLOSED — Install SMS [assigned: DEVIKA]') + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM interaction WHERE type_code='project'`))!.n).toBe(2) - // One summary audit per section (plus the series seed) — and never a plaintext secret. + // One summary audit per section (plus the series seed) — never a plaintext secret. const auditEntities = (await db.all<{ entity: string }>( `SELECT entity FROM audit_log WHERE action='import'`)).map((r) => r.entity).sort() expect(auditEntities).toEqual(['bills', 'branches', 'clients', 'doc_series', 'rtgs_clients', @@ -290,45 +411,41 @@ describe('APEX full importer v2', () => { expect(auditDump).not.toContain('smsPass9') }) - it('refuses plaintext passwords loudly when no encryption key is configured', async () => { + it('dry run needs no key: passwords aggregate into blocking problems; commit refuses', async () => { const db = await freshDb() const files = fullFiles() - const report = await stageApexFull(db, files, '') - expect(report.clients.problems).toEqual([ - 'row 1: DB_PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted', + const { sections } = await stageApexFull(db, files, '') + expect(sections.clients.problems).toEqual([ + '1 row(s) carry a plaintext DB_PASSWORD but HQ_SECRET_KEY is not set — commit refuses until the key is provided (values are never stored unencrypted)', ]) - expect(report.sms_clients.problems).toEqual([ - 'row 1: PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted', + expect(sections.sms_clients.problems).toEqual([ + '1 row(s) carry a plaintext portal PASSWORD but HQ_SECRET_KEY is not set — commit refuses until the key is provided (values are never stored unencrypted)', ]) await expect(commitApexFull(db, 'u-test', files, '')).rejects.toThrow(/HQ_SECRET_KEY/) - // Refused before the transaction — nothing landed. expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) }) - it('duplicate doc_no is a problem that blocks the whole commit', async () => { + it('re-staging after commit reports duplicates as blocking problems (idempotency guard)', async () => { const db = await freshDb() - const files = { - clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), - bills: csv(BILL_HDR, [ - { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '100' }, - { BILL_DATE: '2025-05-02 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '200' }, - ]), - } - const report = await stageApexFull(db, files, KEY) - expect(report.bills.problems).toEqual(['row 2: duplicate doc_no: INV-1']) - await expect(commitApexFull(db, 'u-test', files, KEY)).rejects.toThrow(/duplicate doc_no/) - expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0) - expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) + const files = fullFiles() + await commitApexFull(db, 'u-test', files, KEY) + const { sections } = await stageApexFull(db, files, KEY) + expect(sections.clients.problems.some((p) => p.includes('duplicate SID_NO'))).toBe(true) + expect(sections.bills.problems.some((p) => p.includes('already exists in HQ'))).toBe(true) + await expect(commitApexFull(db, 'u-test', files, KEY)).rejects.toThrow(/problem/) }) - it('rejects an unknown INVOICE_TYPE instead of guessing', async () => { + it('rejects an unknown INVOICE_TYPE and an inconsistent line group instead of guessing', async () => { const db = await freshDb() - const report = await stageApexFull(db, { - clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), + const { sections } = await stageApexFull(db, { + clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }, { SID_NO: '102', BANK: 'Beta Bank' }]), bills: csv(BILL_HDR, [ - { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'X-1', INVOICE_TYPE: 'RECEIPT', INVOICE_AMOUNT: '100' }, + { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'X-1', SRNO: '1', INVOICE_TYPE: 'RECEIPT', AMT: '100' }, + { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'X-2', SRNO: '1', INVOICE_TYPE: 'INVOICE', AMT: '100' }, + { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '102', INVOICE_NO: 'X-2', SRNO: '2', INVOICE_TYPE: 'INVOICE', AMT: '50' }, ]), }, KEY) - expect(report.bills.problems).toEqual(['row 1: unknown INVOICE_TYPE: RECEIPT']) + expect(sections.bills.problems.some((p) => p.includes('unknown INVOICE_TYPE: RECEIPT'))).toBe(true) + expect(sections.bills.problems.some((p) => p.includes('inconsistent BANK_ID'))).toBe(true) }) })