diff --git a/apps/hq/package.json b/apps/hq/package.json index 81069e2..2171dca 100644 --- a/apps/hq/package.json +++ b/apps/hq/package.json @@ -7,6 +7,7 @@ "build": "node build-server.mjs", "start": "npm run build && node dist/server.cjs", "import": "tsx src/import-apex.ts", + "import:full": "tsx scripts/import-apex-full.ts", "typecheck": "tsc -p tsconfig.json" }, "dependencies": { diff --git a/apps/hq/scripts/import-apex-full.ts b/apps/hq/scripts/import-apex-full.ts new file mode 100644 index 0000000..126ea7d --- /dev/null +++ b/apps/hq/scripts/import-apex-full.ts @@ -0,0 +1,61 @@ +/** + * Full-APEX importer CLI (D20): `npm run import:full -- --dir [--commit]` + * Reads the nine workbook CSVs from (all optional except clients.csv), + * prints the per-section stage report, and — only with --commit and a clean + * report — applies everything in one transaction. Without --commit it is a dry + * run that writes nothing. DB path comes from HQ_DATA_DIR (run from the repo + * root otherwise — the default resolves ./data from CWD); passwords in the + * export need HQ_SECRET_KEY set or their rows are refused. + */ +import fs from 'node:fs' +import path from 'node:path' +import { openDb } from '../src/db' +import { seedIfEmpty } from '../src/seed' +import { APEX_FILES, commitApexFull, stageApexFull } from '../src/import-apex-full' + +void (async () => { + const args = process.argv.slice(2) + const dirIdx = args.indexOf('--dir') + const dir = dirIdx >= 0 ? args[dirIdx + 1] : undefined + if (dir === undefined) { + console.error('Usage: npm run import:full -- --dir [--commit]') + process.exit(1) + } + + const files: Record = {} + for (const name of APEX_FILES) { + const p = path.join(dir, `${name}.csv`) + if (fs.existsSync(p)) files[name] = fs.readFileSync(p, 'utf8') + } + if (files['clients'] === undefined) { + console.error(`clients.csv not found in ${dir} — it is the one required file (SID_NO is the join key)`) + process.exit(1) + } + const absent = APEX_FILES.filter((n) => files[n] === undefined) + if (absent.length > 0) console.log(`Not present (skipped): ${absent.join(', ')}`) + + const db = openDb(process.env['HQ_DATA_DIR']) + await seedIfEmpty(db) + const keyHex = process.env['HQ_SECRET_KEY'] ?? '' + + const report = await stageApexFull(db, files, keyHex) + let problemTotal = 0 + for (const section of APEX_FILES) { + const r = report[section] + if (r.staged === 0 && r.problems.length === 0) continue + console.log(`${section}: ${r.staged} staged, ${r.problems.length} problem(s)`) + for (const p of r.problems) console.log(` - ${p}`) + problemTotal += r.problems.length + } + + if (!args.includes('--commit')) { + console.log('Dry run — nothing written. Re-run with --commit to apply.') + } else if (problemTotal > 0) { + console.error(`Refusing to commit: ${problemTotal} problem row(s) above — fix the CSVs and re-run.`) + process.exitCode = 1 + } else { + const counts = await commitApexFull(db, 'system', files, keyHex) + console.log('Committed:', JSON.stringify(counts, null, 2)) + } + await db.close() +})() diff --git a/apps/hq/src/import-apex-full.ts b/apps/hq/src/import-apex-full.ts new file mode 100644 index 0000000..7b486e6 --- /dev/null +++ b/apps/hq/src/import-apex-full.ts @@ -0,0 +1,794 @@ +import { fromRupees, fyOf, uuidv7 } from '@sims/domain' +import { writeAudit } from './audit' +import { encrypt } from './crypto' +import type { DB } from './db' +import { seedSeries } from './series' + +/** + * Full-APEX importer v2 (D20 — docs/superpowers/specs/2026-07-17-apex-parity-design.md §4). + * Follows v1's discipline: stage → per-row problems → refuse commit while any problem + * exists. Unlike v1 there are no staging tables — `stageApexFull` is a pure validation + * pass that writes nothing; `commitApexFull` re-runs the same validation and applies + * everything inside ONE transaction, auditing one summary row per section (per-row + * client/document audits are deliberately skipped — 5k+ rows would drown the log). + * Rupee amounts convert to integer paise via `fromRupees` at this edge; plaintext + * passwords (client DB + SMS portal) are AES-256-GCM encrypted or the row is refused. + * v1 (`import-apex.ts`) stays untouched for the simple owner-only web flow. + */ + +// ---------- tiny CSV parser (copied from v1, which keeps it private) ---------- + +function parseCsv(text: string): Record[] { + const rows: string[][] = [] + let field = '' + let row: string[] = [] + let inQuotes = false + const endField = (): void => { row.push(field); field = '' } + const endRow = (): void => { endField(); rows.push(row); row = [] } + for (let i = 0; i < text.length; i++) { + const ch = text[i]! + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { field += '"'; i++ } else inQuotes = false + } else field += ch + } else if (ch === '"') inQuotes = true + else if (ch === ',') endField() + else if (ch === '\n') endRow() + else if (ch !== '\r') field += ch + } + if (field !== '' || row.length > 0) endRow() + const header = (rows.shift() ?? []).map((h) => h.trim()) + return rows + .filter((r) => r.some((c) => c.trim() !== '')) + .map((r) => Object.fromEntries(header.map((h, i) => [h, (r[i] ?? '').trim()]))) +} + +// ---------- typed workbook rows ---------- + +/** The nine CSVs of the APEX export (clients required, the rest optional). */ +export const APEX_FILES = [ + 'clients', 'branches', 'bills', 'tickets', + 'sms_clients', 'sms_modules', 'rtgs_clients', 'sms_projects', 'rtgs_projects', +] as const +export type ApexFileName = (typeof APEX_FILES)[number] + +type RowOf = { [K in H[number]]: string } + +const CLIENT_H = ['SID_NO', 'BANK', 'ADDRESS', 'PHONE', 'MAIL', 'DISTRICT', 'SMS', 'RTGS', + 'WHATSAPP', 'MOBILE_APP', 'DB_PASSWORD', 'ATM', 'ANYDESK', 'SECRETARY_CONTACT_DETAILS', + 'OS', 'SECTOR'] as const +const BRANCH_H = ['ID', 'BRANCH', 'SMS', 'MID', 'BANK', 'BRCODE'] as const +const BILL_H = ['DAY_DT', 'BILL_TYP', 'BILL_DATE', 'BILL_NO', 'BANK_ID', 'BANK_NAME', 'ADDRESS', + 'MOBILE', 'MAIL', 'REMARK', 'USR', 'INVOICE_NO', 'QTY', 'PER_QTY', 'DESCRIPTION', + 'INVOICE_TYPE', 'INVOICE_AMOUNT', 'PAYMENT_STATUS', 'PAYMENT_DATE', 'PAYMENT_AMOUNT', + 'NOTES', 'AMT', 'SRNO', 'GST_RATE', 'GST_AMOUNT', 'TOTAL_AMOUNT', + 'CONVERTED_FROM_PROFORMA_ID', 'GST_APPLICABLE', 'INVOICE_ID', 'SCROLL'] as const +const TICKET_H = ['ID', 'DAY_DT', 'BANK_NAME', 'BANK_ID', 'BRID', 'TYPE', 'GROUP', 'ASSIGN_DATE', + 'MOBILE_NO', 'ASSIGNED_TO', 'DESCRIPTION', 'STATUS', 'CREATED', 'CREATED_BY', 'UPDATED', + 'UPDATED_BY', 'ONLINE_OFFLINE', 'MAIL_BLOB', 'MAIL', 'TRNS_ID', 'TRNS_FRM', 'DAY_ID', + 'ENAME', 'TRANS', 'TICKET_DROPED', 'DAY_CLS', 'OLD_ID', 'SCROLL'] as const +const SMS_CLIENT_H = ['RNO', 'CID', 'BANK', 'DISTRICT', 'SMS', 'USERNAME', 'PASSWORD', 'PAYMENT', + 'SMS_PROVIDER', 'ADDED_FEATURES', 'CREATED_BY', 'UPDATED_BY', 'PHONE', 'TID', 'SMS_BALANCE', + 'ERR', 'REMARK', 'UPDATION', 'RESELLER', 'PAYMENT_DATE', 'INSTALLATION_DATE'] as const +const SMS_MODULE_H = ['RNO', 'SID_NO', 'SMS_ID', 'HEAD', 'COLS', 'PKG', 'ACTIV', 'REMARK', + 'IMP_BY', 'ON_DAY_DT'] as const +const RTGS_CLIENT_H = ['SID_NO', 'BANK_NAME', 'ADDRESS_IP', 'PHONE_NO', 'RTGS', 'IP_CODE', + 'E_COLLECTION_CODE', 'RID', 'TID', 'ABB', 'INTEGRATED_BANK_PARTNERS', 'REMARK', 'PAYMENT', + 'INSTALLATION_DATE', 'INSTALLED_ON', 'GENERATE_QR', 'SBI_INWARD_REPORT', 'PASSBOOK_PROGRAM', + 'IFSC_2025', 'NOTIFICATION', 'MISC_RTGS_OUTWARD', 'PAYMENT_DATE'] as const +const SMS_PROJECT_H = ['CID', 'BANK', 'DISTRICT', 'DT_INFO', 'RNO', 'SMS', 'PHONE', 'ASSIGNED_TO', + 'TICKET_STATUS', 'DESCRIPTION', 'NEW_FORM', 'CREATED_BY', 'UPDATED_BY', 'TID'] as const +const RTGS_PROJECT_H = ['CID', 'BANK', 'DT_INFO', 'RNO', 'RTGS', 'PHONE', 'ASSIGNED_TO', + 'TICKET_STATUS', 'DESCRIPTION', 'CREATED_BY', 'UPDATED_BY', 'TID'] as const + +export type ApexClientRow = RowOf +export type ApexBranchRow = RowOf +export type ApexBillRow = RowOf +export type ApexTicketRow = RowOf +export type ApexSmsClientRow = RowOf +export type ApexSmsModuleRow = RowOf +export type ApexRtgsClientRow = RowOf +export type ApexSmsProjectRow = RowOf +export type ApexRtgsProjectRow = RowOf + +export interface ApexWorkbook { + clients: ApexClientRow[] + branches: ApexBranchRow[] + bills: ApexBillRow[] + tickets: ApexTicketRow[] + sms_clients: ApexSmsClientRow[] + sms_modules: ApexSmsModuleRow[] + rtgs_clients: ApexRtgsClientRow[] + sms_projects: ApexSmsProjectRow[] + rtgs_projects: ApexRtgsProjectRow[] +} + +/** Oracle CSV exports embed carriage returns as the literal token `_x000D_`. */ +const clean = (s: string): string => s.replace(/_x000D_/g, '\n').replace(/\r/g, '').trim() +const orNull = (v: string): string | null => (v === '' ? null : v) + +function typed(records: Record[], headers: H): RowOf[] { + return records.map((r) => + Object.fromEntries(headers.map((h) => [h, clean(r[h] ?? '')])) as RowOf) +} + +/** Parse the csv-name→csv-text map (keys with or without `.csv`) into typed row arrays. */ +export function parseWorkbookCsvs(files: Record): ApexWorkbook { + const byName = new Map() + for (const [k, v] of Object.entries(files)) byName.set(k.toLowerCase().replace(/\.csv$/, ''), v) + const rows = (name: ApexFileName): Record[] => { + const text = byName.get(name) + return text === undefined ? [] : parseCsv(text) + } + return { + clients: typed(rows('clients'), CLIENT_H), + branches: typed(rows('branches'), BRANCH_H), + bills: typed(rows('bills'), BILL_H), + tickets: typed(rows('tickets'), TICKET_H), + sms_clients: typed(rows('sms_clients'), SMS_CLIENT_H), + sms_modules: typed(rows('sms_modules'), SMS_MODULE_H), + rtgs_clients: typed(rows('rtgs_clients'), RTGS_CLIENT_H), + sms_projects: typed(rows('sms_projects'), SMS_PROJECT_H), + rtgs_projects: typed(rows('rtgs_projects'), RTGS_PROJECT_H), + } +} + +// ---------- tolerant APEX date parser ---------- + +const MONTHS: Record = { + JAN: 1, FEB: 2, MAR: 3, APR: 4, MAY: 5, JUN: 6, + JUL: 7, AUG: 8, SEP: 9, OCT: 10, NOV: 11, DEC: 12, +} + +function validDate(y: number, month: number, day: number): string | null { + if (month < 1 || month > 12 || day < 1 || day > 31) return null + const iso = `${y}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}` + return Number.isNaN(Date.parse(iso)) ? null : iso +} + +/** + * Best-effort date-part extraction for the formats seen in the APEX export: + * ISO ('2025-08-18 00:00:00'), DD-MON-YYYY ('10-NOV-2023 10:57:18'), and + * X-Y-ZZZZ / X/Y/ZZZZ where whichever of X/Y is >12 must be the day + * ('08/29/2024' → MM/DD, '13-05-2024' → DD-MM); ambiguous → assume DD-MM + * (the dominant APEX habit). Returns 'YYYY-MM-DD' or null. + */ +export function parseApexDate(raw: string): string | null { + const s = raw.trim() + if (s === '') return null + let m = /^(\d{4})-(\d{1,2})-(\d{1,2})/.exec(s) + if (m !== null) return validDate(Number(m[1]!), Number(m[2]!), Number(m[3]!)) + m = /^(\d{1,2})-([A-Za-z]{3})-(\d{4})/.exec(s) + if (m !== null) { + const month = MONTHS[m[2]!.toUpperCase()] + return month === undefined ? null : validDate(Number(m[3]!), month, Number(m[1]!)) + } + m = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4})/.exec(s) + if (m !== null) { + const a = Number(m[1]!) + const b = Number(m[2]!) + const [day, month] = a > 12 ? [a, b] : b > 12 ? [b, a] : [a, b] + return validDate(Number(m[3]!), month!, day!) + } + return null +} + +// ---------- report + plan ---------- + +export interface SectionReport { staged: number; problems: string[] } +export type FullImportReport = Record + +interface PlannedClient { + id: string; code: string; name: string; address: string + district: string | null; sector: string | null; os: string | null; anydesk: string | null + contacts: { name: string; phone?: string; email?: string; role?: string }[] + dbPasswordEnc: string | null +} + +interface PlannedAssignment { + id: string; clientId: string; moduleCode: string + provider: string | null; username: string | null; passwordEnc: string | null + details: { label: string; value: string }[] + remark: string | null + /** True once sms/rtgs service data landed — gates updates onto pre-existing rows. */ + serviceTouched: boolean +} + +interface PlannedBranch { id: string; clientId: string; name: string; code: string | null } + +interface PlannedDoc { + id: string; docType: 'INVOICE' | 'PROFORMA' | 'QUOTATION'; docNo: string; fy: string + clientId: string; docDate: string; status: string + taxable: number; cgst: number; sgst: number; igst: number; roundOff: number; total: number + payment: { amount: number; receivedOn: string } | null +} + +interface PlannedTicket { + id: string; clientId: string; moduleCode: string | null; kind: string; description: string + status: 'open' | 'in_progress' | 'waiting' | 'closed' | 'dropped' + onlineOffline: string | null; openedOn: string; closedOn: string | null; createdAt: string +} + +interface PlannedInteraction { + id: string; clientId: string; onDate: string; notes: string + section: 'sms_projects' | 'rtgs_projects' +} + +interface Plan { + report: FullImportReport + ourState: string + clients: PlannedClient[] + assignments: PlannedAssignment[] + branches: PlannedBranch[] + docs: PlannedDoc[] + tickets: PlannedTicket[] + interactions: PlannedInteraction[] + currentFy: string + /** Max numeric tail among current-FY invoice numbers, for the series seed. */ + seedTail: number | null +} + +async function companyStateCode(db: DB): Promise { + const row = await db.get<{ value: string }>(`SELECT value FROM setting WHERE key='company.state_code'`) + // Silent defaults on place-of-supply are how wrong GST reaches the portal — fail loudly. + if (row === undefined) throw new Error(`Setting 'company.state_code' is not configured`) + return row.value +} + +function rupeesToPaise(raw: string): number | null { + if (raw === '') return null + const n = Number(raw) + return Number.isFinite(n) ? fromRupees(n) : null +} + +/** '' and the APEX placeholder 'NO DATA' both mean "nothing here". */ +const noData = (v: string): boolean => v === '' || v.toUpperCase() === 'NO DATA' + +const MODULE_NAMES: Record = { + SMS: 'Bulk SMS', RTGS: 'RTGS', WHATSAPP: 'WhatsApp', MOBILEAPP: 'Mobile App', ATM: 'ATM', +} + +const FLAG_MODULES: { col: 'SMS' | 'RTGS' | 'WHATSAPP' | 'MOBILE_APP' | 'ATM'; code: string }[] = [ + { col: 'SMS', code: 'SMS' }, + { col: 'RTGS', code: 'RTGS' }, + { col: 'WHATSAPP', code: 'WHATSAPP' }, + { col: 'MOBILE_APP', code: 'MOBILEAPP' }, + { col: 'ATM', code: 'ATM' }, +] + +function mapTicketStatus(status: string, dropped: string): PlannedTicket['status'] { + const s = status.toUpperCase() + if (s === 'CLOSED') return 'closed' + if (s === 'OPEN') return 'open' + if (s.includes('WAITING')) return 'waiting' + if (dropped.toUpperCase() === 'Y') return 'dropped' + return 'open' +} + +/** Set-or-replace one labeled detail (skip empty values, replace an existing label). */ +function setDetail(a: PlannedAssignment, label: string, value: string): void { + if (value === '') return + const existing = a.details.find((d) => d.label === label) + if (existing !== undefined) existing.value = value + else a.details.push({ label, value }) +} + +/** + * The single validation + mapping pass shared by stage and commit. Reads the DB + * (existing codes/doc numbers/state codes) but NEVER writes. Rows with problems + * are reported and excluded from the plan; commit refuses while any exist. + */ +async function buildPlan(db: DB, wb: ApexWorkbook, keyHex: string): Promise { + const ourState = await companyStateCode(db) + const report = Object.fromEntries( + APEX_FILES.map((s) => [s, { staged: wb[s].length, problems: [] as string[] }]), + ) as FullImportReport + const plan: Plan = { + report, ourState, clients: [], assignments: [], branches: [], docs: [], tickets: [], + interactions: [], currentFy: fyOf(new Date().toISOString().slice(0, 10)), seedTail: null, + } + const todayIso = new Date().toISOString().slice(0, 10) + const now = new Date().toISOString() + + const existing = await db.all<{ id: string; code: string; name: string; state_code: string }>( + `SELECT id, code, name, state_code FROM client`, + ) + const existingByCode = new Map(existing.map((c) => [c.code, c])) + const existingByName = new Map() + for (const c of existing) if (!existingByName.has(c.name)) existingByName.set(c.name, c.id) + const existingDocNos = new Set( + (await db.all<{ doc_no: string }>(`SELECT doc_no FROM document WHERE doc_no IS NOT NULL`)) + .map((r) => r.doc_no), + ) + + const planByCode = new Map() + const planByName = new Map() + const resolveClient = (code: string): { id: string; state: string } | undefined => { + const p = planByCode.get(code) + if (p !== undefined) return { id: p.id, state: ourState } // imported clients take our state + const e = existingByCode.get(code) + return e === undefined ? undefined : { id: e.id, state: e.state_code } + } + + const assignmentByKey = new Map() + const ensureAssignment = (clientId: string, moduleCode: string): PlannedAssignment => { + const key = `${clientId}|${moduleCode}` + let a = assignmentByKey.get(key) + if (a === undefined) { + a = { + id: uuidv7(), clientId, moduleCode, + provider: null, username: null, passwordEnc: null, details: [], remark: null, + serviceTouched: false, + } + assignmentByKey.set(key, a) + } + return a + } + + // ----- clients (SID_NO is the cross-file join key → client.code) ----- + for (let i = 0; i < wb.clients.length; i++) { + const r = wb.clients[i]! + const probs: string[] = [] + const code = r.SID_NO + if (code === '') probs.push('missing SID_NO') + else if (planByCode.has(code) || existingByCode.has(code)) probs.push(`duplicate SID_NO: ${code}`) + if (r.BANK === '') probs.push('missing BANK') + let dbPasswordEnc: string | null = null + if (!noData(r.DB_PASSWORD)) { + if (keyHex === '') { + probs.push('DB_PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted') + } else dbPasswordEnc = encrypt(r.DB_PASSWORD, keyHex) + } + if (probs.length > 0) { + report.clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + const contacts: PlannedClient['contacts'] = [] + if (r.PHONE !== '' || r.MAIL !== '') { + contacts.push({ + name: r.BANK, + ...(r.PHONE !== '' ? { phone: r.PHONE } : {}), + ...(r.MAIL !== '' ? { email: r.MAIL } : {}), + }) + } + if (r.SECRETARY_CONTACT_DETAILS !== '') { + contacts.push({ name: r.SECRETARY_CONTACT_DETAILS, role: 'secretary' }) + } + const pc: PlannedClient = { + id: uuidv7(), code, name: r.BANK, address: r.ADDRESS, + district: orNull(r.DISTRICT), sector: orNull(r.SECTOR), os: orNull(r.OS), + anydesk: noData(r.ANYDESK) ? null : r.ANYDESK, + contacts, dbPasswordEnc, + } + planByCode.set(code, pc) + if (!planByName.has(pc.name)) planByName.set(pc.name, pc.id) + plan.clients.push(pc) + // YES/Y* service flags → live module assignments. + for (const f of FLAG_MODULES) { + if (r[f.col].toUpperCase().startsWith('Y')) ensureAssignment(pc.id, f.code) + } + } + + // ----- branches (MID → client.code) ----- + for (let i = 0; i < wb.branches.length; i++) { + const r = wb.branches[i]! + const probs: string[] = [] + const c = r.MID === '' ? undefined : resolveClient(r.MID) + if (r.MID === '') probs.push('missing MID') + else if (c === undefined) probs.push(`unknown client MID=${r.MID}`) + if (r.BRANCH === '') probs.push('missing BRANCH') + if (probs.length > 0) { + report.branches.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + plan.branches.push({ id: uuidv7(), clientId: c!.id, name: r.BRANCH, code: orNull(r.BRCODE) }) + } + + // ----- bills (documents + payments; legacy numbers kept, series seeded past them) ----- + const TYPE_MAP: Record = { + 'INVOICE': 'INVOICE', 'PROFORMA INVOICE': 'PROFORMA', 'QUOTATION': 'QUOTATION', + } + const seenDocNos = new Set() + for (let i = 0; i < wb.bills.length; i++) { + const r = wb.bills[i]! + const probs: string[] = [] + const c = r.BANK_ID === '' ? undefined : resolveClient(r.BANK_ID) + if (r.BANK_ID === '') probs.push('missing BANK_ID') + else if (c === undefined) probs.push(`unknown client BANK_ID=${r.BANK_ID}`) + const docType = TYPE_MAP[r.INVOICE_TYPE.toUpperCase()] + if (docType === undefined) probs.push(`unknown INVOICE_TYPE: ${r.INVOICE_TYPE}`) + const docNo = r.INVOICE_NO + if (docNo === '') probs.push('missing INVOICE_NO') + else if (seenDocNos.has(docNo) || existingDocNos.has(docNo)) probs.push(`duplicate doc_no: ${docNo}`) + if (docNo !== '') seenDocNos.add(docNo) + const docDate = parseApexDate(r.BILL_DATE) + if (docDate === null) probs.push(`bad BILL_DATE: ${r.BILL_DATE}`) + const taxable = rupeesToPaise(r.INVOICE_AMOUNT) + if (taxable === null) probs.push(`bad INVOICE_AMOUNT: ${r.INVOICE_AMOUNT}`) + const tax = r.GST_AMOUNT === '' ? 0 : rupeesToPaise(r.GST_AMOUNT) + if (tax === null) probs.push(`bad GST_AMOUNT: ${r.GST_AMOUNT}`) + let total: number | null = null + if (taxable !== null && tax !== null) { + total = r.TOTAL_AMOUNT === '' ? taxable + tax : rupeesToPaise(r.TOTAL_AMOUNT) + if (total === null) probs.push(`bad TOTAL_AMOUNT: ${r.TOTAL_AMOUNT}`) + } + if (probs.length > 0) { + report.bills.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + // GST split by place of supply (v1 pattern): intra-state → CGST+SGST, else IGST. + const intra = c!.state === ourState + const cgst = intra ? Math.floor(tax! / 2) : 0 + const sgst = intra ? tax! - cgst : 0 + const igst = intra ? 0 : tax! + const status = docType === 'INVOICE' + ? (r.PAYMENT_STATUS.toUpperCase() === 'PAID' ? 'paid' : 'sent') + : 'sent' + const payAmt = rupeesToPaise(r.PAYMENT_AMOUNT) + const payDate = parseApexDate(r.PAYMENT_DATE) + const payment = docType === 'INVOICE' && payAmt !== null && payAmt > 0 && payDate !== null + ? { amount: Math.min(payAmt, total!), receivedOn: payDate } + : null + plan.docs.push({ + id: uuidv7(), docType: docType!, docNo, fy: fyOf(docDate!), clientId: c!.id, + docDate: docDate!, status, + taxable: taxable!, cgst, sgst, igst, roundOff: total! - taxable! - tax!, total: total!, + payment, + }) + } + // Series seed (v1 logic): continue INVOICE numbering after the last current-FY + // legacy number — prefixes differ, so only the numeric tail matters. + const tails = plan.docs + .filter((d) => d.docType === 'INVOICE' && d.fy === plan.currentFy) + .map((d) => /(\d+)$/.exec(d.docNo)?.[1]) + .filter((t): t is string => t !== undefined) + .map((t) => Number(t)) + plan.seedTail = tails.length > 0 ? Math.max(...tails) : null + + // ----- tickets (BANK_ID → code; blank BANK_ID falls back to exact BANK_NAME) ----- + for (let i = 0; i < wb.tickets.length; i++) { + const r = wb.tickets[i]! + const probs: string[] = [] + let clientId: string | undefined + if (r.BANK_ID !== '') { + const c = resolveClient(r.BANK_ID) + if (c === undefined) probs.push(`unknown client BANK_ID=${r.BANK_ID}`) + else clientId = c.id + } else if (r.BANK_NAME !== '') { + clientId = planByName.get(r.BANK_NAME) ?? existingByName.get(r.BANK_NAME) + if (clientId === undefined) probs.push(`no client named '${r.BANK_NAME}'`) + } else probs.push('missing BANK_ID and BANK_NAME') + const openedRaw = r.ASSIGN_DATE !== '' ? r.ASSIGN_DATE : (r.DAY_DT !== '' ? r.DAY_DT : r.CREATED) + const openedOn = parseApexDate(openedRaw) + if (openedOn === null) probs.push(`no parseable open date (ASSIGN_DATE='${r.ASSIGN_DATE}', DAY_DT='${r.DAY_DT}', CREATED='${r.CREATED}')`) + if (probs.length > 0) { + report.tickets.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + const status = mapTicketStatus(r.STATUS, r.TICKET_DROPED) + const oo = r.ONLINE_OFFLINE.toUpperCase() + plan.tickets.push({ + id: uuidv7(), clientId: clientId!, + moduleCode: orNull(r.GROUP), kind: r.TYPE, description: r.DESCRIPTION, status, + onlineOffline: oo === 'ONLINE' || oo === 'OFFLINE' ? oo.toLowerCase() : null, + openedOn: openedOn!, + closedOn: status === 'closed' || status === 'dropped' ? parseApexDate(r.UPDATED) : null, + createdAt: r.CREATED !== '' ? r.CREATED : now, + // BRID has no reliable branch join and APEX staff ids don't map to HQ + // staff — branch_id and assigned_to stay NULL by design. + }) + } + + // ----- SMS service data (sms_clients: CID → code → the client's SMS module) ----- + const seenSmsCids = new Set() + for (let i = 0; i < wb.sms_clients.length; i++) { + const r = wb.sms_clients[i]! + const probs: string[] = [] + const c = r.CID === '' ? undefined : resolveClient(r.CID) + if (r.CID === '') probs.push('missing CID') + else if (c === undefined) probs.push(`unknown client CID=${r.CID}`) + else if (seenSmsCids.has(r.CID)) probs.push(`duplicate CID: ${r.CID}`) + if (r.CID !== '') seenSmsCids.add(r.CID) + let passwordEnc: string | null = null + if (r.PASSWORD !== '') { + if (keyHex === '') { + probs.push('PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted') + } else passwordEnc = encrypt(r.PASSWORD, keyHex) + } + if (probs.length > 0) { + report.sms_clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + // Flag said NO but this table has a row → data wins, create the assignment. + const a = ensureAssignment(c!.id, 'SMS') + a.provider = orNull(r.SMS_PROVIDER) + a.username = orNull(r.USERNAME) + a.passwordEnc = passwordEnc + a.remark = orNull(r.REMARK) + setDetail(a, 'SMS balance', r.SMS_BALANCE) + setDetail(a, 'Reseller', r.RESELLER) + setDetail(a, 'Features', r.ADDED_FEATURES) + setDetail(a, 'Updation', r.UPDATION) + setDetail(a, 'Payment', r.PAYMENT) + setDetail(a, 'Phone', r.PHONE) + setDetail(a, 'Installed', r.INSTALLATION_DATE) + a.serviceTouched = true + } + + // ----- SMS alert types (sms_modules: group by SID_NO, HEAD where ACTIV='Y') ----- + const alertsByClient = new Map() + for (let i = 0; i < wb.sms_modules.length; i++) { + const r = wb.sms_modules[i]! + const c = r.SID_NO === '' ? undefined : resolveClient(r.SID_NO) + if (c === undefined) { + report.sms_modules.problems.push(`row ${i + 1}: ${r.SID_NO === '' ? 'missing SID_NO' : `unknown client SID_NO=${r.SID_NO}`}`) + continue + } + if (r.ACTIV.toUpperCase() === 'Y' && r.HEAD !== '') { + const list = alertsByClient.get(c.id) ?? [] + list.push(r.HEAD) + alertsByClient.set(c.id, list) + } + } + for (const [clientId, heads] of alertsByClient) { + const a = ensureAssignment(clientId, 'SMS') + setDetail(a, 'Active alerts', heads.join(', ')) + a.serviceTouched = true + } + + // ----- RTGS service data (rtgs_clients: SID_NO → code → the RTGS module) ----- + const seenRtgsSids = new Set() + for (let i = 0; i < wb.rtgs_clients.length; i++) { + const r = wb.rtgs_clients[i]! + const probs: string[] = [] + const c = r.SID_NO === '' ? undefined : resolveClient(r.SID_NO) + if (r.SID_NO === '') probs.push('missing SID_NO') + else if (c === undefined) probs.push(`unknown client SID_NO=${r.SID_NO}`) + else if (seenRtgsSids.has(r.SID_NO)) probs.push(`duplicate SID_NO: ${r.SID_NO}`) + if (r.SID_NO !== '') seenRtgsSids.add(r.SID_NO) + if (probs.length > 0) { + report.rtgs_clients.problems.push(...probs.map((p) => `row ${i + 1}: ${p}`)) + continue + } + const a = ensureAssignment(c!.id, 'RTGS') + setDetail(a, 'IP', r.ADDRESS_IP) + setDetail(a, 'IP code', r.IP_CODE) + setDetail(a, 'E-collection code', r.E_COLLECTION_CODE) + setDetail(a, 'ABB', r.ABB) + setDetail(a, 'Partner bank', r.INTEGRATED_BANK_PARTNERS) + setDetail(a, 'QR', r.GENERATE_QR) + setDetail(a, 'SBI inward report', r.SBI_INWARD_REPORT) + setDetail(a, 'Passbook program', r.PASSBOOK_PROGRAM) + setDetail(a, 'IFSC 2025', r.IFSC_2025) + setDetail(a, 'Notification', r.NOTIFICATION) + setDetail(a, 'Misc RTGS outward', r.MISC_RTGS_OUTWARD) + setDetail(a, 'Phone', r.PHONE_NO) + setDetail(a, 'Installed', r.INSTALLED_ON !== '' ? r.INSTALLED_ON : r.INSTALLATION_DATE) + setDetail(a, 'Payment', r.PAYMENT) + a.remark = orNull(r.REMARK) + a.serviceTouched = true + } + + // ----- project lists → interaction rows ----- + const planProjects = ( + rows: { CID: string; DT_INFO: string; TICKET_STATUS: string; DESCRIPTION: string }[], + tag: string, section: 'sms_projects' | 'rtgs_projects', + ): void => { + for (let i = 0; i < rows.length; i++) { + const r = rows[i]! + const c = r.CID === '' ? undefined : resolveClient(r.CID) + if (c === undefined) { + report[section].problems.push(`row ${i + 1}: ${r.CID === '' ? 'missing CID' : `unknown client CID=${r.CID}`}`) + continue + } + const parsed = parseApexDate(r.DT_INFO) + const bits = [r.TICKET_STATUS, r.DESCRIPTION].filter((s) => s !== '') + let notes = bits.length > 0 ? `${tag} ${bits.join(' — ')}` : tag + if (parsed === null && r.DT_INFO !== '') notes += ` (DT_INFO unparsed: ${r.DT_INFO})` + plan.interactions.push({ + id: uuidv7(), clientId: c.id, onDate: parsed ?? todayIso, notes, section, + }) + } + } + planProjects(wb.sms_projects, '[SMS project]', 'sms_projects') + planProjects(wb.rtgs_projects, '[RTGS project]', 'rtgs_projects') + + plan.assignments = [...assignmentByKey.values()] + return plan +} + +// ---------- stage (pure validation — writes nothing) ---------- + +/** + * Validation pass over the whole workbook: per-section staged counts and per-row + * problems. Reads the DB for join/duplicate checks but writes nothing permanent + * (nothing at all, in fact — there are no v2 staging tables). + */ +export async function stageApexFull( + db: DB, files: Record, keyHex = '', +): Promise { + const plan = await buildPlan(db, parseWorkbookCsvs(files), keyHex) + return plan.report +} + +// ---------- commit (same validation, ONE transaction) ---------- + +export interface FullCommitResult { + clients: number; client_modules: number; branches: number + bills: number; payments: number; tickets: number + sms_clients: number; sms_modules: number; rtgs_clients: number + sms_projects: number; rtgs_projects: number + seeded: { fy: string; lastSeq: number } | null +} + +const ALL_KINDS_JSON = JSON.stringify(['one_time', 'monthly', 'yearly', 'usage']) + +async function ensureModule(db: DB, code: string): Promise { + const row = await db.get<{ id: string }>(`SELECT id FROM module WHERE code=?`, code) + if (row !== undefined) return row.id + const id = uuidv7() + await db.run( + `INSERT INTO module (id, code, name, sac, allowed_kinds, multi_subscription, quote_content) + VALUES (?, ?, ?, '998313', ?, 0, '[]')`, + id, code, MODULE_NAMES[code] ?? code, ALL_KINDS_JSON, + ) + return id +} + +/** + * Re-run the validation and, only if EVERY section is clean, apply the whole + * workbook in one transaction. Auditing is one summary row per section (the + * deliberate volume call — 5k tickets would drown per-row audits). + */ +export async function commitApexFull( + db: DB, userId: string, files: Record, keyHex = '', +): Promise { + const wb = parseWorkbookCsvs(files) + const plan = await buildPlan(db, wb, keyHex) + const allProblems = APEX_FILES.flatMap((s) => plan.report[s].problems.map((p) => `${s} ${p}`)) + if (allProblems.length > 0) { + throw new Error(`${allProblems.length} problem row(s) block the import — fix the CSVs and re-stage:\n${allProblems.join('\n')}`) + } + const now = new Date().toISOString() + return db.transaction(async (): Promise => { + // Clients — source 'apex' marks the cutover rows; code IS the APEX SID_NO. + for (const c of plan.clients) { + await db.run( + `INSERT INTO client (id, code, name, gstin, state_code, address, contacts, status, + anydesk, os, district, sector, db_password_enc, source, created_at) + VALUES (?, ?, ?, NULL, ?, ?, ?, 'active', ?, ?, ?, ?, ?, 'apex', ?)`, + c.id, c.code, c.name, plan.ourState, c.address, JSON.stringify(c.contacts), + c.anydesk, c.os, c.district, c.sector, c.dbPasswordEnc, now, + ) + } + + // Module assignments — direct inserts, NOT assignModule (which audits per row). + const moduleIds = new Map() + for (const a of plan.assignments) { + let moduleId = moduleIds.get(a.moduleCode) + if (moduleId === undefined) { + moduleId = await ensureModule(db, a.moduleCode) + moduleIds.set(a.moduleCode, moduleId) + } + const existingCm = await db.get<{ id: string }>( + `SELECT id FROM client_module WHERE client_id=? AND module_id=? AND active=1`, + a.clientId, moduleId, + ) + if (existingCm === undefined) { + await db.run( + `INSERT INTO client_module (id, client_id, module_id, status, kind, edition, active, + provider, username, password_enc, details, remark) + VALUES (?, ?, ?, 'live', 'yearly', 'standard', 1, ?, ?, ?, ?, ?)`, + a.id, a.clientId, moduleId, + a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, + ) + } else if (a.serviceTouched) { + // Client pre-dated this import with a live assignment: service data wins. + await db.run( + `UPDATE client_module SET provider=?, username=?, password_enc=?, details=?, remark=? WHERE id=?`, + a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, existingCm.id, + ) + } + } + + // Branches. + for (const b of plan.branches) { + await db.run( + `INSERT INTO client_branch (id, client_id, name, code, active) VALUES (?, ?, ?, ?, 1)`, + b.id, b.clientId, b.name, b.code, + ) + } + + // Documents (history-only: empty payload lines, legacy numbers) + payments. + let payments = 0 + for (const d of plan.docs) { + const totals = { + grossPaise: d.taxable, discountPaise: 0, taxablePaise: d.taxable, + cgstPaise: d.cgst, sgstPaise: d.sgst, igstPaise: d.igst, cessPaise: 0, + roundOffPaise: d.roundOff, payablePaise: d.total, savingsVsMrpPaise: 0, + } + await db.run( + `INSERT INTO document (id, doc_type, doc_no, fy, client_id, doc_date, status, ref_doc_id, + taxable_paise, cgst_paise, sgst_paise, igst_paise, round_off_paise, payable_paise, + payload, source, created_by, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, 'apex', ?, ?)`, + d.id, d.docType, d.docNo, d.fy, d.clientId, d.docDate, d.status, + d.taxable, d.cgst, d.sgst, d.igst, d.roundOff, d.total, + JSON.stringify({ lines: [], totals }), userId, now, + ) + if (d.payment !== null) { + const paymentId = uuidv7() + await db.run( + `INSERT INTO payment (id, client_id, received_on, mode, reference, amount_paise, + tds_paise, created_by, created_at) + VALUES (?, ?, ?, 'bank', 'APEX import', ?, 0, ?, ?)`, + paymentId, d.clientId, d.payment.receivedOn, d.payment.amount, userId, now, + ) + await db.run( + `INSERT INTO payment_allocation (id, payment_id, document_id, amount_paise) + VALUES (?, ?, ?, ?)`, + uuidv7(), paymentId, d.id, d.payment.amount, + ) + payments++ + } + } + + // Series seed: continue INVOICE numbering after the last current-FY legacy number. + let seeded: FullCommitResult['seeded'] = null + if (plan.seedTail !== null) { + await seedSeries(db, 'INVOICE', plan.currentFy, plan.seedTail) + seeded = { fy: plan.currentFy, lastSeq: plan.seedTail } + await writeAudit(db, userId, 'import', 'doc_series', `INVOICE/${plan.currentFy}`, undefined, + { lastSeq: plan.seedTail }) + } + + // Tickets. + for (const t of plan.tickets) { + await db.run( + `INSERT INTO ticket (id, client_id, branch_id, module_code, kind, description, status, + assigned_to, online_offline, opened_on, closed_on, created_by, created_at, source) + VALUES (?, ?, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, 'apex')`, + t.id, t.clientId, t.moduleCode, t.kind, t.description, t.status, + t.onlineOffline, t.openedOn, t.closedOn, userId, t.createdAt, + ) + } + + // Project lists → interaction rows under a get-or-create 'project' type. + if (plan.interactions.length > 0) { + // Portability quirk: ON CONFLICT upsert is SQLite/Postgres dialect (standard SQL has MERGE). + await db.run( + `INSERT INTO interaction_type (code, label) VALUES ('project', 'Project note') + ON CONFLICT (code) DO NOTHING`, + ) + for (const x of plan.interactions) { + await db.run( + `INSERT INTO interaction (id, client_id, type_code, on_date, staff_id, notes, + outcome, follow_up_on, created_at) + VALUES (?, ?, 'project', ?, ?, ?, NULL, NULL, ?)`, + x.id, x.clientId, x.onDate, userId, x.notes, now, + ) + } + } + + const counts: FullCommitResult = { + clients: plan.clients.length, + client_modules: plan.assignments.length, + branches: plan.branches.length, + bills: plan.docs.length, + payments, + tickets: plan.tickets.length, + sms_clients: plan.report.sms_clients.staged, + sms_modules: plan.report.sms_modules.staged, + rtgs_clients: plan.report.rtgs_clients.staged, + sms_projects: plan.interactions.filter((x) => x.section === 'sms_projects').length, + rtgs_projects: plan.interactions.filter((x) => x.section === 'rtgs_projects').length, + seeded, + } + // One summary audit row per section that carried data — same txn as the writes. + for (const s of APEX_FILES) { + const count = counts[s] + if (count > 0) await writeAudit(db, userId, 'import', s, 'apex', undefined, { count }) + } + return counts + }) +} diff --git a/apps/hq/test/import-apex-full.test.ts b/apps/hq/test/import-apex-full.test.ts new file mode 100644 index 0000000..27b2f9c --- /dev/null +++ b/apps/hq/test/import-apex-full.test.ts @@ -0,0 +1,334 @@ +// apps/hq/test/import-apex-full.test.ts — D20 full-APEX importer v2 (stage → problems → one-txn commit) +import { describe, it, expect } from 'vitest' +import { fyOf } from '@sims/domain' +import { openDb, type DB } from '../src/db' +import { seedIfEmpty } from '../src/seed' +import { decrypt } from '../src/crypto' +import { commitApexFull, parseApexDate, stageApexFull } from '../src/import-apex-full' + +const KEY = '22'.repeat(32) // 64 hex chars = 32 bytes +const TODAY = new Date().toISOString().slice(0, 10) +const FY = fyOf(TODAY) + +// Build a CSV from sparse row objects — headers EXACTLY as the APEX export ships them. +const csv = (headers: string, rows: Record[]): string => + headers + '\n' + + rows.map((r) => headers.split(',').map((h) => r[h] ?? '').join(',')).join('\n') + '\n' + +const CLIENT_HDR = 'SID_NO,BANK,ADDRESS,PHONE,MAIL,DISTRICT,SMS,RTGS,WHATSAPP,MOBILE_APP,DB_PASSWORD,ATM,ANYDESK,SECRETARY_CONTACT_DETAILS,OS,SECTOR' +const BRANCH_HDR = 'ID,BRANCH,SMS,MID,BANK,BRCODE' +const BILL_HDR = 'DAY_DT,BILL_TYP,BILL_DATE,BILL_NO,BANK_ID,BANK_NAME,ADDRESS,MOBILE,MAIL,REMARK,USR,INVOICE_NO,QTY,PER_QTY,DESCRIPTION,INVOICE_TYPE,INVOICE_AMOUNT,PAYMENT_STATUS,PAYMENT_DATE,PAYMENT_AMOUNT,NOTES,AMT,SRNO,GST_RATE,GST_AMOUNT,TOTAL_AMOUNT,CONVERTED_FROM_PROFORMA_ID,GST_APPLICABLE,INVOICE_ID,SCROLL' +const TICKET_HDR = 'ID,DAY_DT,BANK_NAME,BANK_ID,BRID,TYPE,GROUP,ASSIGN_DATE,MOBILE_NO,ASSIGNED_TO,DESCRIPTION,STATUS,CREATED,CREATED_BY,UPDATED,UPDATED_BY,ONLINE_OFFLINE,MAIL_BLOB,MAIL,TRNS_ID,TRNS_FRM,DAY_ID,ENAME,TRANS,TICKET_DROPED,DAY_CLS,OLD_ID,SCROLL' +const SMS_CLIENT_HDR = 'RNO,CID,BANK,DISTRICT,SMS,USERNAME,PASSWORD,PAYMENT,SMS_PROVIDER,ADDED_FEATURES,CREATED_BY,UPDATED_BY,PHONE,TID,SMS_BALANCE,ERR,REMARK,UPDATION,RESELLER,PAYMENT_DATE,INSTALLATION_DATE' +const SMS_MODULE_HDR = 'RNO,SID_NO,SMS_ID,HEAD,COLS,PKG,ACTIV,REMARK,IMP_BY,ON_DAY_DT' +const RTGS_CLIENT_HDR = 'SID_NO,BANK_NAME,ADDRESS_IP,PHONE_NO,RTGS,IP_CODE,E_COLLECTION_CODE,RID,TID,ABB,INTEGRATED_BANK_PARTNERS,REMARK,PAYMENT,INSTALLATION_DATE,INSTALLED_ON,GENERATE_QR,SBI_INWARD_REPORT,PASSBOOK_PROGRAM,IFSC_2025,NOTIFICATION,MISC_RTGS_OUTWARD,PAYMENT_DATE' +const SMS_PROJECT_HDR = 'CID,BANK,DISTRICT,DT_INFO,RNO,SMS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,NEW_FORM,CREATED_BY,UPDATED_BY,TID' +const RTGS_PROJECT_HDR = 'CID,BANK,DT_INFO,RNO,RTGS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,CREATED_BY,UPDATED_BY,TID' + +/** A small but complete workbook exercising every section. */ +function fullFiles(): Record { + return { + clients: csv(CLIENT_HDR, [ + { + SID_NO: '101', BANK: 'Acme Bank', ADDRESS: 'Kochi HQ', PHONE: '9876543210', + MAIL: 'acme@bank.in', DISTRICT: 'Ernakulam', SMS: 'YES', RTGS: 'NO', WHATSAPP: 'Y', + MOBILE_APP: 'NO', DB_PASSWORD: 's3cret-db', ATM: 'NO', ANYDESK: '123 456 789', + SECRETARY_CONTACT_DETAILS: 'Secretary John 9999', OS: 'Windows 10', SECTOR: 'Co-op', + }, + { + SID_NO: '102', BANK: 'Beta Bank', DISTRICT: 'Kollam', SMS: 'NO', RTGS: 'YES', + WHATSAPP: 'NO', MOBILE_APP: 'NO', DB_PASSWORD: 'NO DATA', ATM: 'NO', + ANYDESK: 'NO DATA', SECTOR: 'Service', + }, + ]), + branches: csv(BRANCH_HDR, [ + { ID: '1', BRANCH: 'Main Branch', MID: '101', BANK: 'Acme Bank', BRCODE: 'BR001' }, + ]), + bills: csv(BILL_HDR, [ + { // paid current-FY invoice — rupees→paise, payment+allocation, series seed + BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '101', INVOICE_NO: 'SMS/2026/0042', + INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '1000', GST_AMOUNT: '180', TOTAL_AMOUNT: '1180', + PAYMENT_STATUS: 'PAID', PAYMENT_DATE: `${TODAY} 00:00:00`, PAYMENT_AMOUNT: '1180', + }, + { // proforma with '' tax ('→0) and '' total ('→taxable+tax); never gets a payment + BILL_DATE: '2025-08-18 00:00:00', BANK_ID: '102', INVOICE_NO: 'PI/2025/007', + INVOICE_TYPE: 'PROFORMA INVOICE', INVOICE_AMOUNT: '500', GST_AMOUNT: '', TOTAL_AMOUNT: '', + }, + ]), + tickets: csv(TICKET_HDR, [ + { + ID: '1', BANK_ID: '101', TYPE: 'MODIFICATION', GROUP: 'SMS', + ASSIGN_DATE: '2025-02-28 00:00:00', DESCRIPTION: 'Fix template', STATUS: 'CLOSED', + CREATED: '2025-02-28 09:00:00', UPDATED: '2025-03-02 10:00:00', ONLINE_OFFLINE: 'ONLINE', + }, + { // blank BANK_ID → exact BANK_NAME match; DD-MON-YYYY open date; WAITING* status + ID: '2', BANK_NAME: 'Acme Bank', DAY_DT: '01-MAR-2025', + STATUS: 'WAITING FOR CONFIRMATION', DESCRIPTION: 'Waiting one', + }, + { // TICKET_DROPED='Y' with a non-terminal status → dropped, closed_on from UPDATED + ID: '3', BANK_ID: '102', STATUS: '', TICKET_DROPED: 'Y', DESCRIPTION: 'Dropped one', + CREATED: '2025-04-01 09:00:00', UPDATED: '2025-04-02 00:00:00', + }, + ]), + sms_clients: csv(SMS_CLIENT_HDR, [ + { // CID 102's SMS flag said NO — this row wins and creates the assignment + RNO: '1', CID: '102', BANK: 'Beta Bank', USERNAME: 'betauser', PASSWORD: 'smsPass9', + SMS_PROVIDER: 'AltSMS', PHONE: '0474222', SMS_BALANCE: '5000', REMARK: 'ok', + RESELLER: 'RSL', PAYMENT: 'PAID 2025', INSTALLATION_DATE: '10-NOV-2023', + }, + ]), + sms_modules: csv(SMS_MODULE_HDR, [ + { RNO: '1', SID_NO: '102', HEAD: 'ATM ALERT', ACTIV: 'Y' }, + { RNO: '2', SID_NO: '102', HEAD: 'BALANCE ALERT', ACTIV: 'Y' }, + { RNO: '3', SID_NO: '102', HEAD: 'OLD ALERT', ACTIV: 'N' }, + ]), + rtgs_clients: csv(RTGS_CLIENT_HDR, [ + { + SID_NO: '102', BANK_NAME: 'Beta Bank', ADDRESS_IP: '10.0.0.5', PHONE_NO: '333', + IP_CODE: 'IPC1', ABB: 'BETA', INTEGRATED_BANK_PARTNERS: 'SBI', + INSTALLED_ON: '2024-01-05 00:00:00', GENERATE_QR: 'YES', PAYMENT: 'PAID', + }, + ]), + sms_projects: csv(SMS_PROJECT_HDR, [ + { CID: '101', DT_INFO: '13-05-2024', TICKET_STATUS: 'COMPLETED', DESCRIPTION: 'Install SMS' }, + ]), + rtgs_projects: csv(RTGS_PROJECT_HDR, [ + { CID: '102', DT_INFO: 'not-a-date', TICKET_STATUS: 'PENDING', DESCRIPTION: 'RTGS setup' }, + ]), + } +} + +async function freshDb(): Promise { + const db = openDb(':memory:') + await seedIfEmpty(db) + return db +} + +describe('parseApexDate', () => { + it('handles every format seen in the APEX export', () => { + expect(parseApexDate('2025-08-18 00:00:00')).toBe('2025-08-18') // ISO datetime + expect(parseApexDate('10-NOV-2023 10:57:18')).toBe('2023-11-10') // DD-MON-YYYY + expect(parseApexDate('08/29/2024')).toBe('2024-08-29') // second >12 → MM/DD + expect(parseApexDate('13-05-2024')).toBe('2024-05-13') // first >12 → DD-MM + expect(parseApexDate('10-11-2023')).toBe('2023-11-10') // ambiguous → assume DD-MM + expect(parseApexDate('')).toBeNull() + expect(parseApexDate('not-a-date')).toBeNull() + expect(parseApexDate('99-99-2024')).toBeNull() + }) +}) + +describe('APEX full importer v2', () => { + it('stage: clean workbook → per-section counts, zero problems, and NO writes', async () => { + const db = await freshDb() + const report = await stageApexFull(db, fullFiles(), KEY) + expect(report.clients).toEqual({ staged: 2, problems: [] }) + expect(report.branches).toEqual({ staged: 1, problems: [] }) + expect(report.bills).toEqual({ staged: 2, problems: [] }) + expect(report.tickets).toEqual({ staged: 3, problems: [] }) + expect(report.sms_clients).toEqual({ staged: 1, problems: [] }) + expect(report.sms_modules).toEqual({ staged: 3, problems: [] }) + expect(report.rtgs_clients).toEqual({ staged: 1, problems: [] }) + expect(report.sms_projects).toEqual({ staged: 1, problems: [] }) + expect(report.rtgs_projects).toEqual({ staged: 1, problems: [] }) + // Pure validation: no clients, no documents, no import audit rows. + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM audit_log WHERE action='import'`))!.n).toBe(0) + }) + + it('stage: unknown join keys are per-row problems with row numbers', async () => { + const db = await freshDb() + const report = await stageApexFull(db, { + clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), + branches: csv(BRANCH_HDR, [{ ID: '9', BRANCH: 'Ghost', MID: '999' }]), + tickets: csv(TICKET_HDR, [{ ID: '9', BANK_ID: '999', STATUS: 'OPEN', DAY_DT: '2025-01-01 00:00:00' }]), + sms_projects: csv(SMS_PROJECT_HDR, [{ CID: '999', DT_INFO: '2025-01-01' }]), + }, KEY) + expect(report.branches.problems).toEqual(['row 1: unknown client MID=999']) + expect(report.tickets.problems).toEqual(['row 1: unknown client BANK_ID=999']) + expect(report.sms_projects.problems).toEqual(['row 1: unknown client CID=999']) + }) + + it('commit: full happy path lands every section in one transaction', async () => { + const db = await freshDb() + const res = await commitApexFull(db, 'u-test', fullFiles(), KEY) + expect(res.clients).toBe(2) + expect(res.client_modules).toBe(4) // 101: SMS+WHATSAPP (flags); 102: RTGS (flag) + SMS (data wins) + expect(res.branches).toBe(1) + expect(res.bills).toBe(2) + expect(res.payments).toBe(1) + expect(res.tickets).toBe(3) + expect(res.sms_projects).toBe(1) + expect(res.rtgs_projects).toBe(1) + expect(res.seeded).toEqual({ fy: FY, lastSeq: 42 }) + + // Clients: code = SID_NO, contacts carry bank + secretary, support fields ride along. + const c101 = (await db.get<{ + id: string; name: string; address: string; district: string; sector: string + os: string; anydesk: string; contacts: string; status: string; source: string + state_code: string; db_password_enc: string + }>(`SELECT * FROM client WHERE code='101'`))! + expect(c101).toMatchObject({ + name: 'Acme Bank', address: 'Kochi HQ', district: 'Ernakulam', sector: 'Co-op', + os: 'Windows 10', anydesk: '123 456 789', status: 'active', source: 'apex', state_code: '32', + }) + expect(JSON.parse(c101.contacts)).toEqual([ + { name: 'Acme Bank', phone: '9876543210', email: 'acme@bank.in' }, + { name: 'Secretary John 9999', role: 'secretary' }, + ]) + // DB password encrypted at rest, decryptable with the key. + expect(decrypt(c101.db_password_enc, KEY)).toBe('s3cret-db') + const c102 = (await db.get<{ anydesk: string | null; db_password_enc: string | null }>( + `SELECT anydesk, db_password_enc FROM client WHERE code='102'`))! + expect(c102).toEqual({ anydesk: null, db_password_enc: null }) // 'NO DATA' → nothing stored + + // Flags → live yearly assignments on get-or-created modules. + const modules = await db.all<{ code: string; name: string }>( + `SELECT code, name FROM module WHERE code IN ('SMS','RTGS','WHATSAPP') ORDER BY code`) + expect(modules).toEqual([ + { code: 'RTGS', name: 'RTGS' }, { code: 'SMS', name: 'Bulk SMS' }, + { code: 'WHATSAPP', name: 'WhatsApp' }, + ]) + const codes101 = await db.all<{ code: string; status: string; kind: string }>( + `SELECT m.code, cm.status, cm.kind FROM client_module cm + JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id + WHERE c.code='101' ORDER BY m.code`) + expect(codes101).toEqual([ + { code: 'SMS', status: 'live', kind: 'yearly' }, + { code: 'WHATSAPP', status: 'live', kind: 'yearly' }, + ]) + + // Branch joined by MID. + const branch = await db.get(`SELECT b.name, b.code, b.active FROM client_branch b + JOIN client c ON c.id=b.client_id WHERE c.code='101'`) + expect(branch).toEqual({ name: 'Main Branch', code: 'BR001', active: 1 }) + + // Bills: rupees→paise, intra-state split, payment + allocation, series seed. + const inv = (await db.get<{ + id: string; doc_type: string; fy: string; status: string; taxable_paise: number + cgst_paise: number; sgst_paise: number; igst_paise: number; round_off_paise: number + payable_paise: number; source: string; payload: string + }>(`SELECT * FROM document WHERE doc_no='SMS/2026/0042'`))! + expect(inv).toMatchObject({ + doc_type: 'INVOICE', fy: FY, status: 'paid', taxable_paise: 100000, + cgst_paise: 9000, sgst_paise: 9000, igst_paise: 0, round_off_paise: 0, + payable_paise: 118000, source: 'apex', + }) + expect(JSON.parse(inv.payload)).toMatchObject({ lines: [], totals: { payablePaise: 118000 } }) + const pi = await db.get(`SELECT doc_type, status, taxable_paise, payable_paise, cgst_paise + FROM document WHERE doc_no='PI/2025/007'`) + expect(pi).toEqual({ + doc_type: 'PROFORMA', status: 'sent', taxable_paise: 50000, payable_paise: 50000, cgst_paise: 0, + }) + const pay = (await db.get<{ id: string; mode: string; reference: string; amount_paise: number; received_on: string }>( + `SELECT * FROM payment`))! + expect(pay).toMatchObject({ + mode: 'bank', reference: 'APEX import', amount_paise: 118000, received_on: TODAY, + }) + const alloc = await db.get(`SELECT payment_id, document_id, amount_paise FROM payment_allocation`) + expect(alloc).toEqual({ payment_id: pay.id, document_id: inv.id, amount_paise: 118000 }) + const series = await db.get(`SELECT next_seq FROM doc_series WHERE doc_type='INVOICE' AND fy=?`, FY) + expect(series).toEqual({ next_seq: 43 }) // continues after the legacy 0042 tail + + // Tickets: status mapping + join fallbacks. + const t1 = await db.get(`SELECT status, opened_on, closed_on, online_offline, module_code, source + FROM ticket WHERE description='Fix template'`) + expect(t1).toEqual({ + status: 'closed', opened_on: '2025-02-28', closed_on: '2025-03-02', + online_offline: 'online', module_code: 'SMS', source: 'apex', + }) + const t2 = (await db.get<{ status: string; opened_on: string; client_id: string }>( + `SELECT status, opened_on, client_id FROM ticket WHERE description='Waiting one'`))! + expect(t2.status).toBe('waiting') + expect(t2.opened_on).toBe('2025-03-01') // DD-MON-YYYY DAY_DT + expect(t2.client_id).toBe(c101.id) // blank BANK_ID resolved by exact BANK_NAME + const t3 = await db.get(`SELECT status, opened_on, closed_on FROM ticket WHERE description='Dropped one'`) + expect(t3).toEqual({ status: 'dropped', opened_on: '2025-04-01', closed_on: '2025-04-02' }) + + // SMS service data landed on 102's (data-created) SMS module. + const sms102 = (await db.get<{ provider: string; username: string; password_enc: string; details: string; remark: string }>( + `SELECT cm.provider, cm.username, cm.password_enc, cm.details, cm.remark FROM client_module cm + JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id + WHERE c.code='102' AND m.code='SMS'`))! + expect(sms102.provider).toBe('AltSMS') + expect(sms102.username).toBe('betauser') + expect(decrypt(sms102.password_enc, KEY)).toBe('smsPass9') + expect(sms102.remark).toBe('ok') + const smsDetails = JSON.parse(sms102.details) as { label: string; value: string }[] + expect(smsDetails).toContainEqual({ label: 'SMS balance', value: '5000' }) + expect(smsDetails).toContainEqual({ label: 'Installed', value: '10-NOV-2023' }) + expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT, BALANCE ALERT' }) + + // RTGS service data on 102's RTGS module. + const rtgs102 = (await db.get<{ details: string }>( + `SELECT cm.details FROM client_module cm + JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id + WHERE c.code='102' AND m.code='RTGS'`))! + const rtgsDetails = JSON.parse(rtgs102.details) as { label: string; value: string }[] + expect(rtgsDetails).toContainEqual({ label: 'IP', value: '10.0.0.5' }) + expect(rtgsDetails).toContainEqual({ label: 'Installed', value: '2024-01-05 00:00:00' }) + expect(rtgsDetails).toContainEqual({ label: 'QR', value: 'YES' }) + + // Projects → interaction rows; unparseable DT_INFO falls back to today + a note. + const sp = (await db.get<{ on_date: string; notes: string; type_code: string }>( + `SELECT on_date, notes, type_code FROM interaction WHERE notes LIKE '[SMS project]%'`))! + expect(sp.type_code).toBe('project') + expect(sp.on_date).toBe('2024-05-13') + expect(sp.notes).toBe('[SMS project] COMPLETED — Install SMS') + const rp = (await db.get<{ on_date: string; notes: string }>( + `SELECT on_date, notes FROM interaction WHERE notes LIKE '[RTGS project]%'`))! + expect(rp.on_date).toBe(TODAY) + expect(rp.notes).toContain('DT_INFO unparsed: not-a-date') + + // One summary audit per section (plus the series seed) — and never a plaintext secret. + const auditEntities = (await db.all<{ entity: string }>( + `SELECT entity FROM audit_log WHERE action='import'`)).map((r) => r.entity).sort() + expect(auditEntities).toEqual(['bills', 'branches', 'clients', 'doc_series', 'rtgs_clients', + 'rtgs_projects', 'sms_clients', 'sms_modules', 'sms_projects', 'tickets']) + const auditDump = JSON.stringify(await db.all(`SELECT * FROM audit_log`)) + expect(auditDump).not.toContain('s3cret-db') + expect(auditDump).not.toContain('smsPass9') + }) + + it('refuses plaintext passwords loudly when no encryption key is configured', async () => { + const db = await freshDb() + const files = fullFiles() + const report = await stageApexFull(db, files, '') + expect(report.clients.problems).toEqual([ + 'row 1: DB_PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted', + ]) + expect(report.sms_clients.problems).toEqual([ + 'row 1: PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted', + ]) + await expect(commitApexFull(db, 'u-test', files, '')).rejects.toThrow(/HQ_SECRET_KEY/) + // Refused before the transaction — nothing landed. + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) + }) + + it('duplicate doc_no is a problem that blocks the whole commit', async () => { + const db = await freshDb() + const files = { + clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), + bills: csv(BILL_HDR, [ + { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '100' }, + { BILL_DATE: '2025-05-02 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '200' }, + ]), + } + const report = await stageApexFull(db, files, KEY) + expect(report.bills.problems).toEqual(['row 2: duplicate doc_no: INV-1']) + await expect(commitApexFull(db, 'u-test', files, KEY)).rejects.toThrow(/duplicate doc_no/) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0) + expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0) + }) + + it('rejects an unknown INVOICE_TYPE instead of guessing', async () => { + const db = await freshDb() + const report = await stageApexFull(db, { + clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]), + bills: csv(BILL_HDR, [ + { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'X-1', INVOICE_TYPE: 'RECEIPT', INVOICE_AMOUNT: '100' }, + ]), + }, KEY) + expect(report.bills.problems).toEqual(['row 1: unknown INVOICE_TYPE: RECEIPT']) + }) +})