import fs from 'node:fs' import path from 'node:path' import { fromRupees, fyOf, uuidv7, validateGstin } from '@sims/domain' import { writeAudit } from './audit' import { openDb, type DB } from './db' import { seedIfEmpty } from './seed' import { seedSeries } from './series' /** * APEX cutover importer (spec §4): stage CSVs → verification report → * commit + INVOICE series seed. Staged rows carry their own problem list; * commit refuses to run until every staged row is clean. */ // ---------- tiny CSV parser (header row, comma, double-quote escaping) ---------- function parseCsv(text: string): Record[] { const rows: string[][] = [] let field = '' let row: string[] = [] let inQuotes = false const endField = (): void => { row.push(field); field = '' } const endRow = (): void => { endField(); rows.push(row); row = [] } for (let i = 0; i < text.length; i++) { const ch = text[i]! if (inQuotes) { if (ch === '"') { if (text[i + 1] === '"') { field += '"'; i++ } else inQuotes = false } else field += ch } else if (ch === '"') inQuotes = true else if (ch === ',') endField() else if (ch === '\n') endRow() else if (ch !== '\r') field += ch } if (field !== '' || row.length > 0) endRow() const header = (rows.shift() ?? []).map((h) => h.trim()) return rows .filter((r) => r.some((c) => c.trim() !== '')) .map((r) => Object.fromEntries(header.map((h, i) => [h, (r[i] ?? '').trim()]))) } // ---------- staging ---------- const CLIENT_STATUSES = new Set(['lead', 'active', 'dormant', 'lost']) const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/ function stageClients(db: DB, records: Record[]): number { db.prepare(`DELETE FROM stg_client`).run() const existing = new Set( (db.prepare(`SELECT code FROM client`).all() as { code: string }[]).map((r) => r.code), ) const seen = new Set() const insert = db.prepare( `INSERT INTO stg_client (row_no, code, name, gstin, state_code, address, phone, email, status, problems) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) records.forEach((r, i) => { const problems: string[] = [] const code = r['code'] ?? '' if (code === '') problems.push('missing code') else if (seen.has(code) || existing.has(code)) problems.push(`duplicate code: ${code}`) if (code !== '') seen.add(code) if ((r['name'] ?? '') === '') problems.push('missing name') const gstin = r['gstin'] ?? '' if (gstin !== '' && !validateGstin(gstin).ok) problems.push(`bad GSTIN: ${gstin}`) const status = r['status'] ?? '' if (status !== '' && !CLIENT_STATUSES.has(status)) problems.push(`bad status: ${status}`) insert.run( i + 1, code, r['name'] ?? '', gstin, r['state_code'] ?? '', r['address'] ?? '', r['phone'] ?? '', r['email'] ?? '', status, JSON.stringify(problems), ) }) return records.length } function paiseOrNull(raw: string): number | null { if (raw === '') return null const n = Number(raw) return Number.isFinite(n) ? fromRupees(n) : null } function stageInvoices(db: DB, records: Record[]): number { db.prepare(`DELETE FROM stg_invoice`).run() const existing = new Set( (db.prepare(`SELECT doc_no FROM document WHERE doc_no IS NOT NULL`).all() as { doc_no: string }[]) .map((r) => r.doc_no), ) const seen = new Set() const insert = db.prepare( `INSERT INTO stg_invoice (row_no, client_code, doc_no, doc_date, taxable_paise, tax_paise, total_paise, paid, problems) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) records.forEach((r, i) => { const problems: string[] = [] if ((r['client_code'] ?? '') === '') problems.push('missing client_code') const docNo = r['doc_no'] ?? '' if (docNo === '') problems.push('missing doc_no') else if (seen.has(docNo) || existing.has(docNo)) problems.push(`duplicate doc_no: ${docNo}`) if (docNo !== '') seen.add(docNo) const docDate = r['doc_date'] ?? '' if (!ISO_DATE.test(docDate) || Number.isNaN(Date.parse(docDate))) problems.push(`bad date: ${docDate}`) const taxable = paiseOrNull(r['taxable'] ?? '') const tax = paiseOrNull(r['tax'] ?? '') const total = paiseOrNull(r['total'] ?? '') if (taxable === null) problems.push(`bad taxable: ${r['taxable'] ?? ''}`) if (tax === null) problems.push(`bad tax: ${r['tax'] ?? ''}`) if (total === null) problems.push(`bad total: ${r['total'] ?? ''}`) insert.run( i + 1, r['client_code'] ?? '', docNo, docDate, taxable, tax, total, r['paid'] === '1' ? 1 : 0, JSON.stringify(problems), ) }) return records.length } export function stageCsv(db: DB, kind: 'clients' | 'invoices', csvText: string): { staged: number } { const records = parseCsv(csvText) const staged = kind === 'clients' ? stageClients(db, records) : stageInvoices(db, records) writeAudit(db, 'system', 'stage', `stg_${kind}`, kind, undefined, { staged }) return { staged } } // ---------- verification report ---------- export interface VerificationReport { clients: { staged: number; problems: number } invoices: { staged: number; problems: number; totalPaise: number } samples: { firstClients: string[]; lastInvoices: string[] } } export function verificationReport(db: DB): VerificationReport { const c = db.prepare( `SELECT COUNT(*) AS staged, COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems FROM stg_client`, ).get() as { staged: number; problems: number } const i = db.prepare( `SELECT COUNT(*) AS staged, COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems, COALESCE(SUM(total_paise), 0) AS totalPaise FROM stg_invoice`, ).get() as { staged: number; problems: number; totalPaise: number } const firstClients = (db.prepare( `SELECT code, name FROM stg_client ORDER BY row_no LIMIT 5`, ).all() as { code: string; name: string }[]).map((r) => `${r.code} ${r.name}`.trim()) const lastInvoices = (db.prepare( `SELECT doc_no FROM stg_invoice ORDER BY row_no DESC LIMIT 5`, ).all() as { doc_no: string }[]).map((r) => r.doc_no) return { clients: { staged: c.staged, problems: c.problems }, invoices: { staged: i.staged, problems: i.problems, totalPaise: i.totalPaise }, samples: { firstClients, lastInvoices }, } } // ---------- commit ---------- interface StgClientRow { row_no: number; code: string; name: string; gstin: string; state_code: string address: string; phone: string; email: string; status: string; problems: string } interface StgInvoiceRow { row_no: number; client_code: string; doc_no: string; doc_date: string taxable_paise: number; tax_paise: number; total_paise: number; paid: number; problems: string } export interface CommitResult { clients: number invoices: number seeded: { fy: string; lastSeq: number } | null } function companyStateCode(db: DB): string { const row = db.prepare(`SELECT value FROM setting WHERE key='company.state_code'`) .get() as { value: string } | undefined // Silent defaults on place-of-supply are how wrong GST reaches the portal — fail loudly. if (row === undefined) throw new Error(`Setting 'company.state_code' is not configured`) return row.value } export function commitImport(db: DB, userId: string): CommitResult { return db.transaction((): CommitResult => { const bad = (db.prepare( `SELECT (SELECT COUNT(*) FROM stg_client WHERE problems <> '[]') + (SELECT COUNT(*) FROM stg_invoice WHERE problems <> '[]') AS n`, ).get() as { n: number }).n if (bad > 0) throw new Error(`${bad} staged row(s) have problems — fix the CSV and re-stage before commit`) const ourState = companyStateCode(db) const now = new Date().toISOString() // Clients — source 'apex' marks the cutover rows. const stgClients = db.prepare(`SELECT * FROM stg_client ORDER BY row_no`).all() as StgClientRow[] const insertClient = db.prepare( `INSERT INTO client (id, code, name, gstin, state_code, address, contacts, status, source, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'apex', ?)`, ) for (const c of stgClients) { const id = uuidv7() const contacts = c.phone !== '' || c.email !== '' ? [{ name: c.name, ...(c.phone !== '' ? { phone: c.phone } : {}), ...(c.email !== '' ? { email: c.email } : {}), }] : [] insertClient.run( id, c.code, c.name, c.gstin !== '' ? c.gstin : null, c.state_code !== '' ? c.state_code : ourState, c.address, JSON.stringify(contacts), c.status !== '' ? c.status : 'active', now, ) writeAudit(db, userId, 'import', 'client', id, undefined, { code: c.code, name: c.name, source: 'apex' }) } // Invoices — issued, history-only documents (empty payload lines) that keep // their legacy APEX numbers; the series is seeded past them below. const stgInvoices = db.prepare(`SELECT * FROM stg_invoice ORDER BY row_no`).all() as StgInvoiceRow[] const insertDoc = db.prepare( `INSERT INTO document (id, doc_type, doc_no, fy, client_id, doc_date, status, ref_doc_id, taxable_paise, cgst_paise, sgst_paise, igst_paise, round_off_paise, payable_paise, payload, source, created_by, created_at) VALUES (?, 'INVOICE', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, 'apex', ?, ?)`, ) for (const inv of stgInvoices) { const client = db.prepare(`SELECT id, state_code FROM client WHERE code=?`) .get(inv.client_code) as { id: string; state_code: string } | undefined if (client === undefined) { throw new Error(`Invoice ${inv.doc_no}: client code ${inv.client_code} not found — stage clients.csv first`) } const intra = client.state_code === ourState const cgst = intra ? Math.floor(inv.tax_paise / 2) : 0 const sgst = intra ? inv.tax_paise - cgst : 0 const igst = intra ? 0 : inv.tax_paise const roundOff = inv.total_paise - inv.taxable_paise - inv.tax_paise const totals = { grossPaise: inv.taxable_paise, discountPaise: 0, taxablePaise: inv.taxable_paise, cgstPaise: cgst, sgstPaise: sgst, igstPaise: igst, cessPaise: 0, roundOffPaise: roundOff, payablePaise: inv.total_paise, savingsVsMrpPaise: 0, } const id = uuidv7() const status = inv.paid === 1 ? 'paid' : 'sent' insertDoc.run( id, inv.doc_no, fyOf(inv.doc_date), client.id, inv.doc_date, status, inv.taxable_paise, cgst, sgst, igst, roundOff, inv.total_paise, JSON.stringify({ lines: [], totals }), userId, now, ) writeAudit(db, userId, 'import', 'document', id, undefined, { docNo: inv.doc_no, status, payablePaise: inv.total_paise, source: 'apex' }) } // Series seed (spec §4): continue INVOICE numbering after the last APEX // number of the current FY. Legacy prefixes differ, so only the numeric // tail matters. const currentFy = fyOf(new Date().toISOString().slice(0, 10)) let seeded: CommitResult['seeded'] = null const tails = stgInvoices .filter((inv) => fyOf(inv.doc_date) === currentFy) .map((inv) => /(\d+)$/.exec(inv.doc_no)?.[1]) .filter((t): t is string => t !== undefined) .map((t) => Number(t)) if (tails.length > 0) { const lastSeq = Math.max(...tails) seedSeries(db, 'INVOICE', currentFy, lastSeq) seeded = { fy: currentFy, lastSeq } writeAudit(db, userId, 'import', 'doc_series', `INVOICE/${currentFy}`, undefined, { lastSeq }) } // Commit consumes the staging area. db.prepare(`DELETE FROM stg_client`).run() db.prepare(`DELETE FROM stg_invoice`).run() return { clients: stgClients.length, invoices: stgInvoices.length, seeded } })() } // ---------- CLI: npm run import -- --dir [--commit] ---------- if (process.argv[1] !== undefined && /import-apex\.(ts|cjs|js)$/.test(process.argv[1])) { const args = process.argv.slice(2) const dirIdx = args.indexOf('--dir') const dir = dirIdx >= 0 ? args[dirIdx + 1] : undefined if (dir === undefined) { console.error('Usage: npm run import -- --dir [--commit]') process.exit(1) } const db = openDb(process.env['HQ_DATA_DIR']) seedIfEmpty(db) const clients = stageCsv(db, 'clients', fs.readFileSync(path.join(dir, 'clients.csv'), 'utf8')) const invoices = stageCsv(db, 'invoices', fs.readFileSync(path.join(dir, 'invoices.csv'), 'utf8')) console.log(`Staged ${clients.staged} clients, ${invoices.staged} invoices from ${dir}`) console.log(JSON.stringify(verificationReport(db), null, 2)) if (args.includes('--commit')) { const out = commitImport(db, 'system') console.log(`Committed: ${out.clients} clients, ${out.invoices} invoices; series seeded:`, out.seeded ?? 'no current-FY invoices') } else { console.log('Dry run — verify the report above, then re-run with --commit to apply.') } }