feat(hq): seed + apex csv importer with verification and series seeding (HQ-1 task 13)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/hq/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "build": "node build-server.mjs",
     "start": "npm run build && node dist/server.cjs",
+    "import": "tsx src/import-apex.ts",
     "typecheck": "tsc -p tsconfig.json"
   },
   "dependencies": {

apps/hq/src/import-apex.ts

@@ -0,0 +1,302 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { fromRupees, fyOf, uuidv7, validateGstin } from '@sims/domain'
+import { writeAudit } from './audit'
+import { openDb, type DB } from './db'
+import { seedIfEmpty } from './seed'
+import { seedSeries } from './series'
+
+/**
+ * APEX cutover importer (spec §4): stage CSVs → verification report →
+ * commit + INVOICE series seed. Staged rows carry their own problem list;
+ * commit refuses to run until every staged row is clean.
+ */
+
+// ---------- tiny CSV parser (header row, comma, double-quote escaping) ----------
+
+function parseCsv(text: string): Record<string, string>[] {
+  const rows: string[][] = []
+  let field = ''
+  let row: string[] = []
+  let inQuotes = false
+  const endField = (): void => { row.push(field); field = '' }
+  const endRow = (): void => { endField(); rows.push(row); row = [] }
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]!
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') { field += '"'; i++ } else inQuotes = false
+      } else field += ch
+    } else if (ch === '"') inQuotes = true
+    else if (ch === ',') endField()
+    else if (ch === '\n') endRow()
+    else if (ch !== '\r') field += ch
+  }
+  if (field !== '' || row.length > 0) endRow()
+  const header = (rows.shift() ?? []).map((h) => h.trim())
+  return rows
+    .filter((r) => r.some((c) => c.trim() !== ''))
+    .map((r) => Object.fromEntries(header.map((h, i) => [h, (r[i] ?? '').trim()])))
+}
+
+// ---------- staging ----------
+
+const CLIENT_STATUSES = new Set(['lead', 'active', 'dormant', 'lost'])
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
+
+function stageClients(db: DB, records: Record<string, string>[]): number {
+  db.prepare(`DELETE FROM stg_client`).run()
+  const existing = new Set(
+    (db.prepare(`SELECT code FROM client`).all() as { code: string }[]).map((r) => r.code),
+  )
+  const seen = new Set<string>()
+  const insert = db.prepare(
+    `INSERT INTO stg_client (row_no, code, name, gstin, state_code, address, phone, email, status, problems)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  )
+  records.forEach((r, i) => {
+    const problems: string[] = []
+    const code = r['code'] ?? ''
+    if (code === '') problems.push('missing code')
+    else if (seen.has(code) || existing.has(code)) problems.push(`duplicate code: ${code}`)
+    if (code !== '') seen.add(code)
+    if ((r['name'] ?? '') === '') problems.push('missing name')
+    const gstin = r['gstin'] ?? ''
+    if (gstin !== '' && !validateGstin(gstin).ok) problems.push(`bad GSTIN: ${gstin}`)
+    const status = r['status'] ?? ''
+    if (status !== '' && !CLIENT_STATUSES.has(status)) problems.push(`bad status: ${status}`)
+    insert.run(
+      i + 1, code, r['name'] ?? '', gstin, r['state_code'] ?? '',
+      r['address'] ?? '', r['phone'] ?? '', r['email'] ?? '', status,
+      JSON.stringify(problems),
+    )
+  })
+  return records.length
+}
+
+function paiseOrNull(raw: string): number | null {
+  if (raw === '') return null
+  const n = Number(raw)
+  return Number.isFinite(n) ? fromRupees(n) : null
+}
+
+function stageInvoices(db: DB, records: Record<string, string>[]): number {
+  db.prepare(`DELETE FROM stg_invoice`).run()
+  const existing = new Set(
+    (db.prepare(`SELECT doc_no FROM document WHERE doc_no IS NOT NULL`).all() as { doc_no: string }[])
+      .map((r) => r.doc_no),
+  )
+  const seen = new Set<string>()
+  const insert = db.prepare(
+    `INSERT INTO stg_invoice (row_no, client_code, doc_no, doc_date, taxable_paise, tax_paise, total_paise, paid, problems)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  )
+  records.forEach((r, i) => {
+    const problems: string[] = []
+    if ((r['client_code'] ?? '') === '') problems.push('missing client_code')
+    const docNo = r['doc_no'] ?? ''
+    if (docNo === '') problems.push('missing doc_no')
+    else if (seen.has(docNo) || existing.has(docNo)) problems.push(`duplicate doc_no: ${docNo}`)
+    if (docNo !== '') seen.add(docNo)
+    const docDate = r['doc_date'] ?? ''
+    if (!ISO_DATE.test(docDate) || Number.isNaN(Date.parse(docDate))) problems.push(`bad date: ${docDate}`)
+    const taxable = paiseOrNull(r['taxable'] ?? '')
+    const tax = paiseOrNull(r['tax'] ?? '')
+    const total = paiseOrNull(r['total'] ?? '')
+    if (taxable === null) problems.push(`bad taxable: ${r['taxable'] ?? ''}`)
+    if (tax === null) problems.push(`bad tax: ${r['tax'] ?? ''}`)
+    if (total === null) problems.push(`bad total: ${r['total'] ?? ''}`)
+    insert.run(
+      i + 1, r['client_code'] ?? '', docNo, docDate,
+      taxable, tax, total, r['paid'] === '1' ? 1 : 0,
+      JSON.stringify(problems),
+    )
+  })
+  return records.length
+}
+
+export function stageCsv(db: DB, kind: 'clients' | 'invoices', csvText: string): { staged: number } {
+  const records = parseCsv(csvText)
+  const staged = kind === 'clients' ? stageClients(db, records) : stageInvoices(db, records)
+  writeAudit(db, 'system', 'stage', `stg_${kind}`, kind, undefined, { staged })
+  return { staged }
+}
+
+// ---------- verification report ----------
+
+export interface VerificationReport {
+  clients: { staged: number; problems: number }
+  invoices: { staged: number; problems: number; totalPaise: number }
+  samples: { firstClients: string[]; lastInvoices: string[] }
+}
+
+export function verificationReport(db: DB): VerificationReport {
+  const c = db.prepare(
+    `SELECT COUNT(*) AS staged,
+            COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems
+     FROM stg_client`,
+  ).get() as { staged: number; problems: number }
+  const i = db.prepare(
+    `SELECT COUNT(*) AS staged,
+            COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems,
+            COALESCE(SUM(total_paise), 0) AS totalPaise
+     FROM stg_invoice`,
+  ).get() as { staged: number; problems: number; totalPaise: number }
+  const firstClients = (db.prepare(
+    `SELECT code, name FROM stg_client ORDER BY row_no LIMIT 5`,
+  ).all() as { code: string; name: string }[]).map((r) => `${r.code} ${r.name}`.trim())
+  const lastInvoices = (db.prepare(
+    `SELECT doc_no FROM stg_invoice ORDER BY row_no DESC LIMIT 5`,
+  ).all() as { doc_no: string }[]).map((r) => r.doc_no)
+  return {
+    clients: { staged: c.staged, problems: c.problems },
+    invoices: { staged: i.staged, problems: i.problems, totalPaise: i.totalPaise },
+    samples: { firstClients, lastInvoices },
+  }
+}
+
+// ---------- commit ----------
+
+interface StgClientRow {
+  row_no: number; code: string; name: string; gstin: string; state_code: string
+  address: string; phone: string; email: string; status: string; problems: string
+}
+
+interface StgInvoiceRow {
+  row_no: number; client_code: string; doc_no: string; doc_date: string
+  taxable_paise: number; tax_paise: number; total_paise: number; paid: number; problems: string
+}
+
+export interface CommitResult {
+  clients: number
+  invoices: number
+  seeded: { fy: string; lastSeq: number } | null
+}
+
+function companyStateCode(db: DB): string {
+  const row = db.prepare(`SELECT value FROM setting WHERE key='company.state_code'`)
+    .get() as { value: string } | undefined
+  // Silent defaults on place-of-supply are how wrong GST reaches the portal — fail loudly.
+  if (row === undefined) throw new Error(`Setting 'company.state_code' is not configured`)
+  return row.value
+}
+
+export function commitImport(db: DB, userId: string): CommitResult {
+  return db.transaction((): CommitResult => {
+    const bad = (db.prepare(
+      `SELECT (SELECT COUNT(*) FROM stg_client WHERE problems <> '[]')
+            + (SELECT COUNT(*) FROM stg_invoice WHERE problems <> '[]') AS n`,
+    ).get() as { n: number }).n
+    if (bad > 0) throw new Error(`${bad} staged row(s) have problems — fix the CSV and re-stage before commit`)
+
+    const ourState = companyStateCode(db)
+    const now = new Date().toISOString()
+
+    // Clients — source 'apex' marks the cutover rows.
+    const stgClients = db.prepare(`SELECT * FROM stg_client ORDER BY row_no`).all() as StgClientRow[]
+    const insertClient = db.prepare(
+      `INSERT INTO client (id, code, name, gstin, state_code, address, contacts, status, source, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'apex', ?)`,
+    )
+    for (const c of stgClients) {
+      const id = uuidv7()
+      const contacts = c.phone !== '' || c.email !== ''
+        ? [{
+            name: c.name,
+            ...(c.phone !== '' ? { phone: c.phone } : {}),
+            ...(c.email !== '' ? { email: c.email } : {}),
+          }]
+        : []
+      insertClient.run(
+        id, c.code, c.name, c.gstin !== '' ? c.gstin : null,
+        c.state_code !== '' ? c.state_code : ourState, c.address,
+        JSON.stringify(contacts), c.status !== '' ? c.status : 'active', now,
+      )
+      writeAudit(db, userId, 'import', 'client', id, undefined, { code: c.code, name: c.name, source: 'apex' })
+    }
+
+    // Invoices — issued, history-only documents (empty payload lines) that keep
+    // their legacy APEX numbers; the series is seeded past them below.
+    const stgInvoices = db.prepare(`SELECT * FROM stg_invoice ORDER BY row_no`).all() as StgInvoiceRow[]
+    const insertDoc = db.prepare(
+      `INSERT INTO document (id, doc_type, doc_no, fy, client_id, doc_date, status, ref_doc_id,
+         taxable_paise, cgst_paise, sgst_paise, igst_paise, round_off_paise, payable_paise,
+         payload, source, created_by, created_at)
+       VALUES (?, 'INVOICE', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, 'apex', ?, ?)`,
+    )
+    for (const inv of stgInvoices) {
+      const client = db.prepare(`SELECT id, state_code FROM client WHERE code=?`)
+        .get(inv.client_code) as { id: string; state_code: string } | undefined
+      if (client === undefined) {
+        throw new Error(`Invoice ${inv.doc_no}: client code ${inv.client_code} not found — stage clients.csv first`)
+      }
+      const intra = client.state_code === ourState
+      const cgst = intra ? Math.floor(inv.tax_paise / 2) : 0
+      const sgst = intra ? inv.tax_paise - cgst : 0
+      const igst = intra ? 0 : inv.tax_paise
+      const roundOff = inv.total_paise - inv.taxable_paise - inv.tax_paise
+      const totals = {
+        grossPaise: inv.taxable_paise, discountPaise: 0, taxablePaise: inv.taxable_paise,
+        cgstPaise: cgst, sgstPaise: sgst, igstPaise: igst, cessPaise: 0,
+        roundOffPaise: roundOff, payablePaise: inv.total_paise, savingsVsMrpPaise: 0,
+      }
+      const id = uuidv7()
+      const status = inv.paid === 1 ? 'paid' : 'sent'
+      insertDoc.run(
+        id, inv.doc_no, fyOf(inv.doc_date), client.id, inv.doc_date, status,
+        inv.taxable_paise, cgst, sgst, igst, roundOff, inv.total_paise,
+        JSON.stringify({ lines: [], totals }), userId, now,
+      )
+      writeAudit(db, userId, 'import', 'document', id, undefined,
+        { docNo: inv.doc_no, status, payablePaise: inv.total_paise, source: 'apex' })
+    }
+
+    // Series seed (spec §4): continue INVOICE numbering after the last APEX
+    // number of the current FY. Legacy prefixes differ, so only the numeric
+    // tail matters.
+    const currentFy = fyOf(new Date().toISOString().slice(0, 10))
+    let seeded: CommitResult['seeded'] = null
+    const tails = stgInvoices
+      .filter((inv) => fyOf(inv.doc_date) === currentFy)
+      .map((inv) => /(\d+)$/.exec(inv.doc_no)?.[1])
+      .filter((t): t is string => t !== undefined)
+      .map((t) => Number(t))
+    if (tails.length > 0) {
+      const lastSeq = Math.max(...tails)
+      seedSeries(db, 'INVOICE', currentFy, lastSeq)
+      seeded = { fy: currentFy, lastSeq }
+      writeAudit(db, userId, 'import', 'doc_series', `INVOICE/${currentFy}`, undefined, { lastSeq })
+    }
+
+    // Commit consumes the staging area.
+    db.prepare(`DELETE FROM stg_client`).run()
+    db.prepare(`DELETE FROM stg_invoice`).run()
+
+    return { clients: stgClients.length, invoices: stgInvoices.length, seeded }
+  })()
+}
+
+// ---------- CLI: npm run import -- --dir <folder> [--commit] ----------
+
+if (process.argv[1] !== undefined && /import-apex\.(ts|cjs|js)$/.test(process.argv[1])) {
+  const args = process.argv.slice(2)
+  const dirIdx = args.indexOf('--dir')
+  const dir = dirIdx >= 0 ? args[dirIdx + 1] : undefined
+  if (dir === undefined) {
+    console.error('Usage: npm run import -- --dir <folder> [--commit]')
+    process.exit(1)
+  }
+  const db = openDb(process.env['HQ_DATA_DIR'])
+  seedIfEmpty(db)
+  const clients = stageCsv(db, 'clients', fs.readFileSync(path.join(dir, 'clients.csv'), 'utf8'))
+  const invoices = stageCsv(db, 'invoices', fs.readFileSync(path.join(dir, 'invoices.csv'), 'utf8'))
+  console.log(`Staged ${clients.staged} clients, ${invoices.staged} invoices from ${dir}`)
+  console.log(JSON.stringify(verificationReport(db), null, 2))
+  if (args.includes('--commit')) {
+    const out = commitImport(db, 'system')
+    console.log(`Committed: ${out.clients} clients, ${out.invoices} invoices; series seeded:`,
+      out.seeded ?? 'no current-FY invoices')
+  } else {
+    console.log('Dry run — verify the report above, then re-run with --commit to apply.')
+  }
+}

apps/hq/src/seed.ts

@@ -0,0 +1,46 @@
+import { randomBytes } from 'node:crypto'
+import { createStaff } from './auth'
+import { writeAudit } from './audit'
+import type { DB } from './db'
+
+/** First-boot seed: owner login, company settings placeholders, GST18 tax class. */
+
+const OWNER_EMAIL = 'admin@tecnostac.com'
+
+const SETTING_DEFAULTS: Record<string, string> = {
+  'company.name': 'Tecnostac',
+  'company.address': '',
+  'company.gstin': '',
+  'company.state_code': '32', // Kerala — founder to confirm
+  'company.phone': '',
+  'company.email': OWNER_EMAIL,
+  'company.bank': '',
+}
+
+export function seedIfEmpty(db: DB): void {
+  const staff = db.prepare(`SELECT COUNT(*) AS n FROM staff_user`).get() as { n: number }
+  if (staff.n === 0) {
+    // 9 random bytes → 12 base64url chars; printed exactly once, never stored in clear.
+    const password = randomBytes(9).toString('base64url')
+    createStaff(db, { email: OWNER_EMAIL, displayName: 'Owner', role: 'owner', password })
+    console.log(`SiMS HQ first boot — owner ${OWNER_EMAIL} password: ${password} (change after first login)`)
+  }
+  const insert = db.prepare(
+    // Portability quirk: ON CONFLICT upsert is SQLite/Postgres dialect (standard SQL has MERGE).
+    `INSERT INTO setting (key, value) VALUES (?, ?) ON CONFLICT (key) DO NOTHING`,
+  )
+  let seededSettings = 0
+  for (const [key, value] of Object.entries(SETTING_DEFAULTS)) {
+    seededSettings += insert.run(key, value).changes
+  }
+  if (seededSettings > 0) {
+    writeAudit(db, 'system', 'seed', 'setting', 'company.*', undefined, SETTING_DEFAULTS)
+  }
+  const gst = db.prepare(`SELECT COUNT(*) AS n FROM tax_class WHERE class_code='GST18'`).get() as { n: number }
+  if (gst.n === 0) {
+    db.prepare(
+      `INSERT INTO tax_class (class_code, rate_pct_bp, effective_from) VALUES ('GST18', 1800, '2017-07-01')`,
+    ).run()
+    writeAudit(db, 'system', 'seed', 'tax_class', 'GST18', undefined, { ratePctBp: 1800, effectiveFrom: '2017-07-01' })
+  }
+}

apps/hq/src/server.ts

@@ -2,10 +2,12 @@ import express from 'express'
 import type http from 'node:http'
 import { apiRouter } from './api'
 import { openDb } from './db'
+import { seedIfEmpty } from './seed'
 
 export function startServer(port: number): http.Server {
   const app = express()
   const db = openDb(process.env['HQ_DATA_DIR'])
+  seedIfEmpty(db) // first boot prints the owner password once
   app.locals['db'] = db
   app.use(express.json({ limit: '2mb' }))
   app.use('/api', apiRouter(db))

apps/hq/test/fixtures/clients.csv

@@ -0,0 +1,3 @@
+code,name,gstin,state_code,address,phone,email,status
+AC001,Acme Traders,,32,Kochi,9744000001,acme@x.in,active
+AC002,"Malabar Stores, Kochi",,32,MG Road,9744000002,hello@malabar.in,active

apps/hq/test/fixtures/invoices.csv

@@ -0,0 +1,4 @@
+client_code,doc_no,doc_date,taxable,tax,total,paid
+AC001,TS/26-27/0411,2026-05-02,10000.00,1800.00,11800.00,1
+AC001,TS/26-27/0412,2026-06-15,5000.00,900.00,5900.00,0
+AC002,TS/26-27/0413,2026-07-01,25000.00,4500.00,29500.00,0

apps/hq/test/import.test.ts

@@ -0,0 +1,33 @@
+// apps/hq/test/import.test.ts
+import { describe, it, expect } from 'vitest'
+import { openDb } from '../src/db'
+import { seedIfEmpty } from '../src/seed'
+import { stageCsv, verificationReport, commitImport } from '../src/import-apex'
+
+const CLIENTS = `code,name,gstin,state_code,address,phone,email,status
+AC001,Acme Traders,,32,Kochi,9744000001,acme@x.in,active
+,No Code Shop,,32,,,,active`
+const INVOICES = `client_code,doc_no,doc_date,taxable,tax,total,paid
+AC001,TS/26-27/0411,2026-05-02,10000.00,1800.00,11800.00,1
+AC001,TS/26-27/0412,2026-06-15,5000.00,900.00,5900.00,0`
+
+describe('apex import', () => {
+  it('stages, reports problems, refuses commit until clean', () => {
+    const db = openDb(':memory:'); seedIfEmpty(db)
+    stageCsv(db, 'clients', CLIENTS)
+    stageCsv(db, 'invoices', INVOICES)
+    const rep = verificationReport(db)
+    expect(rep.clients.staged).toBe(2)
+    expect(rep.clients.problems).toBe(1) // missing code
+    expect(() => commitImport(db, 'u1')).toThrow(/problem/)
+  })
+  it('commits clean data and seeds the invoice series from the last APEX number', () => {
+    const db = openDb(':memory:'); seedIfEmpty(db)
+    stageCsv(db, 'clients', CLIENTS.split('\n').slice(0, 2).join('\n'))
+    stageCsv(db, 'invoices', INVOICES)
+    const out = commitImport(db, 'u1')
+    expect(out).toMatchObject({ clients: 1, invoices: 2, seeded: { fy: '2026-27', lastSeq: 412 } })
+    const paid = db.prepare(`SELECT status FROM document WHERE doc_no='TS/26-27/0411'`).get() as { status: string }
+    expect(paid.status).toBe('paid')
+  })
+})