|
|
|
|
@ -0,0 +1,302 @@
|
|
|
|
|
import fs from 'node:fs'
|
|
|
|
|
import path from 'node:path'
|
|
|
|
|
import { fromRupees, fyOf, uuidv7, validateGstin } from '@sims/domain'
|
|
|
|
|
import { writeAudit } from './audit'
|
|
|
|
|
import { openDb, type DB } from './db'
|
|
|
|
|
import { seedIfEmpty } from './seed'
|
|
|
|
|
import { seedSeries } from './series'
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* APEX cutover importer (spec §4): stage CSVs → verification report →
|
|
|
|
|
* commit + INVOICE series seed. Staged rows carry their own problem list;
|
|
|
|
|
* commit refuses to run until every staged row is clean.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
// ---------- tiny CSV parser (header row, comma, double-quote escaping) ----------
|
|
|
|
|
|
|
|
|
|
function parseCsv(text: string): Record<string, string>[] {
|
|
|
|
|
const rows: string[][] = []
|
|
|
|
|
let field = ''
|
|
|
|
|
let row: string[] = []
|
|
|
|
|
let inQuotes = false
|
|
|
|
|
const endField = (): void => { row.push(field); field = '' }
|
|
|
|
|
const endRow = (): void => { endField(); rows.push(row); row = [] }
|
|
|
|
|
for (let i = 0; i < text.length; i++) {
|
|
|
|
|
const ch = text[i]!
|
|
|
|
|
if (inQuotes) {
|
|
|
|
|
if (ch === '"') {
|
|
|
|
|
if (text[i + 1] === '"') { field += '"'; i++ } else inQuotes = false
|
|
|
|
|
} else field += ch
|
|
|
|
|
} else if (ch === '"') inQuotes = true
|
|
|
|
|
else if (ch === ',') endField()
|
|
|
|
|
else if (ch === '\n') endRow()
|
|
|
|
|
else if (ch !== '\r') field += ch
|
|
|
|
|
}
|
|
|
|
|
if (field !== '' || row.length > 0) endRow()
|
|
|
|
|
const header = (rows.shift() ?? []).map((h) => h.trim())
|
|
|
|
|
return rows
|
|
|
|
|
.filter((r) => r.some((c) => c.trim() !== ''))
|
|
|
|
|
.map((r) => Object.fromEntries(header.map((h, i) => [h, (r[i] ?? '').trim()])))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------- staging ----------
|
|
|
|
|
|
|
|
|
|
const CLIENT_STATUSES = new Set(['lead', 'active', 'dormant', 'lost'])
|
|
|
|
|
const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
|
|
|
|
|
|
|
|
|
|
function stageClients(db: DB, records: Record<string, string>[]): number {
|
|
|
|
|
db.prepare(`DELETE FROM stg_client`).run()
|
|
|
|
|
const existing = new Set(
|
|
|
|
|
(db.prepare(`SELECT code FROM client`).all() as { code: string }[]).map((r) => r.code),
|
|
|
|
|
)
|
|
|
|
|
const seen = new Set<string>()
|
|
|
|
|
const insert = db.prepare(
|
|
|
|
|
`INSERT INTO stg_client (row_no, code, name, gstin, state_code, address, phone, email, status, problems)
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
|
|
|
)
|
|
|
|
|
records.forEach((r, i) => {
|
|
|
|
|
const problems: string[] = []
|
|
|
|
|
const code = r['code'] ?? ''
|
|
|
|
|
if (code === '') problems.push('missing code')
|
|
|
|
|
else if (seen.has(code) || existing.has(code)) problems.push(`duplicate code: ${code}`)
|
|
|
|
|
if (code !== '') seen.add(code)
|
|
|
|
|
if ((r['name'] ?? '') === '') problems.push('missing name')
|
|
|
|
|
const gstin = r['gstin'] ?? ''
|
|
|
|
|
if (gstin !== '' && !validateGstin(gstin).ok) problems.push(`bad GSTIN: ${gstin}`)
|
|
|
|
|
const status = r['status'] ?? ''
|
|
|
|
|
if (status !== '' && !CLIENT_STATUSES.has(status)) problems.push(`bad status: ${status}`)
|
|
|
|
|
insert.run(
|
|
|
|
|
i + 1, code, r['name'] ?? '', gstin, r['state_code'] ?? '',
|
|
|
|
|
r['address'] ?? '', r['phone'] ?? '', r['email'] ?? '', status,
|
|
|
|
|
JSON.stringify(problems),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
return records.length
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function paiseOrNull(raw: string): number | null {
|
|
|
|
|
if (raw === '') return null
|
|
|
|
|
const n = Number(raw)
|
|
|
|
|
return Number.isFinite(n) ? fromRupees(n) : null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function stageInvoices(db: DB, records: Record<string, string>[]): number {
|
|
|
|
|
db.prepare(`DELETE FROM stg_invoice`).run()
|
|
|
|
|
const existing = new Set(
|
|
|
|
|
(db.prepare(`SELECT doc_no FROM document WHERE doc_no IS NOT NULL`).all() as { doc_no: string }[])
|
|
|
|
|
.map((r) => r.doc_no),
|
|
|
|
|
)
|
|
|
|
|
const seen = new Set<string>()
|
|
|
|
|
const insert = db.prepare(
|
|
|
|
|
`INSERT INTO stg_invoice (row_no, client_code, doc_no, doc_date, taxable_paise, tax_paise, total_paise, paid, problems)
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
|
|
|
)
|
|
|
|
|
records.forEach((r, i) => {
|
|
|
|
|
const problems: string[] = []
|
|
|
|
|
if ((r['client_code'] ?? '') === '') problems.push('missing client_code')
|
|
|
|
|
const docNo = r['doc_no'] ?? ''
|
|
|
|
|
if (docNo === '') problems.push('missing doc_no')
|
|
|
|
|
else if (seen.has(docNo) || existing.has(docNo)) problems.push(`duplicate doc_no: ${docNo}`)
|
|
|
|
|
if (docNo !== '') seen.add(docNo)
|
|
|
|
|
const docDate = r['doc_date'] ?? ''
|
|
|
|
|
if (!ISO_DATE.test(docDate) || Number.isNaN(Date.parse(docDate))) problems.push(`bad date: ${docDate}`)
|
|
|
|
|
const taxable = paiseOrNull(r['taxable'] ?? '')
|
|
|
|
|
const tax = paiseOrNull(r['tax'] ?? '')
|
|
|
|
|
const total = paiseOrNull(r['total'] ?? '')
|
|
|
|
|
if (taxable === null) problems.push(`bad taxable: ${r['taxable'] ?? ''}`)
|
|
|
|
|
if (tax === null) problems.push(`bad tax: ${r['tax'] ?? ''}`)
|
|
|
|
|
if (total === null) problems.push(`bad total: ${r['total'] ?? ''}`)
|
|
|
|
|
insert.run(
|
|
|
|
|
i + 1, r['client_code'] ?? '', docNo, docDate,
|
|
|
|
|
taxable, tax, total, r['paid'] === '1' ? 1 : 0,
|
|
|
|
|
JSON.stringify(problems),
|
|
|
|
|
)
|
|
|
|
|
})
|
|
|
|
|
return records.length
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function stageCsv(db: DB, kind: 'clients' | 'invoices', csvText: string): { staged: number } {
|
|
|
|
|
const records = parseCsv(csvText)
|
|
|
|
|
const staged = kind === 'clients' ? stageClients(db, records) : stageInvoices(db, records)
|
|
|
|
|
writeAudit(db, 'system', 'stage', `stg_${kind}`, kind, undefined, { staged })
|
|
|
|
|
return { staged }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------- verification report ----------
|
|
|
|
|
|
|
|
|
|
export interface VerificationReport {
|
|
|
|
|
clients: { staged: number; problems: number }
|
|
|
|
|
invoices: { staged: number; problems: number; totalPaise: number }
|
|
|
|
|
samples: { firstClients: string[]; lastInvoices: string[] }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function verificationReport(db: DB): VerificationReport {
|
|
|
|
|
const c = db.prepare(
|
|
|
|
|
`SELECT COUNT(*) AS staged,
|
|
|
|
|
COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems
|
|
|
|
|
FROM stg_client`,
|
|
|
|
|
).get() as { staged: number; problems: number }
|
|
|
|
|
const i = db.prepare(
|
|
|
|
|
`SELECT COUNT(*) AS staged,
|
|
|
|
|
COALESCE(SUM(CASE WHEN problems <> '[]' THEN 1 ELSE 0 END), 0) AS problems,
|
|
|
|
|
COALESCE(SUM(total_paise), 0) AS totalPaise
|
|
|
|
|
FROM stg_invoice`,
|
|
|
|
|
).get() as { staged: number; problems: number; totalPaise: number }
|
|
|
|
|
const firstClients = (db.prepare(
|
|
|
|
|
`SELECT code, name FROM stg_client ORDER BY row_no LIMIT 5`,
|
|
|
|
|
).all() as { code: string; name: string }[]).map((r) => `${r.code} ${r.name}`.trim())
|
|
|
|
|
const lastInvoices = (db.prepare(
|
|
|
|
|
`SELECT doc_no FROM stg_invoice ORDER BY row_no DESC LIMIT 5`,
|
|
|
|
|
).all() as { doc_no: string }[]).map((r) => r.doc_no)
|
|
|
|
|
return {
|
|
|
|
|
clients: { staged: c.staged, problems: c.problems },
|
|
|
|
|
invoices: { staged: i.staged, problems: i.problems, totalPaise: i.totalPaise },
|
|
|
|
|
samples: { firstClients, lastInvoices },
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------- commit ----------
|
|
|
|
|
|
|
|
|
|
interface StgClientRow {
|
|
|
|
|
row_no: number; code: string; name: string; gstin: string; state_code: string
|
|
|
|
|
address: string; phone: string; email: string; status: string; problems: string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface StgInvoiceRow {
|
|
|
|
|
row_no: number; client_code: string; doc_no: string; doc_date: string
|
|
|
|
|
taxable_paise: number; tax_paise: number; total_paise: number; paid: number; problems: string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface CommitResult {
|
|
|
|
|
clients: number
|
|
|
|
|
invoices: number
|
|
|
|
|
seeded: { fy: string; lastSeq: number } | null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function companyStateCode(db: DB): string {
|
|
|
|
|
const row = db.prepare(`SELECT value FROM setting WHERE key='company.state_code'`)
|
|
|
|
|
.get() as { value: string } | undefined
|
|
|
|
|
// Silent defaults on place-of-supply are how wrong GST reaches the portal — fail loudly.
|
|
|
|
|
if (row === undefined) throw new Error(`Setting 'company.state_code' is not configured`)
|
|
|
|
|
return row.value
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function commitImport(db: DB, userId: string): CommitResult {
|
|
|
|
|
return db.transaction((): CommitResult => {
|
|
|
|
|
const bad = (db.prepare(
|
|
|
|
|
`SELECT (SELECT COUNT(*) FROM stg_client WHERE problems <> '[]')
|
|
|
|
|
+ (SELECT COUNT(*) FROM stg_invoice WHERE problems <> '[]') AS n`,
|
|
|
|
|
).get() as { n: number }).n
|
|
|
|
|
if (bad > 0) throw new Error(`${bad} staged row(s) have problems — fix the CSV and re-stage before commit`)
|
|
|
|
|
|
|
|
|
|
const ourState = companyStateCode(db)
|
|
|
|
|
const now = new Date().toISOString()
|
|
|
|
|
|
|
|
|
|
// Clients — source 'apex' marks the cutover rows.
|
|
|
|
|
const stgClients = db.prepare(`SELECT * FROM stg_client ORDER BY row_no`).all() as StgClientRow[]
|
|
|
|
|
const insertClient = db.prepare(
|
|
|
|
|
`INSERT INTO client (id, code, name, gstin, state_code, address, contacts, status, source, created_at)
|
|
|
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'apex', ?)`,
|
|
|
|
|
)
|
|
|
|
|
for (const c of stgClients) {
|
|
|
|
|
const id = uuidv7()
|
|
|
|
|
const contacts = c.phone !== '' || c.email !== ''
|
|
|
|
|
? [{
|
|
|
|
|
name: c.name,
|
|
|
|
|
...(c.phone !== '' ? { phone: c.phone } : {}),
|
|
|
|
|
...(c.email !== '' ? { email: c.email } : {}),
|
|
|
|
|
}]
|
|
|
|
|
: []
|
|
|
|
|
insertClient.run(
|
|
|
|
|
id, c.code, c.name, c.gstin !== '' ? c.gstin : null,
|
|
|
|
|
c.state_code !== '' ? c.state_code : ourState, c.address,
|
|
|
|
|
JSON.stringify(contacts), c.status !== '' ? c.status : 'active', now,
|
|
|
|
|
)
|
|
|
|
|
writeAudit(db, userId, 'import', 'client', id, undefined, { code: c.code, name: c.name, source: 'apex' })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Invoices — issued, history-only documents (empty payload lines) that keep
|
|
|
|
|
// their legacy APEX numbers; the series is seeded past them below.
|
|
|
|
|
const stgInvoices = db.prepare(`SELECT * FROM stg_invoice ORDER BY row_no`).all() as StgInvoiceRow[]
|
|
|
|
|
const insertDoc = db.prepare(
|
|
|
|
|
`INSERT INTO document (id, doc_type, doc_no, fy, client_id, doc_date, status, ref_doc_id,
|
|
|
|
|
taxable_paise, cgst_paise, sgst_paise, igst_paise, round_off_paise, payable_paise,
|
|
|
|
|
payload, source, created_by, created_at)
|
|
|
|
|
VALUES (?, 'INVOICE', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, 'apex', ?, ?)`,
|
|
|
|
|
)
|
|
|
|
|
for (const inv of stgInvoices) {
|
|
|
|
|
const client = db.prepare(`SELECT id, state_code FROM client WHERE code=?`)
|
|
|
|
|
.get(inv.client_code) as { id: string; state_code: string } | undefined
|
|
|
|
|
if (client === undefined) {
|
|
|
|
|
throw new Error(`Invoice ${inv.doc_no}: client code ${inv.client_code} not found — stage clients.csv first`)
|
|
|
|
|
}
|
|
|
|
|
const intra = client.state_code === ourState
|
|
|
|
|
const cgst = intra ? Math.floor(inv.tax_paise / 2) : 0
|
|
|
|
|
const sgst = intra ? inv.tax_paise - cgst : 0
|
|
|
|
|
const igst = intra ? 0 : inv.tax_paise
|
|
|
|
|
const roundOff = inv.total_paise - inv.taxable_paise - inv.tax_paise
|
|
|
|
|
const totals = {
|
|
|
|
|
grossPaise: inv.taxable_paise, discountPaise: 0, taxablePaise: inv.taxable_paise,
|
|
|
|
|
cgstPaise: cgst, sgstPaise: sgst, igstPaise: igst, cessPaise: 0,
|
|
|
|
|
roundOffPaise: roundOff, payablePaise: inv.total_paise, savingsVsMrpPaise: 0,
|
|
|
|
|
}
|
|
|
|
|
const id = uuidv7()
|
|
|
|
|
const status = inv.paid === 1 ? 'paid' : 'sent'
|
|
|
|
|
insertDoc.run(
|
|
|
|
|
id, inv.doc_no, fyOf(inv.doc_date), client.id, inv.doc_date, status,
|
|
|
|
|
inv.taxable_paise, cgst, sgst, igst, roundOff, inv.total_paise,
|
|
|
|
|
JSON.stringify({ lines: [], totals }), userId, now,
|
|
|
|
|
)
|
|
|
|
|
writeAudit(db, userId, 'import', 'document', id, undefined,
|
|
|
|
|
{ docNo: inv.doc_no, status, payablePaise: inv.total_paise, source: 'apex' })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Series seed (spec §4): continue INVOICE numbering after the last APEX
|
|
|
|
|
// number of the current FY. Legacy prefixes differ, so only the numeric
|
|
|
|
|
// tail matters.
|
|
|
|
|
const currentFy = fyOf(new Date().toISOString().slice(0, 10))
|
|
|
|
|
let seeded: CommitResult['seeded'] = null
|
|
|
|
|
const tails = stgInvoices
|
|
|
|
|
.filter((inv) => fyOf(inv.doc_date) === currentFy)
|
|
|
|
|
.map((inv) => /(\d+)$/.exec(inv.doc_no)?.[1])
|
|
|
|
|
.filter((t): t is string => t !== undefined)
|
|
|
|
|
.map((t) => Number(t))
|
|
|
|
|
if (tails.length > 0) {
|
|
|
|
|
const lastSeq = Math.max(...tails)
|
|
|
|
|
seedSeries(db, 'INVOICE', currentFy, lastSeq)
|
|
|
|
|
seeded = { fy: currentFy, lastSeq }
|
|
|
|
|
writeAudit(db, userId, 'import', 'doc_series', `INVOICE/${currentFy}`, undefined, { lastSeq })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Commit consumes the staging area.
|
|
|
|
|
db.prepare(`DELETE FROM stg_client`).run()
|
|
|
|
|
db.prepare(`DELETE FROM stg_invoice`).run()
|
|
|
|
|
|
|
|
|
|
return { clients: stgClients.length, invoices: stgInvoices.length, seeded }
|
|
|
|
|
})()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ---------- CLI: npm run import -- --dir <folder> [--commit] ----------
|
|
|
|
|
|
|
|
|
|
if (process.argv[1] !== undefined && /import-apex\.(ts|cjs|js)$/.test(process.argv[1])) {
|
|
|
|
|
const args = process.argv.slice(2)
|
|
|
|
|
const dirIdx = args.indexOf('--dir')
|
|
|
|
|
const dir = dirIdx >= 0 ? args[dirIdx + 1] : undefined
|
|
|
|
|
if (dir === undefined) {
|
|
|
|
|
console.error('Usage: npm run import -- --dir <folder> [--commit]')
|
|
|
|
|
process.exit(1)
|
|
|
|
|
}
|
|
|
|
|
const db = openDb(process.env['HQ_DATA_DIR'])
|
|
|
|
|
seedIfEmpty(db)
|
|
|
|
|
const clients = stageCsv(db, 'clients', fs.readFileSync(path.join(dir, 'clients.csv'), 'utf8'))
|
|
|
|
|
const invoices = stageCsv(db, 'invoices', fs.readFileSync(path.join(dir, 'invoices.csv'), 'utf8'))
|
|
|
|
|
console.log(`Staged ${clients.staged} clients, ${invoices.staged} invoices from ${dir}`)
|
|
|
|
|
console.log(JSON.stringify(verificationReport(db), null, 2))
|
|
|
|
|
if (args.includes('--commit')) {
|
|
|
|
|
const out = commitImport(db, 'system')
|
|
|
|
|
console.log(`Committed: ${out.clients} clients, ${out.invoices} invoices; series seeded:`,
|
|
|
|
|
out.seeded ?? 'no current-FY invoices')
|
|
|
|
|
} else {
|
|
|
|
|
console.log('Dry run — verify the report above, then re-run with --commit to apply.')
|
|
|
|
|
}
|
|
|
|
|
}
|
Thomas Joise
2 weeks ago