sims-hq/apps/hq/test/import-apex-full.test.ts

// apps/hq/test/import-apex-full.test.ts — D20 full-APEX importer v2 (stage → problems → one-txn commit)
import { describe, it, expect } from 'vitest'
import { fyOf } from '@sims/domain'
import { openDb, type DB } from '../src/db'
import { seedIfEmpty } from '../src/seed'
import { decrypt } from '../src/crypto'
import { commitApexFull, parseApexDate, stageApexFull } from '../src/import-apex-full'

const KEY = '22'.repeat(32) // 64 hex chars = 32 bytes
const TODAY = new Date().toISOString().slice(0, 10)
const FY = fyOf(TODAY)

// Build a CSV from sparse row objects — headers EXACTLY as the APEX export ships them.
const csv = (headers: string, rows: Record<string, string>[]): string =>
  headers + '\n'
  + rows.map((r) => headers.split(',').map((h) => r[h] ?? '').join(',')).join('\n') + '\n'

const CLIENT_HDR = 'SID_NO,BANK,ADDRESS,PHONE,MAIL,DISTRICT,SMS,RTGS,WHATSAPP,MOBILE_APP,DB_PASSWORD,ATM,ANYDESK,SECRETARY_CONTACT_DETAILS,OS,SECTOR'
const BRANCH_HDR = 'ID,BRANCH,SMS,MID,BANK,BRCODE'
const BILL_HDR = 'DAY_DT,BILL_TYP,BILL_DATE,BILL_NO,BANK_ID,BANK_NAME,ADDRESS,MOBILE,MAIL,REMARK,USR,INVOICE_NO,QTY,PER_QTY,DESCRIPTION,INVOICE_TYPE,INVOICE_AMOUNT,PAYMENT_STATUS,PAYMENT_DATE,PAYMENT_AMOUNT,NOTES,AMT,SRNO,GST_RATE,GST_AMOUNT,TOTAL_AMOUNT,CONVERTED_FROM_PROFORMA_ID,GST_APPLICABLE,INVOICE_ID,SCROLL'
const TICKET_HDR = 'ID,DAY_DT,BANK_NAME,BANK_ID,BRID,TYPE,GROUP,ASSIGN_DATE,MOBILE_NO,ASSIGNED_TO,DESCRIPTION,STATUS,CREATED,CREATED_BY,UPDATED,UPDATED_BY,ONLINE_OFFLINE,MAIL_BLOB,MAIL,TRNS_ID,TRNS_FRM,DAY_ID,ENAME,TRANS,TICKET_DROPED,DAY_CLS,OLD_ID,SCROLL'
const SMS_CLIENT_HDR = 'RNO,CID,BANK,DISTRICT,SMS,USERNAME,PASSWORD,PAYMENT,SMS_PROVIDER,ADDED_FEATURES,CREATED_BY,UPDATED_BY,PHONE,TID,SMS_BALANCE,ERR,REMARK,UPDATION,RESELLER,PAYMENT_DATE,INSTALLATION_DATE'
const SMS_MODULE_HDR = 'RNO,SID_NO,SMS_ID,HEAD,COLS,PKG,ACTIV,REMARK,IMP_BY,ON_DAY_DT'
const RTGS_CLIENT_HDR = 'SID_NO,BANK_NAME,ADDRESS_IP,PHONE_NO,RTGS,IP_CODE,E_COLLECTION_CODE,RID,TID,ABB,INTEGRATED_BANK_PARTNERS,REMARK,PAYMENT,INSTALLATION_DATE,INSTALLED_ON,GENERATE_QR,SBI_INWARD_REPORT,PASSBOOK_PROGRAM,IFSC_2025,NOTIFICATION,MISC_RTGS_OUTWARD,PAYMENT_DATE'
const SMS_PROJECT_HDR = 'CID,BANK,DISTRICT,DT_INFO,RNO,SMS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,NEW_FORM,CREATED_BY,UPDATED_BY,TID'
const RTGS_PROJECT_HDR = 'CID,BANK,DT_INFO,RNO,RTGS,PHONE,ASSIGNED_TO,TICKET_STATUS,DESCRIPTION,CREATED_BY,UPDATED_BY,TID'

/** A small but complete workbook exercising every section. */
function fullFiles(): Record<string, string> {
  return {
    clients: csv(CLIENT_HDR, [
      {
        SID_NO: '101', BANK: 'Acme Bank', ADDRESS: 'Kochi HQ', PHONE: '9876543210',
        MAIL: 'acme@bank.in', DISTRICT: 'Ernakulam', SMS: 'YES', RTGS: 'NO', WHATSAPP: 'Y',
        MOBILE_APP: 'NO', DB_PASSWORD: 's3cret-db', ATM: 'NO', ANYDESK: '123 456 789',
        SECRETARY_CONTACT_DETAILS: 'Secretary John 9999', OS: 'Windows 10', SECTOR: 'Co-op',
      },
      {
        SID_NO: '102', BANK: 'Beta Bank', DISTRICT: 'Kollam', SMS: 'NO', RTGS: 'YES',
        WHATSAPP: 'NO', MOBILE_APP: 'NO', DB_PASSWORD: 'NO DATA', ATM: 'NO',
        ANYDESK: 'NO DATA', SECTOR: 'Service',
      },
    ]),
    branches: csv(BRANCH_HDR, [
      { ID: '1', BRANCH: 'Main Branch', MID: '101', BANK: 'Acme Bank', BRCODE: 'BR001' },
    ]),
    bills: csv(BILL_HDR, [
      { // paid current-FY invoice — rupees→paise, payment+allocation, series seed
        BILL_DATE: `${TODAY} 00:00:00`, BANK_ID: '101', INVOICE_NO: 'SMS/2026/0042',
        INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '1000', GST_AMOUNT: '180', TOTAL_AMOUNT: '1180',
        PAYMENT_STATUS: 'PAID', PAYMENT_DATE: `${TODAY} 00:00:00`, PAYMENT_AMOUNT: '1180',
      },
      { // proforma with '' tax ('→0) and '' total ('→taxable+tax); never gets a payment
        BILL_DATE: '2025-08-18 00:00:00', BANK_ID: '102', INVOICE_NO: 'PI/2025/007',
        INVOICE_TYPE: 'PROFORMA INVOICE', INVOICE_AMOUNT: '500', GST_AMOUNT: '', TOTAL_AMOUNT: '',
      },
    ]),
    tickets: csv(TICKET_HDR, [
      {
        ID: '1', BANK_ID: '101', TYPE: 'MODIFICATION', GROUP: 'SMS',
        ASSIGN_DATE: '2025-02-28 00:00:00', DESCRIPTION: 'Fix template', STATUS: 'CLOSED',
        CREATED: '2025-02-28 09:00:00', UPDATED: '2025-03-02 10:00:00', ONLINE_OFFLINE: 'ONLINE',
      },
      { // blank BANK_ID → exact BANK_NAME match; DD-MON-YYYY open date; WAITING* status
        ID: '2', BANK_NAME: 'Acme Bank', DAY_DT: '01-MAR-2025',
        STATUS: 'WAITING FOR CONFIRMATION', DESCRIPTION: 'Waiting one',
      },
      { // TICKET_DROPED='Y' with a non-terminal status → dropped, closed_on from UPDATED
        ID: '3', BANK_ID: '102', STATUS: '', TICKET_DROPED: 'Y', DESCRIPTION: 'Dropped one',
        CREATED: '2025-04-01 09:00:00', UPDATED: '2025-04-02 00:00:00',
      },
    ]),
    sms_clients: csv(SMS_CLIENT_HDR, [
      { // CID 102's SMS flag said NO — this row wins and creates the assignment
        RNO: '1', CID: '102', BANK: 'Beta Bank', USERNAME: 'betauser', PASSWORD: 'smsPass9',
        SMS_PROVIDER: 'AltSMS', PHONE: '0474222', SMS_BALANCE: '5000', REMARK: 'ok',
        RESELLER: 'RSL', PAYMENT: 'PAID 2025', INSTALLATION_DATE: '10-NOV-2023',
      },
    ]),
    sms_modules: csv(SMS_MODULE_HDR, [
      { RNO: '1', SID_NO: '102', HEAD: 'ATM ALERT', ACTIV: 'Y' },
      { RNO: '2', SID_NO: '102', HEAD: 'BALANCE ALERT', ACTIV: 'Y' },
      { RNO: '3', SID_NO: '102', HEAD: 'OLD ALERT', ACTIV: 'N' },
    ]),
    rtgs_clients: csv(RTGS_CLIENT_HDR, [
      {
        SID_NO: '102', BANK_NAME: 'Beta Bank', ADDRESS_IP: '10.0.0.5', PHONE_NO: '333',
        IP_CODE: 'IPC1', ABB: 'BETA', INTEGRATED_BANK_PARTNERS: 'SBI',
        INSTALLED_ON: '2024-01-05 00:00:00', GENERATE_QR: 'YES', PAYMENT: 'PAID',
      },
    ]),
    sms_projects: csv(SMS_PROJECT_HDR, [
      { CID: '101', DT_INFO: '13-05-2024', TICKET_STATUS: 'COMPLETED', DESCRIPTION: 'Install SMS' },
    ]),
    rtgs_projects: csv(RTGS_PROJECT_HDR, [
      { CID: '102', DT_INFO: 'not-a-date', TICKET_STATUS: 'PENDING', DESCRIPTION: 'RTGS setup' },
    ]),
  }
}

async function freshDb(): Promise<DB> {
  const db = openDb(':memory:')
  await seedIfEmpty(db)
  return db
}

describe('parseApexDate', () => {
  it('handles every format seen in the APEX export', () => {
    expect(parseApexDate('2025-08-18 00:00:00')).toBe('2025-08-18') // ISO datetime
    expect(parseApexDate('10-NOV-2023 10:57:18')).toBe('2023-11-10') // DD-MON-YYYY
    expect(parseApexDate('08/29/2024')).toBe('2024-08-29') // second >12 → MM/DD
    expect(parseApexDate('13-05-2024')).toBe('2024-05-13') // first >12 → DD-MM
    expect(parseApexDate('10-11-2023')).toBe('2023-11-10') // ambiguous → assume DD-MM
    expect(parseApexDate('')).toBeNull()
    expect(parseApexDate('not-a-date')).toBeNull()
    expect(parseApexDate('99-99-2024')).toBeNull()
  })
})

describe('APEX full importer v2', () => {
  it('stage: clean workbook → per-section counts, zero problems, and NO writes', async () => {
    const db = await freshDb()
    const report = await stageApexFull(db, fullFiles(), KEY)
    expect(report.clients).toEqual({ staged: 2, problems: [] })
    expect(report.branches).toEqual({ staged: 1, problems: [] })
    expect(report.bills).toEqual({ staged: 2, problems: [] })
    expect(report.tickets).toEqual({ staged: 3, problems: [] })
    expect(report.sms_clients).toEqual({ staged: 1, problems: [] })
    expect(report.sms_modules).toEqual({ staged: 3, problems: [] })
    expect(report.rtgs_clients).toEqual({ staged: 1, problems: [] })
    expect(report.sms_projects).toEqual({ staged: 1, problems: [] })
    expect(report.rtgs_projects).toEqual({ staged: 1, problems: [] })
    // Pure validation: no clients, no documents, no import audit rows.
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0)
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0)
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM audit_log WHERE action='import'`))!.n).toBe(0)
  })

  it('stage: unknown join keys are per-row problems with row numbers', async () => {
    const db = await freshDb()
    const report = await stageApexFull(db, {
      clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]),
      branches: csv(BRANCH_HDR, [{ ID: '9', BRANCH: 'Ghost', MID: '999' }]),
      tickets: csv(TICKET_HDR, [{ ID: '9', BANK_ID: '999', STATUS: 'OPEN', DAY_DT: '2025-01-01 00:00:00' }]),
      sms_projects: csv(SMS_PROJECT_HDR, [{ CID: '999', DT_INFO: '2025-01-01' }]),
    }, KEY)
    expect(report.branches.problems).toEqual(['row 1: unknown client MID=999'])
    expect(report.tickets.problems).toEqual(['row 1: unknown client BANK_ID=999'])
    expect(report.sms_projects.problems).toEqual(['row 1: unknown client CID=999'])
  })

  it('commit: full happy path lands every section in one transaction', async () => {
    const db = await freshDb()
    const res = await commitApexFull(db, 'u-test', fullFiles(), KEY)
    expect(res.clients).toBe(2)
    expect(res.client_modules).toBe(4) // 101: SMS+WHATSAPP (flags); 102: RTGS (flag) + SMS (data wins)
    expect(res.branches).toBe(1)
    expect(res.bills).toBe(2)
    expect(res.payments).toBe(1)
    expect(res.tickets).toBe(3)
    expect(res.sms_projects).toBe(1)
    expect(res.rtgs_projects).toBe(1)
    expect(res.seeded).toEqual({ fy: FY, lastSeq: 42 })

    // Clients: code = SID_NO, contacts carry bank + secretary, support fields ride along.
    const c101 = (await db.get<{
      id: string; name: string; address: string; district: string; sector: string
      os: string; anydesk: string; contacts: string; status: string; source: string
      state_code: string; db_password_enc: string
    }>(`SELECT * FROM client WHERE code='101'`))!
    expect(c101).toMatchObject({
      name: 'Acme Bank', address: 'Kochi HQ', district: 'Ernakulam', sector: 'Co-op',
      os: 'Windows 10', anydesk: '123 456 789', status: 'active', source: 'apex', state_code: '32',
    })
    expect(JSON.parse(c101.contacts)).toEqual([
      { name: 'Acme Bank', phone: '9876543210', email: 'acme@bank.in' },
      { name: 'Secretary John 9999', role: 'secretary' },
    ])
    // DB password encrypted at rest, decryptable with the key.
    expect(decrypt(c101.db_password_enc, KEY)).toBe('s3cret-db')
    const c102 = (await db.get<{ anydesk: string | null; db_password_enc: string | null }>(
      `SELECT anydesk, db_password_enc FROM client WHERE code='102'`))!
    expect(c102).toEqual({ anydesk: null, db_password_enc: null }) // 'NO DATA' → nothing stored

    // Flags → live yearly assignments on get-or-created modules.
    const modules = await db.all<{ code: string; name: string }>(
      `SELECT code, name FROM module WHERE code IN ('SMS','RTGS','WHATSAPP') ORDER BY code`)
    expect(modules).toEqual([
      { code: 'RTGS', name: 'RTGS' }, { code: 'SMS', name: 'Bulk SMS' },
      { code: 'WHATSAPP', name: 'WhatsApp' },
    ])
    const codes101 = await db.all<{ code: string; status: string; kind: string }>(
      `SELECT m.code, cm.status, cm.kind FROM client_module cm
       JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id
       WHERE c.code='101' ORDER BY m.code`)
    expect(codes101).toEqual([
      { code: 'SMS', status: 'live', kind: 'yearly' },
      { code: 'WHATSAPP', status: 'live', kind: 'yearly' },
    ])

    // Branch joined by MID.
    const branch = await db.get(`SELECT b.name, b.code, b.active FROM client_branch b
       JOIN client c ON c.id=b.client_id WHERE c.code='101'`)
    expect(branch).toEqual({ name: 'Main Branch', code: 'BR001', active: 1 })

    // Bills: rupees→paise, intra-state split, payment + allocation, series seed.
    const inv = (await db.get<{
      id: string; doc_type: string; fy: string; status: string; taxable_paise: number
      cgst_paise: number; sgst_paise: number; igst_paise: number; round_off_paise: number
      payable_paise: number; source: string; payload: string
    }>(`SELECT * FROM document WHERE doc_no='SMS/2026/0042'`))!
    expect(inv).toMatchObject({
      doc_type: 'INVOICE', fy: FY, status: 'paid', taxable_paise: 100000,
      cgst_paise: 9000, sgst_paise: 9000, igst_paise: 0, round_off_paise: 0,
      payable_paise: 118000, source: 'apex',
    })
    expect(JSON.parse(inv.payload)).toMatchObject({ lines: [], totals: { payablePaise: 118000 } })
    const pi = await db.get(`SELECT doc_type, status, taxable_paise, payable_paise, cgst_paise
       FROM document WHERE doc_no='PI/2025/007'`)
    expect(pi).toEqual({
      doc_type: 'PROFORMA', status: 'sent', taxable_paise: 50000, payable_paise: 50000, cgst_paise: 0,
    })
    const pay = (await db.get<{ id: string; mode: string; reference: string; amount_paise: number; received_on: string }>(
      `SELECT * FROM payment`))!
    expect(pay).toMatchObject({
      mode: 'bank', reference: 'APEX import', amount_paise: 118000, received_on: TODAY,
    })
    const alloc = await db.get(`SELECT payment_id, document_id, amount_paise FROM payment_allocation`)
    expect(alloc).toEqual({ payment_id: pay.id, document_id: inv.id, amount_paise: 118000 })
    const series = await db.get(`SELECT next_seq FROM doc_series WHERE doc_type='INVOICE' AND fy=?`, FY)
    expect(series).toEqual({ next_seq: 43 }) // continues after the legacy 0042 tail

    // Tickets: status mapping + join fallbacks.
    const t1 = await db.get(`SELECT status, opened_on, closed_on, online_offline, module_code, source
       FROM ticket WHERE description='Fix template'`)
    expect(t1).toEqual({
      status: 'closed', opened_on: '2025-02-28', closed_on: '2025-03-02',
      online_offline: 'online', module_code: 'SMS', source: 'apex',
    })
    const t2 = (await db.get<{ status: string; opened_on: string; client_id: string }>(
      `SELECT status, opened_on, client_id FROM ticket WHERE description='Waiting one'`))!
    expect(t2.status).toBe('waiting')
    expect(t2.opened_on).toBe('2025-03-01') // DD-MON-YYYY DAY_DT
    expect(t2.client_id).toBe(c101.id) // blank BANK_ID resolved by exact BANK_NAME
    const t3 = await db.get(`SELECT status, opened_on, closed_on FROM ticket WHERE description='Dropped one'`)
    expect(t3).toEqual({ status: 'dropped', opened_on: '2025-04-01', closed_on: '2025-04-02' })

    // SMS service data landed on 102's (data-created) SMS module.
    const sms102 = (await db.get<{ provider: string; username: string; password_enc: string; details: string; remark: string }>(
      `SELECT cm.provider, cm.username, cm.password_enc, cm.details, cm.remark FROM client_module cm
       JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id
       WHERE c.code='102' AND m.code='SMS'`))!
    expect(sms102.provider).toBe('AltSMS')
    expect(sms102.username).toBe('betauser')
    expect(decrypt(sms102.password_enc, KEY)).toBe('smsPass9')
    expect(sms102.remark).toBe('ok')
    const smsDetails = JSON.parse(sms102.details) as { label: string; value: string }[]
    expect(smsDetails).toContainEqual({ label: 'SMS balance', value: '5000' })
    expect(smsDetails).toContainEqual({ label: 'Installed', value: '10-NOV-2023' })
    expect(smsDetails).toContainEqual({ label: 'Active alerts', value: 'ATM ALERT, BALANCE ALERT' })

    // RTGS service data on 102's RTGS module.
    const rtgs102 = (await db.get<{ details: string }>(
      `SELECT cm.details FROM client_module cm
       JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id
       WHERE c.code='102' AND m.code='RTGS'`))!
    const rtgsDetails = JSON.parse(rtgs102.details) as { label: string; value: string }[]
    expect(rtgsDetails).toContainEqual({ label: 'IP', value: '10.0.0.5' })
    expect(rtgsDetails).toContainEqual({ label: 'Installed', value: '2024-01-05 00:00:00' })
    expect(rtgsDetails).toContainEqual({ label: 'QR', value: 'YES' })

    // Projects → interaction rows; unparseable DT_INFO falls back to today + a note.
    const sp = (await db.get<{ on_date: string; notes: string; type_code: string }>(
      `SELECT on_date, notes, type_code FROM interaction WHERE notes LIKE '[SMS project]%'`))!
    expect(sp.type_code).toBe('project')
    expect(sp.on_date).toBe('2024-05-13')
    expect(sp.notes).toBe('[SMS project] COMPLETED — Install SMS')
    const rp = (await db.get<{ on_date: string; notes: string }>(
      `SELECT on_date, notes FROM interaction WHERE notes LIKE '[RTGS project]%'`))!
    expect(rp.on_date).toBe(TODAY)
    expect(rp.notes).toContain('DT_INFO unparsed: not-a-date')

    // One summary audit per section (plus the series seed) — and never a plaintext secret.
    const auditEntities = (await db.all<{ entity: string }>(
      `SELECT entity FROM audit_log WHERE action='import'`)).map((r) => r.entity).sort()
    expect(auditEntities).toEqual(['bills', 'branches', 'clients', 'doc_series', 'rtgs_clients',
      'rtgs_projects', 'sms_clients', 'sms_modules', 'sms_projects', 'tickets'])
    const auditDump = JSON.stringify(await db.all(`SELECT * FROM audit_log`))
    expect(auditDump).not.toContain('s3cret-db')
    expect(auditDump).not.toContain('smsPass9')
  })

  it('refuses plaintext passwords loudly when no encryption key is configured', async () => {
    const db = await freshDb()
    const files = fullFiles()
    const report = await stageApexFull(db, files, '')
    expect(report.clients.problems).toEqual([
      'row 1: DB_PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted',
    ])
    expect(report.sms_clients.problems).toEqual([
      'row 1: PASSWORD present but HQ_SECRET_KEY is not configured — refusing to import it unencrypted',
    ])
    await expect(commitApexFull(db, 'u-test', files, '')).rejects.toThrow(/HQ_SECRET_KEY/)
    // Refused before the transaction — nothing landed.
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0)
  })

  it('duplicate doc_no is a problem that blocks the whole commit', async () => {
    const db = await freshDb()
    const files = {
      clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]),
      bills: csv(BILL_HDR, [
        { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '100' },
        { BILL_DATE: '2025-05-02 00:00:00', BANK_ID: '101', INVOICE_NO: 'INV-1', INVOICE_TYPE: 'INVOICE', INVOICE_AMOUNT: '200' },
      ]),
    }
    const report = await stageApexFull(db, files, KEY)
    expect(report.bills.problems).toEqual(['row 2: duplicate doc_no: INV-1'])
    await expect(commitApexFull(db, 'u-test', files, KEY)).rejects.toThrow(/duplicate doc_no/)
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM document`))!.n).toBe(0)
    expect((await db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM client`))!.n).toBe(0)
  })

  it('rejects an unknown INVOICE_TYPE instead of guessing', async () => {
    const db = await freshDb()
    const report = await stageApexFull(db, {
      clients: csv(CLIENT_HDR, [{ SID_NO: '101', BANK: 'Acme Bank' }]),
      bills: csv(BILL_HDR, [
        { BILL_DATE: '2025-05-01 00:00:00', BANK_ID: '101', INVOICE_NO: 'X-1', INVOICE_TYPE: 'RECEIPT', INVOICE_AMOUNT: '100' },
      ]),
    }, KEY)
    expect(report.bills.problems).toEqual(['row 1: unknown INVOICE_TYPE: RECEIPT'])
  })
})