From aa541270d684f368ae8fcdc60cfa330521c73d3d Mon Sep 17 00:00:00 2001 From: Thomas Joise Date: Tue, 21 Jul 2026 00:22:53 +0530 Subject: [PATCH] fix(onboarding): APEX-imported projects were invisible on the stalled tile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit client_module.created_at was added and stamped in assignModule, and stalledProjects now correctly excludes no-tick projects whose created_at is NULL — but the APEX importer (the path that created the real ~300 clients' projects) inserted client_module WITHOUT created_at. Every imported project with zero ticks was therefore permanently invisible on the "Onboarding stalled" tile, exactly the population it exists for. Three parts: - Stamp created_at in the APEX importer's client_module insert. - Backfill existing NULL rows from installed_on where known (both db.ts migrate() for SQLite and migrations-pg.ts for Postgres); never fabricate a date when installed_on is also unknown. - stalledProjects now returns { rows, unknownAgeExcluded } instead of a bare array, so projects excluded for unprovable age are counted, not silently dropped. GET /projects/stalled responds with { ok, projects, unknownAgeExcluded }; UI wiring left to the parallel agent. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/hq/src/api.ts | 6 ++- apps/hq/src/db.ts | 9 ++++ apps/hq/src/import-apex-full.ts | 9 ++-- apps/hq/src/migrations-pg.ts | 8 +++ apps/hq/src/repos-milestones.ts | 41 +++++++++++---- apps/hq/test/db.test.ts | 57 +++++++++++++++++++++ apps/hq/test/import-apex-full.test.ts | 7 ++- apps/hq/test/projects-stalled-route.test.ts | 5 +- apps/hq/test/stalled-onboarding.test.ts | 28 +++++++--- 9 files changed, 146 insertions(+), 24 deletions(-) diff --git a/apps/hq/src/api.ts b/apps/hq/src/api.ts index aff394f..3158544 100644 --- a/apps/hq/src/api.ts +++ b/apps/hq/src/api.ts @@ -815,7 +815,11 @@ export function apiRouter( days = await getNumberSetting(db, 'project.stall_days', 30) } const today = new Date().toISOString().slice(0, 10) - res.json({ ok: true, projects: await stalledProjects(db, days, today) }) + // FIX 4 (no silent caps): unknownAgeExcluded surfaces the count of pending projects the + // query could not prove old (no ticks and no client_module.created_at) instead of + // silently dropping them off the tile with no trace. + const { rows, unknownAgeExcluded } = await stalledProjects(db, days, today) + res.json({ ok: true, projects: rows, unknownAgeExcluded }) } catch (err) { res.status(400).json({ ok: false, error: err instanceof Error ? err.message : String(err) }) } diff --git a/apps/hq/src/db.ts b/apps/hq/src/db.ts index 2e1664a..cd7a9a6 100644 --- a/apps/hq/src/db.ts +++ b/apps/hq/src/db.ts @@ -534,6 +534,15 @@ function migrate(db: SqliteRaw): void { if (!cmCols.some((c) => c.name === 'created_at')) { db.exec(`ALTER TABLE client_module ADD COLUMN created_at TEXT`) } + // FIX 4 (regression): the APEX importer inserted client_module rows WITHOUT created_at + // (only assignModule stamped it), so every imported project — the ~300-client real book — + // permanently NULL'd out of the "Onboarding stalled" tile's age measurement. Backfill from + // installed_on where it's known; a row with neither is genuinely unknown-age and stays + // NULL (never fabricated a date). Runs every migrate() pass — idempotent (only touches rows + // still NULL), so it also heals DBs that already had the column from before this fix. + db.exec( + `UPDATE client_module SET created_at = installed_on WHERE created_at IS NULL AND installed_on IS NOT NULL`, + ) const modCols2 = db.prepare(`PRAGMA table_info(module)`).all() as { name: string }[] if (!modCols2.some((c) => c.name === 'field_spec')) { db.exec(`ALTER TABLE module ADD COLUMN field_spec TEXT NOT NULL DEFAULT '[]'`) diff --git a/apps/hq/src/import-apex-full.ts b/apps/hq/src/import-apex-full.ts index 39bdaca..f985a36 100644 --- a/apps/hq/src/import-apex-full.ts +++ b/apps/hq/src/import-apex-full.ts @@ -1138,12 +1138,15 @@ export async function commitApexFull( ) if (existingCm === undefined) { await db.run( + // FIX 4 (regression): stamp created_at like assignModule does — otherwise every + // imported project with zero ticks has a NULL created_at and is permanently + // invisible on the "Onboarding stalled" tile, exactly the population it exists for. `INSERT INTO client_module (id, client_id, module_id, status, kind, edition, active, - installed_on, provider, username, password_enc, details, remark, field_values) - VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?, ?)`, + installed_on, provider, username, password_enc, details, remark, field_values, created_at) + VALUES (?, ?, ?, ?, 'yearly', 'standard', 1, ?, ?, ?, ?, ?, ?, ?, ?)`, a.id, a.clientId, moduleId, a.status, a.installedOn, a.provider, a.username, a.passwordEnc, JSON.stringify(a.details), a.remark, - JSON.stringify(a.fieldValues), + JSON.stringify(a.fieldValues), now, ) } else if (a.serviceTouched) { // Client pre-dated this import with a live assignment: service data wins. diff --git a/apps/hq/src/migrations-pg.ts b/apps/hq/src/migrations-pg.ts index 9397ecf..f87c66c 100644 --- a/apps/hq/src/migrations-pg.ts +++ b/apps/hq/src/migrations-pg.ts @@ -373,4 +373,12 @@ END; id: '018-client-module-created-at', sql: `ALTER TABLE client_module ADD COLUMN IF NOT EXISTS created_at text;`, }, + { + // FIX 4 (regression): the APEX importer inserted client_module rows WITHOUT created_at + // (only assignModule stamped it going forward), so every one of the ~300 imported clients' + // projects permanently NULL'd out of the stalled-onboarding age measurement. Backfill from + // installed_on where known; a row with neither stays NULL (never fabricated). + id: '019-client-module-created-at-backfill', + sql: `UPDATE client_module SET created_at = installed_on WHERE created_at IS NULL AND installed_on IS NOT NULL;`, + }, ] diff --git a/apps/hq/src/repos-milestones.ts b/apps/hq/src/repos-milestones.ts index 9d66095..9e4b04c 100644 --- a/apps/hq/src/repos-milestones.ts +++ b/apps/hq/src/repos-milestones.ts @@ -414,6 +414,16 @@ export async function projectsPendingMilestone(db: DB, key: string): Promise { - const rows = await db.all( +export async function stalledProjects( + db: DB, olderThanDays: number, today: string, +): Promise { + const allRows = await db.all( `SELECT pm.client_module_id AS "clientModuleId", MIN(cm.client_id) AS "clientId", MIN(c.name) AS "clientName", MIN(m.code) AS "moduleCode", MIN(m.name) AS "moduleName", MAX(pm.done_on) AS last_done, MIN(cm.created_at) AS created_at @@ -441,11 +454,19 @@ export async function stalledProjects(db: DB, olderThanDays: number, today: stri HAVING SUM(CASE WHEN pm.done = 0 THEN 1 ELSE 0 END) > 0`, ) const cutoff = new Date(Date.parse(today) - olderThanDays * 86_400_000).toISOString().slice(0, 10) - return rows - .filter((r) => { - if (r.last_done !== null) return r.last_done < cutoff - if (r.created_at !== null) return r.created_at.slice(0, 10) < cutoff - return false - }) - .map(({ last_done: _last_done, created_at: _created_at, ...rest }) => rest) + const rows: PendingProject[] = [] + let unknownAgeExcluded = 0 + for (const r of allRows) { + const { last_done, created_at, ...rest } = r + if (last_done !== null) { + if (last_done < cutoff) rows.push(rest) + continue + } + if (created_at !== null) { + if (created_at.slice(0, 10) < cutoff) rows.push(rest) + continue + } + unknownAgeExcluded++ + } + return { rows, unknownAgeExcluded } } diff --git a/apps/hq/test/db.test.ts b/apps/hq/test/db.test.ts index 3ff4591..c1f5502 100644 --- a/apps/hq/test/db.test.ts +++ b/apps/hq/test/db.test.ts @@ -1,6 +1,12 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' import { describe, it, expect } from 'vitest' import { openDb } from '../src/db' import { writeAudit, listAudit } from '../src/audit' +import { seedIfEmpty } from '../src/seed' +import { createClient } from '../src/repos-clients' +import { createModule, assignModule } from '../src/repos-modules' describe('hq db', () => { it('creates every HQ-1 table', async () => { @@ -20,4 +26,55 @@ describe('hq db', () => { expect(rows[0]!.action).toBe('update') expect(JSON.parse(rows[0]!.after_json!)).toEqual({ name: 'Acme Ltd' }) }) + + // FIX 4 (regression): the APEX importer inserted client_module rows WITHOUT created_at, so + // every imported project with zero ticks was permanently NULL-aged. migrate() must backfill + // it from installed_on where that's known — using a real (non-:memory:) DB reopened twice, + // since the backfill runs as part of migrate() on every open, not just column-creation. + describe('client_module.created_at backfill (FIX 4)', () => { + it('backfills created_at from installed_on for a legacy row missing it', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hq-db-backfill-')) + try { + let db = openDb(dir) + await seedIfEmpty(db) + const client = await createClient(db, 'u1', { name: 'Legacy Client', stateCode: '32' }) + const mod = await createModule(db, 'u1', { code: 'LEGACYBF', name: 'Legacy Module' }) + const cm = await assignModule(db, 'u1', { clientId: client.id, moduleId: mod.id, kind: 'yearly' }) + // Simulate the actual APEX-importer bug: installed_on known, created_at never stamped. + await db.run(`UPDATE client_module SET created_at=NULL, installed_on=? WHERE id=?`, '2025-11-03', cm.id) + await db.close() + + db = openDb(dir) // reopening re-runs migrate() — the backfill must heal it unprompted + const row = await db.get<{ created_at: string | null; installed_on: string | null }>( + `SELECT created_at, installed_on FROM client_module WHERE id=?`, cm.id, + ) + expect(row!.created_at).toBe('2025-11-03') + await db.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + + it('leaves created_at NULL when installed_on is also unknown — never fabricates a date', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hq-db-backfill-')) + try { + let db = openDb(dir) + await seedIfEmpty(db) + const client = await createClient(db, 'u1', { name: 'Legacy Client 2', stateCode: '32' }) + const mod = await createModule(db, 'u1', { code: 'LEGACYBF2', name: 'Legacy Module 2' }) + const cm = await assignModule(db, 'u1', { clientId: client.id, moduleId: mod.id, kind: 'yearly' }) + await db.run(`UPDATE client_module SET created_at=NULL WHERE id=?`, cm.id) // installed_on stays NULL too + await db.close() + + db = openDb(dir) + const row = await db.get<{ created_at: string | null }>( + `SELECT created_at FROM client_module WHERE id=?`, cm.id, + ) + expect(row!.created_at).toBeNull() + await db.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) + }) }) diff --git a/apps/hq/test/import-apex-full.test.ts b/apps/hq/test/import-apex-full.test.ts index 45d6cfc..bec20e3 100644 --- a/apps/hq/test/import-apex-full.test.ts +++ b/apps/hq/test/import-apex-full.test.ts @@ -201,10 +201,10 @@ const cmFor = async (db: DB, clientCode: string, moduleCode: string) => db.get<{ status: string; kind: string; provider: string | null; username: string | null password_enc: string | null; details: string; remark: string | null; installed_on: string | null - field_values: string + field_values: string; created_at: string | null }>( `SELECT cm.status, cm.kind, cm.provider, cm.username, cm.password_enc, cm.details, cm.remark, - cm.installed_on, cm.field_values + cm.installed_on, cm.field_values, cm.created_at FROM client_module cm JOIN module m ON m.id=cm.module_id JOIN client c ON c.id=cm.client_id WHERE c.code=? AND m.code=?`, clientCode, moduleCode) @@ -281,6 +281,9 @@ describe('APEX full importer v2 (real-export shapes)', () => { const atm = (await cmFor(db, '103', 'ATM'))! expect(atm.status).toBe('live') expect(atm.provider).toBe('EWIRE') + // FIX 4 (regression): the importer must stamp created_at like assignModule does — a NULL + // here means the project permanently vanishes off the "Onboarding stalled" tile. + expect(atm.created_at).not.toBeNull() const mob = (await cmFor(db, '103', 'MOBILEAPP'))! expect(JSON.parse(mob.field_values)).toEqual({ contact: '8301932166' }) // D21 declared field expect(JSON.parse(mob.details)).toEqual([]) diff --git a/apps/hq/test/projects-stalled-route.test.ts b/apps/hq/test/projects-stalled-route.test.ts index c776045..b5bb490 100644 --- a/apps/hq/test/projects-stalled-route.test.ts +++ b/apps/hq/test/projects-stalled-route.test.ts @@ -34,9 +34,12 @@ describe('GET /projects/stalled — hostile ?days= must not crash the server', ( const { base, staffH } = await httpWorld() const res = await fetch(`${base}/projects/stalled`, { headers: staffH }) expect(res.status).toBe(200) - const body = await res.json() as { ok: boolean; projects: unknown[] } + const body = await res.json() as { ok: boolean; projects: unknown[]; unknownAgeExcluded: number } expect(body.ok).toBe(true) expect(Array.isArray(body.projects)).toBe(true) + // FIX 4: the count of pending projects excluded for unprovable age travels alongside the + // list itself now, so the UI can surface "N of unknown age" instead of a silent drop. + expect(typeof body.unknownAgeExcluded).toBe('number') }) it('a sane ?days= still works normally', async () => { diff --git a/apps/hq/test/stalled-onboarding.test.ts b/apps/hq/test/stalled-onboarding.test.ts index 41ef7d0..6e00225 100644 --- a/apps/hq/test/stalled-onboarding.test.ts +++ b/apps/hq/test/stalled-onboarding.test.ts @@ -27,10 +27,11 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { await setMilestone(db, 'u1', cm.id, 'setup_done', true, '2026-03-01') // last done_on = 2026-03-01, today = 2026-07-19 → ~140 days parked. const stalled = await stalledProjects(db, 30, '2026-07-19') - expect(stalled.map((p) => p.clientModuleId)).toContain(cm.id) - const row = stalled.find((p) => p.clientModuleId === cm.id)! + expect(stalled.rows.map((p) => p.clientModuleId)).toContain(cm.id) + const row = stalled.rows.find((p) => p.clientModuleId === cm.id)! expect(row.clientName).toBe('Kothavara SCB') expect(row.moduleCode).toBe('MISC') + expect(stalled.unknownAgeExcluded).toBe(0) }) it('excludes a recently-progressed project', async () => { @@ -39,7 +40,8 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { await setMilestone(db, 'u1', cm.id, 'advance_paid', true, '2026-03-01') await setMilestone(db, 'u1', cm.id, 'setup_done', true, '2026-03-01') const stalled = await stalledProjects(db, 365, '2026-07-19') - expect(stalled).toHaveLength(0) + expect(stalled.rows).toHaveLength(0) + expect(stalled.unknownAgeExcluded).toBe(0) }) it('excludes projects with no pending steps (fully ticked) and inactive/live projects', async () => { @@ -50,7 +52,7 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { await setMilestone(db, 'u1', done.id, key, true, '2026-01-01') } const stalled = await stalledProjects(db, 1, '2026-07-19') - expect(stalled.map((p) => p.clientModuleId)).not.toContain(done.id) + expect(stalled.rows.map((p) => p.clientModuleId)).not.toContain(done.id) // 'live' status (from ticking go_live) also excludes it, which the loop above proves. expect((await db.get<{ status: string }>(`SELECT status FROM client_module WHERE id=?`, done.id))!.status).toBe('live') }) @@ -69,7 +71,8 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { // to be — this test is about the QUERY logic, not the real-time write path. await db.run(`UPDATE client_module SET created_at=? WHERE id=?`, '2026-07-19T09:00:00.000Z', cm.id) const stalled = await stalledProjects(db, 30, '2026-07-19') - expect(stalled.map((p) => p.clientModuleId)).not.toContain(cm.id) + expect(stalled.rows.map((p) => p.clientModuleId)).not.toContain(cm.id) + expect(stalled.unknownAgeExcluded).toBe(0) }) it('a project created long ago with zero ticks IS stalled', async () => { @@ -77,7 +80,7 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { const cm = await assignModule(db, 'u1', { clientId: c.id, moduleId: m.id, kind: 'yearly' }) await db.run(`UPDATE client_module SET created_at=? WHERE id=?`, '2025-01-01T09:00:00.000Z', cm.id) const stalled = await stalledProjects(db, 30, '2026-07-19') - expect(stalled.map((p) => p.clientModuleId)).toContain(cm.id) + expect(stalled.rows.map((p) => p.clientModuleId)).toContain(cm.id) }) it('a no-tick project with an UNKNOWN creation date (created_at NULL) is excluded, not assumed ancient', async () => { @@ -87,7 +90,18 @@ describe('stalledProjects (D22 — dashboard/MIS tile)', () => { // doesn't stamp it) — created_at is genuinely unknown. await db.run(`UPDATE client_module SET created_at=NULL WHERE id=?`, cm.id) const stalled = await stalledProjects(db, 0, '2026-07-19') // even a 0-day cutoff must not flag it - expect(stalled.map((p) => p.clientModuleId)).not.toContain(cm.id) + expect(stalled.rows.map((p) => p.clientModuleId)).not.toContain(cm.id) + }) + + // FIX 4 (no silent caps): the row above used to just vanish with no trace. It must now be + // counted so the UI can surface "N of unknown age" instead of silently under-reporting. + it('a no-tick row with NULL created_at lands in the unknownAgeExcluded count, not just silently dropped', async () => { + const { db, c, m } = await world() + const cm = await assignModule(db, 'u1', { clientId: c.id, moduleId: m.id, kind: 'yearly' }) + await db.run(`UPDATE client_module SET created_at=NULL WHERE id=?`, cm.id) + const stalled = await stalledProjects(db, 0, '2026-07-19') + expect(stalled.rows.map((p) => p.clientModuleId)).not.toContain(cm.id) + expect(stalled.unknownAgeExcluded).toBe(1) }) it('assignModule itself stamps a real (non-null) created_at', async () => {