fix(deploy): one shared DB-engine resolver so scripts can't silently target a throwaway SQLite file (FIX A)

apply-onboarding-pipeline.ts and migrate-onboarding-templates.ts resolved the
engine as `DATABASE_URL ?? ''`, diverging from server.ts's production
fallback. On the prod box (DATABASE_URL unset) each script opened a brand-new
empty SQLite file, "succeeded" against it, and Postgres was never touched.

resolveDbUrl()/requireDbUrlForEnv()/pgTargetDescription() now live in
db-pg.ts as the one place this is decided; server.ts and both scripts call
it. The scripts additionally hard-fail (never fall back to SQLite) when
NODE_ENV=production resolves nothing, log their target engine before doing
any work, and close the DB handle in a finally.

Also lands client_module.created_at (nullable, additive) — schema groundwork
FIX D uses to measure a no-tick project's age from its real creation date
instead of treating a missing done_on as infinitely old.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
main
Thomas Joise 24 hours ago
parent 0bcbf8d410
commit 31335cd4ad

@ -7,10 +7,11 @@
// template with the settings below — this DOES clobber owner edits, use with care.
// Run from repo root:
// npx tsx apps/hq/scripts/apply-onboarding-pipeline.ts [--force]
import { openDb } from '../src/db'
import { openPgDb } from '../src/db-pg'
import { openDb, sqliteFilePath } from '../src/db'
import { openPgDb, pgTargetDescription, requireDbUrlForEnv } from '../src/db-pg'
import { migrateOnboardingTemplates } from '../src/migrate-onboarding-templates'
import { getSetting, setSetting } from '../src/repos-reminders'
import type { DB } from '../src/db'
const FRONT = [
{ key: 'enquiry', label: 'Enquiry received' },
@ -52,29 +53,45 @@ const TAILS: Record<string, { key: string; label: string }[]> = {
}
async function main(): Promise<void> {
const pgUrl = process.env['DATABASE_URL'] ?? ''
const db = pgUrl !== '' ? await openPgDb(pgUrl) : openDb(process.env['HQ_DATA_DIR'])
const force = process.argv.includes('--force')
// Pre-ship audit FIX A: the SAME resolver server.ts uses (never re-derived locally) — and,
// unlike the server, this one-off script hard-fails rather than silently falling back to a
// throwaway SQLite file when NODE_ENV=production resolves no Postgres target.
const pgUrl = requireDbUrlForEnv()
if (pgUrl !== '') {
console.log(`[db] engine: postgres ${pgTargetDescription(pgUrl)}`)
} else {
console.log(`[db] engine: sqlite at ${sqliteFilePath(process.env['HQ_DATA_DIR'])}`)
}
const db: DB = pgUrl !== '' ? await openPgDb(pgUrl) : openDb(process.env['HQ_DATA_DIR'])
try {
const force = process.argv.includes('--force')
// Audited via setSetting (append-only-audit rule) and, unless --force, skips any key
// that's already set — so an owner's edit in the Modules onboarding-template editor is
// left alone on a re-run instead of being silently clobbered.
const upsert = async (key: string, list: unknown): Promise<void> => {
if (!force && (await getSetting(db, key)) !== null) {
// eslint-disable-next-line no-console
console.log(`Skipped ${key} — already set (pass --force to overwrite an owner edit).`)
return
// Audited via setSetting (append-only-audit rule) and, unless --force, skips any key
// that's already set — so an owner's edit in the Modules onboarding-template editor is
// left alone on a re-run instead of being silently clobbered.
const upsert = async (key: string, list: unknown): Promise<void> => {
if (!force && (await getSetting(db, key)) !== null) {
// eslint-disable-next-line no-console
console.log(`Skipped ${key} — already set (pass --force to overwrite an owner edit).`)
return
}
await setSetting(db, 'system', key, JSON.stringify(list))
}
await setSetting(db, 'system', key, JSON.stringify(list))
}
await upsert('project.milestone_template.front', FRONT)
for (const [code, list] of Object.entries(TAILS)) await upsert(`project.milestone_template:${code}`, list)
// eslint-disable-next-line no-console
console.log('Template settings step done: front + SMS/RTGS/MOBILEAPP tails.')
await upsert('project.milestone_template.front', FRONT)
for (const [code, list] of Object.entries(TAILS)) await upsert(`project.milestone_template:${code}`, list)
// eslint-disable-next-line no-console
console.log('Template settings step done: front + SMS/RTGS/MOBILEAPP tails.')
const res = await migrateOnboardingTemplates(db)
// eslint-disable-next-line no-console
console.log(`Onboarding migration: ${res.projects} project(s) rebuilt, ${res.carried} tick(s) carried, ${res.dropped} dropped, ${res.preserved} custom step(s) preserved.`)
const res = await migrateOnboardingTemplates(db)
// eslint-disable-next-line no-console
console.log(
`Onboarding migration: ${res.projects} project(s) rebuilt, ${res.carried} tick(s) carried, ` +
`${res.dropped} dropped, ${res.collapsed} collapsed (two old ticks -> one key), ` +
`${res.preserved} custom step(s) preserved.`,
)
} finally {
await db.close()
}
}
main().catch((e) => { console.error(e); process.exit(1) })

@ -3,17 +3,33 @@
// checklist onto their new per-module template, carrying mapped ticks. Idempotent — safe
// to re-run (a second run does nothing).
// DATABASE_URL=postgres://… npx tsx apps/hq/scripts/migrate-onboarding-templates.ts
import { openDb } from '../src/db'
import { openPgDb } from '../src/db-pg'
import { openDb, sqliteFilePath } from '../src/db'
import { openPgDb, pgTargetDescription, requireDbUrlForEnv } from '../src/db-pg'
import { migrateOnboardingTemplates } from '../src/migrate-onboarding-templates'
import type { DB } from '../src/db'
async function main() {
const pgUrl = process.env['DATABASE_URL'] ?? ''
const db = pgUrl !== '' ? await openPgDb(pgUrl) : openDb(process.env['HQ_DATA_DIR'])
const res = await migrateOnboardingTemplates(db)
// eslint-disable-next-line no-console
console.log(`Onboarding template migration: ${res.projects} project(s) rebuilt, ${res.carried} tick(s) carried, ${res.dropped} dropped, ${res.preserved} custom step(s) preserved.`)
async function main(): Promise<void> {
// Pre-ship audit FIX A: the SAME resolver server.ts uses (never re-derived locally) — and,
// unlike the server, this one-off script hard-fails rather than silently falling back to a
// throwaway SQLite file when NODE_ENV=production resolves no Postgres target.
const pgUrl = requireDbUrlForEnv()
if (pgUrl !== '') {
console.log(`[db] engine: postgres ${pgTargetDescription(pgUrl)}`)
} else {
console.log(`[db] engine: sqlite at ${sqliteFilePath(process.env['HQ_DATA_DIR'])}`)
}
const db: DB = pgUrl !== '' ? await openPgDb(pgUrl) : openDb(process.env['HQ_DATA_DIR'])
try {
const res = await migrateOnboardingTemplates(db)
// eslint-disable-next-line no-console
console.log(
`Onboarding template migration: ${res.projects} project(s) rebuilt, ${res.carried} tick(s) carried, ` +
`${res.dropped} dropped, ${res.collapsed} collapsed (two old ticks -> one key), ` +
`${res.preserved} custom step(s) preserved.`,
)
} finally {
await db.close()
}
}
main().catch((e) => { console.error(e); process.exit(1) })

@ -102,6 +102,56 @@ export class PgDb implements DB {
}
}
// D19 engine selection, the ONE place it is decided (pre-ship audit FIX A): DATABASE_URL
// (or, in production with no DATABASE_URL set, this hardcoded connection — team decision,
// used instead of a .env DATABASE_URL to match how the other apps on this box are
// configured) → Postgres; otherwise SQLite. Gated to NODE_ENV=production so dev and the
// test suite keep selecting SQLite (an unset/empty DATABASE_URL) and stay green.
// ⚠️ Once the real password is filled in, this file holds a secret — do not push the
// filled-in value to a repo others can read.
const HARDCODED_DATABASE_URL = 'postgres://postgres:inv123@host.docker.internal:5432/hq'
/**
* Resolve which DB engine + target this process should use server.ts AND every deploy
* script MUST call this (never re-derive it locally). Before this fix, two deploy scripts
* computed it differently (`DATABASE_URL ?? ''`, with no production fallback) and, on the
* production box where DATABASE_URL is unset, silently opened a brand-new empty SQLite file
* instead of the real Postgres the migration "succeeded" against a throwaway DB while
* production was never touched. '' means SQLite.
*/
export function resolveDbUrl(env: NodeJS.ProcessEnv = process.env, hardcoded = HARDCODED_DATABASE_URL): string {
return env['DATABASE_URL'] || (env['NODE_ENV'] === 'production' ? hardcoded : '')
}
/**
* Same resolution, but hard-fails instead of silently falling back to SQLite when running in
* production for one-off deploy scripts, where a throwaway SQLite write is never the right
* outcome (unlike the server, which legitimately runs SQLite in dev). `hardcoded` is
* injectable so tests can simulate the hardcoded URL being accidentally emptied out.
*/
export function requireDbUrlForEnv(env: NodeJS.ProcessEnv = process.env, hardcoded = HARDCODED_DATABASE_URL): string {
const pgUrl = resolveDbUrl(env, hardcoded)
if (pgUrl === '' && env['NODE_ENV'] === 'production') {
throw new Error(
'[db] refusing to run: NODE_ENV=production but no Postgres target resolved ' +
'(DATABASE_URL is unset and the hardcoded production URL is empty) — this would ' +
'otherwise silently create a throwaway SQLite database instead of touching production data.',
)
}
return pgUrl
}
/** Human-readable engine target for startup/deploy-script logs. Never includes the password. */
export function pgTargetDescription(pgUrl: string): string {
try {
const u = new URL(pgUrl)
const port = u.port !== '' ? `:${u.port}` : ''
return `${u.hostname}${port}${u.pathname}`
} catch {
return '(unparsable connection string)'
}
}
/** Apply pending numbered migrations, each atomically, recorded in schema_migrations. */
export async function runPgMigrations(db: PgDb): Promise<string[]> {
await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (id text PRIMARY KEY, applied_at text NOT NULL)`)

@ -247,7 +247,10 @@ CREATE TABLE IF NOT EXISTS client_module (
-- module's non-secret declared fields; secrets_enc is ONE AES-256-GCM blob over a JSON
-- map { fieldKey: plaintext } for its secret fields (N secrets, one encrypt; reveal
-- audited, never in list payloads). Keys are validated against the module's field_spec.
field_values TEXT NOT NULL DEFAULT '{}', secrets_enc TEXT
field_values TEXT NOT NULL DEFAULT '{}', secrets_enc TEXT,
-- Pre-ship audit FIX D: when this project was created/assigned (nullable set by
-- assignModule going forward; NULL for rows that predate this column, never fabricated).
created_at TEXT
);
CREATE TABLE IF NOT EXISTS client_branch (
id TEXT PRIMARY KEY, client_id TEXT NOT NULL, name TEXT NOT NULL,
@ -413,16 +416,26 @@ CREATE TABLE IF NOT EXISTS aws_usage (
);
`
/** Absolute path (or ':memory:') the given HQ_DATA_DIR resolves to the same rule openDb
* uses to pick its file, exposed so deploy scripts can log their target BEFORE opening it
* (pre-ship audit FIX A: a script that silently opens the wrong DB is indistinguishable
* from a correct idempotent no-op run unless it logs where it actually landed). */
export function sqliteFilePath(dataDir?: string): string {
if (dataDir === ':memory:') return ':memory:'
const dir = dataDir !== undefined ? path.resolve(dataDir) : path.resolve(process.cwd(), 'data')
return path.join(dir, 'hq.db')
}
/** Open the SQLite engine (dev/tests). Deliberately sync schema + migrate run
* on the raw handle before it is wrapped, so test fixtures stay one-liners. */
export function openDb(dataDir?: string): DB {
let raw: SqliteRaw
if (dataDir === ':memory:') {
const target = sqliteFilePath(dataDir)
if (target === ':memory:') {
raw = new Database(':memory:')
} else {
const dir = dataDir ?? path.resolve(process.cwd(), 'data')
fs.mkdirSync(dir, { recursive: true })
raw = new Database(path.join(dir, 'hq.db'))
fs.mkdirSync(path.dirname(target), { recursive: true })
raw = new Database(target)
raw.pragma('journal_mode = WAL')
}
raw.exec(SCHEMA)
@ -514,6 +527,13 @@ function migrate(db: SqliteRaw): void {
if (!cmCols.some((c) => c.name === 'field_values')) {
db.exec(`ALTER TABLE client_module ADD COLUMN field_values TEXT NOT NULL DEFAULT '{}'`)
}
// Pre-ship audit FIX D: a no-tick project's "parked since" needs its own real creation
// date. Nullable — already-existing rows keep an unknown/NULL creation date (never
// fabricated) so they're never mistakenly flagged as ancient; assignModule stamps every
// NEW client_module going forward.
if (!cmCols.some((c) => c.name === 'created_at')) {
db.exec(`ALTER TABLE client_module ADD COLUMN created_at TEXT`)
}
const modCols2 = db.prepare(`PRAGMA table_info(module)`).all() as { name: string }[]
if (!modCols2.some((c) => c.name === 'field_spec')) {
db.exec(`ALTER TABLE module ADD COLUMN field_spec TEXT NOT NULL DEFAULT '[]'`)

@ -367,4 +367,10 @@ END;
id: '017-ticket-type',
sql: `ALTER TABLE ticket ADD COLUMN IF NOT EXISTS type text NOT NULL DEFAULT 'service';`,
},
{
// Pre-ship audit FIX D: real creation date for the "no-tick project" stalled-onboarding
// measurement (nullable — existing rows keep an unknown/NULL date, never fabricated).
id: '018-client-module-created-at',
sql: `ALTER TABLE client_module ADD COLUMN IF NOT EXISTS created_at text;`,
},
]

@ -7,7 +7,7 @@ import { apiRouter } from './api'
import { maybePullAwsCosts } from './aws-costs'
import { pollBounces } from './bounces'
import { openDb, type DB } from './db'
import { openPgDb } from './db-pg'
import { openPgDb, resolveDbUrl } from './db-pg'
import { makeRateLimiter } from './rate-limit'
import { renderPdf } from './pdf'
import { getClient } from './repos-clients'
@ -144,15 +144,11 @@ export async function startServer(port: number, opts: { scheduler?: boolean } =
// shares the proxy's IP and per-IP rate limiting collapses to one global bucket.
// The proxy MUST strip client-supplied X-Forwarded-For so it cannot be spoofed.
app.set('trust proxy', 1)
// Hardcoded production DB connection (team decision — used instead of a .env DATABASE_URL,
// to match how the other apps on this box are configured). To change the database, edit
// HARDCODED_DATABASE_URL below and rebuild the image. Gated to NODE_ENV=production so dev
// and the test suite keep selecting SQLite (an unset/empty DATABASE_URL) and stay green.
// ⚠️ Once the real password is filled in, this file holds a secret — do not push the
// filled-in value to a repo others can read.
const HARDCODED_DATABASE_URL = 'postgres://postgres:inv123@host.docker.internal:5432/hq'
// D19 engine selection: DATABASE_URL (or the hardcoded prod value) → Postgres; else SQLite.
const pgUrl = process.env['DATABASE_URL'] || (process.env['NODE_ENV'] === 'production' ? HARDCODED_DATABASE_URL : '')
// D19 engine selection (pre-ship audit FIX A: the ONE shared resolver — server.ts and
// every deploy script must call this, never re-derive it locally, or a script can select
// a different DB than the running server without anyone noticing). To change the
// production database, edit HARDCODED_DATABASE_URL in db-pg.ts and rebuild the image.
const pgUrl = resolveDbUrl()
const db = pgUrl !== '' ? await openPgDb(pgUrl) : openDb(process.env['HQ_DATA_DIR'])
if (pgUrl !== '') console.log('[db] engine: postgres')
await seedIfEmpty(db) // first boot prints the owner password once

@ -0,0 +1,59 @@
// apps/hq/test/db-resolve.test.ts — pre-ship audit FIX A: server.ts and the deploy scripts
// (apps/hq/scripts/apply-onboarding-pipeline.ts, apps/hq/scripts/migrate-onboarding-templates.ts)
// must all resolve the DB engine + target through the ONE shared function. Before this fix
// the scripts computed it as `DATABASE_URL ?? ''` (no production fallback), so on the
// production box — where DATABASE_URL is unset — each script silently opened a brand-new
// empty SQLite file instead of the real Postgres.
import { describe, it, expect } from 'vitest'
import { pgTargetDescription, requireDbUrlForEnv, resolveDbUrl } from '../src/db-pg'
const HARDCODED = 'postgres://postgres:inv123@host.docker.internal:5432/hq'
describe('resolveDbUrl (D19 engine selection, the one shared resolver)', () => {
it('DATABASE_URL wins regardless of NODE_ENV', () => {
expect(resolveDbUrl({ DATABASE_URL: 'postgres://x/y', NODE_ENV: 'production' })).toBe('postgres://x/y')
expect(resolveDbUrl({ DATABASE_URL: 'postgres://x/y' })).toBe('postgres://x/y')
})
it('falls back to the hardcoded production URL only when NODE_ENV=production and DATABASE_URL is unset', () => {
expect(resolveDbUrl({ NODE_ENV: 'production' }, HARDCODED)).toBe(HARDCODED)
})
it('resolves to SQLite (empty string) outside production with no DATABASE_URL — dev/tests stay green', () => {
expect(resolveDbUrl({})).toBe('')
expect(resolveDbUrl({ NODE_ENV: 'development' })).toBe('')
expect(resolveDbUrl({ NODE_ENV: 'test' })).toBe('')
})
})
describe('requireDbUrlForEnv (deploy-script guard — hard-fail instead of a throwaway SQLite write)', () => {
it('behaves exactly like resolveDbUrl when a Postgres target is available', () => {
expect(requireDbUrlForEnv({ DATABASE_URL: 'postgres://x/y' })).toBe('postgres://x/y')
expect(requireDbUrlForEnv({ NODE_ENV: 'production' }, HARDCODED)).toBe(HARDCODED)
expect(requireDbUrlForEnv({})).toBe('') // dev with no env: SQLite is fine, no hard-fail
})
it('hard-fails instead of silently returning SQLite when NODE_ENV=production resolves nothing', () => {
// Simulates the hardcoded URL having been accidentally emptied out.
expect(() => requireDbUrlForEnv({ NODE_ENV: 'production' }, '')).toThrow(/refusing to run/i)
})
})
describe('pgTargetDescription (never leaks the password into logs)', () => {
it('extracts host:port/dbname', () => {
expect(pgTargetDescription('postgres://user:secret@db.example.com:5432/hq'))
.toBe('db.example.com:5432/hq')
})
it('omits the port when absent', () => {
expect(pgTargetDescription('postgres://user:secret@db.example.com/hq')).toBe('db.example.com/hq')
})
it('never includes the credentials', () => {
expect(pgTargetDescription('postgres://user:hunter2@host/db')).not.toContain('hunter2')
})
it('degrades gracefully on an unparsable string', () => {
expect(pgTargetDescription('not-a-url')).toMatch(/unparsable/i)
})
})
Loading…
Cancel
Save