merge: script DATABASE_URL docs + SQLite opt-in, always-stamp install/train dates, keep links on re-tick, backup resolves prod DB
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>main
Thomas Joise
22 hours ago
commit
23b0b64de2
@ -0,0 +1,24 @@
|
||||
// apps/hq/scripts/backup.ts — `npm run backup` (D33). Dumps the production Postgres to a
|
||||
// timestamped, compressed, restorable file under ./backups (or $HQ_BACKUP_DIR) and rotates to
|
||||
// the newest $HQ_BACKUP_KEEP (default 14). Run from the repo root:
|
||||
// npm run backup
|
||||
// DATABASE_URL=postgres://… npm run backup (explicit target, overrides resolution)
|
||||
// The target is resolved through the SAME shared resolver the server and every other
|
||||
// maintenance script uses (`requireDbUrlForEnv`, D19) — the old scripts/backup.mjs read
|
||||
// DATABASE_URL directly and so could never find the production database, which is resolved
|
||||
// rather than exported on that box. The resolved host/db is logged (never the password)
|
||||
// before pg_dump runs.
|
||||
//
|
||||
// Restore a dump with:
|
||||
// pg_restore --clean --if-exists --no-owner -d "$DATABASE_URL" backups/hq-<ts>.dump
|
||||
//
|
||||
// SECURITY (D32): credentials are stored in the CLEAR, so these dumps contain plaintext
|
||||
// logins. Keep backups/ access-controlled and off any shared drive (it is gitignored).
|
||||
import { runBackup } from '../src/backup'
|
||||
|
||||
try {
|
||||
runBackup()
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e))
|
||||
process.exit(1)
|
||||
}
|
||||
@ -0,0 +1,128 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { sqliteFilePath } from './db'
|
||||
import { pgTargetDescription, resolveDbUrl } from './db-pg'
|
||||
|
||||
/**
|
||||
* Database backup (D33): `pg_dump` of the production Postgres to a timestamped, compressed,
|
||||
* restorable file under ./backups (or $HQ_BACKUP_DIR), rotated to the newest N.
|
||||
*
|
||||
* Lives in src/ (not scripts/) so it is typechecked and unit-testable; `scripts/backup.ts` is
|
||||
* the thin CLI wrapper `npm run backup` invokes.
|
||||
*
|
||||
* The fix this file exists for: the previous `scripts/backup.mjs` read `process.env.DATABASE_URL`
|
||||
* directly and bailed out when it was unset. On the production box DATABASE_URL is NOT set — the
|
||||
* connection is resolved by `resolveDbUrl` (D19) — so the backup command could never reach the
|
||||
* real database. It now resolves its target through the SAME shared resolver every other
|
||||
* maintenance script uses, and logs the resolved target (never the password) before it runs.
|
||||
*
|
||||
* SECURITY (D32): credentials are stored in the CLEAR, so these dumps contain plaintext logins.
|
||||
* Keep the backups/ directory access-controlled and off any shared drive; it is gitignored so it
|
||||
* can never be committed.
|
||||
*/
|
||||
|
||||
/** Env knobs, defaulted here so the CLI wrapper stays a one-liner. */
|
||||
export const DEFAULT_KEEP = 14
|
||||
|
||||
/**
|
||||
* The Postgres URL this backup run must dump — via `resolveDbUrl`, the one shared resolver
|
||||
* (D19: DATABASE_URL, else the production connection). This is the fix: the target is
|
||||
* RESOLVED exactly as the server resolves it, instead of being read straight off
|
||||
* `process.env.DATABASE_URL`, which is not set on the production box.
|
||||
*
|
||||
* A SQLite outcome is fatal here rather than an opt-in (unlike the migration scripts, which
|
||||
* can legitimately run against SQLite): `pg_dump` cannot read a SQLite file at all, so the
|
||||
* only useful answer is a loud error naming the SQLite snapshot script instead.
|
||||
*/
|
||||
export function resolveBackupTarget(env: NodeJS.ProcessEnv = process.env): string {
|
||||
const url = resolveDbUrl(env)
|
||||
if (url === '') {
|
||||
throw new Error(
|
||||
'[backup] refusing to run: no Postgres target resolved, so this would back up nothing. ' +
|
||||
`The database resolved to SQLite at ${sqliteFilePath(env['HQ_DATA_DIR'])}, and this ` +
|
||||
'backup dumps Postgres (pg_dump). Set DATABASE_URL=postgres://… to back up the real ' +
|
||||
'database, or use apps/hq/scripts/backup-hq.sh for a SQLite snapshot.',
|
||||
)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
/** Locate pg_dump: PATH first, then a standard Windows PostgreSQL install. */
|
||||
export function findPgDump(): string | null {
|
||||
try {
|
||||
execFileSync(process.platform === 'win32' ? 'where' : 'which', ['pg_dump'], { stdio: 'ignore' })
|
||||
return 'pg_dump'
|
||||
} catch { /* not on PATH */ }
|
||||
if (process.platform === 'win32') {
|
||||
const base = 'C:/Program Files/PostgreSQL'
|
||||
try {
|
||||
for (const v of readdirSync(base).sort().reverse()) {
|
||||
const cand = join(base, v, 'bin', 'pg_dump.exe')
|
||||
if (existsSync(cand)) return cand
|
||||
}
|
||||
} catch { /* no install dir */ }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** `hq-YYYYMMDD-HHMMSS.dump` for the given local time. Pure — the rotation regex must match it. */
|
||||
export function backupFileName(now: Date): string {
|
||||
const p2 = (n: number): string => String(n).padStart(2, '0')
|
||||
const ts = `${now.getFullYear()}${p2(now.getMonth() + 1)}${p2(now.getDate())}-` +
|
||||
`${p2(now.getHours())}${p2(now.getMinutes())}${p2(now.getSeconds())}`
|
||||
return `hq-${ts}.dump`
|
||||
}
|
||||
|
||||
const DUMP_RE = /^hq-\d{8}-\d{6}\.dump$/
|
||||
|
||||
/** Which of `files` fall outside the newest `keep` dumps. Pure, so rotation is testable. */
|
||||
export function dumpsToRotate(files: readonly string[], keep: number): string[] {
|
||||
const dumps = files.filter((f) => DUMP_RE.test(f)).sort() // name sorts chronologically
|
||||
return dumps.slice(0, Math.max(0, dumps.length - Math.max(1, keep)))
|
||||
}
|
||||
|
||||
export interface BackupResult { file: string; bytes: number; rotated: string[]; retained: number }
|
||||
|
||||
/**
|
||||
* Run the backup end to end. `log` is injected so the CLI prints and tests stay quiet.
|
||||
* Env: HQ_BACKUP_DIR (default ./backups), HQ_BACKUP_KEEP (default 14).
|
||||
*/
|
||||
export function runBackup(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
log: (msg: string) => void = console.log,
|
||||
): BackupResult {
|
||||
const url = resolveBackupTarget(env)
|
||||
// Log the resolved target BEFORE doing anything: a backup that quietly dumped the wrong
|
||||
// database is indistinguishable from a correct one until the day you need the restore.
|
||||
log(`[db] engine: postgres ${pgTargetDescription(url)}`)
|
||||
|
||||
const pgDump = findPgDump()
|
||||
if (pgDump === null) {
|
||||
throw new Error('[backup] pg_dump not found. Install the PostgreSQL client tools, or add pg_dump to PATH.')
|
||||
}
|
||||
|
||||
const out = env['HQ_BACKUP_DIR'] || join(process.cwd(), 'backups')
|
||||
const keep = Math.max(1, Number(env['HQ_BACKUP_KEEP'] || DEFAULT_KEEP))
|
||||
mkdirSync(out, { recursive: true })
|
||||
const file = join(out, backupFileName(new Date()))
|
||||
|
||||
// -Fc = custom, compressed, restorable with pg_restore. --no-owner keeps it portable.
|
||||
try {
|
||||
execFileSync(pgDump, ['-Fc', '--no-owner', '-f', file, url], { stdio: ['ignore', 'inherit', 'inherit'] })
|
||||
} catch (e) {
|
||||
throw new Error(`[backup] pg_dump failed: ${e instanceof Error ? e.message : String(e)}`)
|
||||
}
|
||||
|
||||
const bytes = statSync(file).size
|
||||
log(`✓ backup written: ${file} (${(bytes / 1024).toFixed(1)} KB)`)
|
||||
|
||||
const rotated = dumpsToRotate(readdirSync(out), keep)
|
||||
for (const f of rotated) {
|
||||
unlinkSync(join(out, f))
|
||||
log(` rotated out: ${f}`)
|
||||
}
|
||||
const retained = readdirSync(out).filter((f) => DUMP_RE.test(f)).length
|
||||
log(`retained ${retained} backup(s) in ${out} (HQ_BACKUP_KEEP=${keep})`)
|
||||
return { file, bytes, rotated, retained }
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
// apps/hq/test/backup.test.ts — `npm run backup` (D33) must find the PRODUCTION database.
|
||||
// It used to read process.env.DATABASE_URL directly and exit when it was unset — but on the
|
||||
// prod box DATABASE_URL is not exported; the connection is RESOLVED (D19). So the backup
|
||||
// command could never reach the real data. It now goes through the one shared resolver, and
|
||||
// a SQLite outcome is a loud error rather than a pg_dump against a file it cannot read.
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { backupFileName, dumpsToRotate, resolveBackupTarget } from '../src/backup'
|
||||
|
||||
describe('resolveBackupTarget (same resolver as the server + every maintenance script)', () => {
|
||||
it('uses DATABASE_URL when it is set', () => {
|
||||
expect(resolveBackupTarget({ DATABASE_URL: 'postgres://x/y' })).toBe('postgres://x/y')
|
||||
})
|
||||
|
||||
it('resolves the production target with no DATABASE_URL exported — the whole point of the fix', () => {
|
||||
// NODE_ENV=production with no DATABASE_URL used to be an instant "DATABASE_URL is not set"
|
||||
// exit; it must now resolve to the same Postgres the server picks.
|
||||
const url = resolveBackupTarget({ NODE_ENV: 'production' })
|
||||
expect(url).toMatch(/^postgres:\/\//)
|
||||
})
|
||||
|
||||
it('refuses (never silently no-ops) when nothing resolves a target', () => {
|
||||
expect(() => resolveBackupTarget({})).toThrow(/refusing to run/i)
|
||||
})
|
||||
|
||||
it('says WHY, naming pg_dump, the SQLite path it landed on, and the SQLite snapshot script', () => {
|
||||
const err = (() => { try { resolveBackupTarget({ HQ_DATA_DIR: 'D:/hq-data' }); return '' } catch (e) { return String(e) } })()
|
||||
expect(err).toMatch(/pg_dump/)
|
||||
expect(err).toMatch(/backup-hq\.sh/)
|
||||
expect(err).toMatch(/hq\.db/)
|
||||
expect(err).toMatch(/DATABASE_URL=postgres/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('backup file naming + rotation', () => {
|
||||
it('names dumps hq-YYYYMMDD-HHMMSS.dump', () => {
|
||||
expect(backupFileName(new Date(2026, 6, 19, 2, 5, 9))).toBe('hq-20260719-020509.dump')
|
||||
})
|
||||
|
||||
it('rotates out everything past the newest KEEP, ignoring unrelated files', () => {
|
||||
const files = [
|
||||
'hq-20260101-000000.dump', 'hq-20260102-000000.dump', 'hq-20260103-000000.dump',
|
||||
'notes.txt', 'hq-old.dump',
|
||||
]
|
||||
expect(dumpsToRotate(files, 2)).toEqual(['hq-20260101-000000.dump'])
|
||||
expect(dumpsToRotate(files, 14)).toEqual([])
|
||||
})
|
||||
|
||||
it('keeps at least one dump however low KEEP is set', () => {
|
||||
expect(dumpsToRotate(['hq-20260101-000000.dump'], 0)).toEqual([])
|
||||
})
|
||||
})
|
||||
@ -0,0 +1,113 @@
|
||||
// apps/hq/test/milestone-link-preservation.test.ts — re-ticking an already-done step must not
|
||||
// wipe the payment / document link it already carries.
|
||||
//
|
||||
// The bug: setMilestone wrote `payment_id`/`document_id` as `done ? linked : null`, so any tick
|
||||
// that didn't resend the ids cleared them. bulkSetMilestone routes through setMilestone and
|
||||
// never passes ids at all (there is one key for many projects), so a single bulk re-tick — the
|
||||
// onboarding-cleanup tool — silently cut every project's step loose from the payment or
|
||||
// quotation it had been linked to one at a time. Unticking must still clear both.
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { openDb } from '../src/db'
|
||||
import { seedIfEmpty } from '../src/seed'
|
||||
import { createClient } from '../src/repos-clients'
|
||||
import { createModule, setPrice, assignModule } from '../src/repos-modules'
|
||||
import { createDraft } from '../src/repos-documents'
|
||||
import { recordPayment } from '../src/repos-payments'
|
||||
import { bulkSetMilestone, listProjectMilestones, setMilestone } from '../src/repos-milestones'
|
||||
|
||||
async function world() {
|
||||
const db = openDb(':memory:')
|
||||
await seedIfEmpty(db)
|
||||
const c = await createClient(db, 'u1', { name: 'Kothavara SCB', stateCode: '32' })
|
||||
const m = await createModule(db, 'u1', { code: 'SMS', name: 'Bulk SMS' })
|
||||
await setPrice(db, 'u1', { moduleId: m.id, kind: 'yearly', pricePaise: 10_000_00, effectiveFrom: '2020-01-01' })
|
||||
const cm = await assignModule(db, 'u1', { clientId: c.id, moduleId: m.id, kind: 'yearly' })
|
||||
const step = async (key: string) => (await listProjectMilestones(db, cm.id)).find((s) => s.key === key)!
|
||||
return { db, clientId: c.id, moduleId: m.id, cmId: cm.id, step }
|
||||
}
|
||||
|
||||
describe('bulk re-tick preserves the payment link (5a)', () => {
|
||||
it('tick with a payment link → bulk re-tick keeps it → bulk untick clears it', async () => {
|
||||
const { db, clientId, cmId, step } = await world()
|
||||
const pay = await recordPayment(db, 'u1', {
|
||||
clientId, receivedOn: '2026-05-01', mode: 'bank', amountPaise: 1000_00,
|
||||
})
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, undefined, pay.payment.id)
|
||||
expect((await step('payment_received')).paymentId).toBe(pay.payment.id)
|
||||
|
||||
// The bulk tool carries no ids — the stored link must survive.
|
||||
const res = await bulkSetMilestone(db, 'u1', [cmId], 'payment_received', true, '2026-06-01')
|
||||
expect(res.updated).toBe(1)
|
||||
const after = await step('payment_received')
|
||||
expect(after.done).toBe(true)
|
||||
expect(after.paymentId).toBe(pay.payment.id)
|
||||
|
||||
// Unticking still cuts the link loose.
|
||||
await bulkSetMilestone(db, 'u1', [cmId], 'payment_received', false)
|
||||
const cleared = await step('payment_received')
|
||||
expect(cleared.done).toBe(false)
|
||||
expect(cleared.doneOn).toBeNull()
|
||||
expect(cleared.paymentId).toBeNull()
|
||||
})
|
||||
|
||||
it('a single re-tick that sends no ids keeps the link too', async () => {
|
||||
const { db, clientId, cmId, step } = await world()
|
||||
const pay = await recordPayment(db, 'u1', {
|
||||
clientId, receivedOn: '2026-05-01', mode: 'bank', amountPaise: 1000_00,
|
||||
})
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, undefined, pay.payment.id)
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, '2026-06-01')
|
||||
const s = await step('payment_received')
|
||||
expect(s.doneOn).toBe('2026-06-01') // the date the caller did send still moves
|
||||
expect(s.paymentId).toBe(pay.payment.id)
|
||||
})
|
||||
|
||||
it('an explicit empty id is still an unlink, not a preserve', async () => {
|
||||
const { db, clientId, cmId, step } = await world()
|
||||
const pay = await recordPayment(db, 'u1', {
|
||||
clientId, receivedOn: '2026-05-01', mode: 'bank', amountPaise: 1000_00,
|
||||
})
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, undefined, pay.payment.id)
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, undefined, '')
|
||||
expect((await step('payment_received')).paymentId).toBeNull()
|
||||
})
|
||||
|
||||
it('the preserved link is what the audit row reports', async () => {
|
||||
const { db, clientId, cmId } = await world()
|
||||
const pay = await recordPayment(db, 'u1', {
|
||||
clientId, receivedOn: '2026-05-01', mode: 'bank', amountPaise: 1000_00,
|
||||
})
|
||||
await setMilestone(db, 'u1', cmId, 'payment_received', true, undefined, pay.payment.id)
|
||||
await bulkSetMilestone(db, 'u1', [cmId], 'payment_received', true, '2026-06-01')
|
||||
const audits = await db.all<{ after_json: string | null }>(
|
||||
`SELECT after_json FROM audit_log WHERE action='set_milestone' ORDER BY id`,
|
||||
)
|
||||
expect(JSON.parse(audits.at(-1)!.after_json!)).toMatchObject({ paymentId: pay.payment.id })
|
||||
})
|
||||
})
|
||||
|
||||
describe('bulk re-tick preserves the document link (5a)', () => {
|
||||
it('tick with a quotation link → bulk re-tick keeps it → untick clears it', async () => {
|
||||
const { db, clientId, moduleId, cmId, step } = await world()
|
||||
const doc = await createDraft(db, 'u1', {
|
||||
docType: 'QUOTATION', clientId, lines: [{ moduleId, qty: 1, kind: 'yearly' }],
|
||||
})
|
||||
await setMilestone(db, 'u1', cmId, 'quotation_sent', true, '2026-05-01', undefined, doc.id)
|
||||
expect((await step('quotation_sent')).documentId).toBe(doc.id)
|
||||
|
||||
await bulkSetMilestone(db, 'u1', [cmId], 'quotation_sent', true, '2026-06-01')
|
||||
expect((await step('quotation_sent')).documentId).toBe(doc.id)
|
||||
|
||||
await bulkSetMilestone(db, 'u1', [cmId], 'quotation_sent', false)
|
||||
expect((await step('quotation_sent')).documentId).toBeNull()
|
||||
})
|
||||
|
||||
it('a step that was never done still starts with no link (nothing to preserve)', async () => {
|
||||
const { db, cmId, step } = await world()
|
||||
await bulkSetMilestone(db, 'u1', [cmId], 'quotation_sent', true, '2026-06-01')
|
||||
const s = await step('quotation_sent')
|
||||
expect(s.done).toBe(true)
|
||||
expect(s.documentId).toBeNull()
|
||||
expect(s.paymentId).toBeNull()
|
||||
})
|
||||
})
|
||||
@ -1,78 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* SiMS HQ database backup (ops slice). Dumps the Postgres database to a timestamped,
|
||||
* compressed file under ./backups (or $HQ_BACKUP_DIR), then rotates to the newest N.
|
||||
*
|
||||
* DATABASE_URL=postgres://user:pass@host:5432/db node scripts/backup.mjs
|
||||
* npm run backup (from apps/hq — passes DATABASE_URL through)
|
||||
*
|
||||
* Restore a dump with:
|
||||
* pg_restore --clean --if-exists --no-owner -d "$DATABASE_URL" backups/hq-<ts>.dump
|
||||
*
|
||||
* SECURITY (D32): credentials are stored in the CLEAR, so these dumps contain plaintext
|
||||
* logins. Keep the backups/ directory access-controlled and off any shared drive; it is
|
||||
* gitignored so it can never be committed.
|
||||
*
|
||||
* Env: DATABASE_URL (required), HQ_BACKUP_DIR (default ./backups), HQ_BACKUP_KEEP (default 14).
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdirSync, readdirSync, statSync, unlinkSync, existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const url = process.env.DATABASE_URL
|
||||
if (!url || url.trim() === '') {
|
||||
console.error('DATABASE_URL is not set. Run: DATABASE_URL=postgres://… node scripts/backup.mjs')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
/** Locate pg_dump: PATH first, then a standard Windows PostgreSQL install. */
|
||||
function findPgDump() {
|
||||
try {
|
||||
execFileSync(process.platform === 'win32' ? 'where' : 'which', ['pg_dump'], { stdio: 'ignore' })
|
||||
return 'pg_dump'
|
||||
} catch { /* not on PATH */ }
|
||||
if (process.platform === 'win32') {
|
||||
const base = 'C:/Program Files/PostgreSQL'
|
||||
try {
|
||||
for (const v of readdirSync(base).sort().reverse()) {
|
||||
const cand = join(base, v, 'bin', 'pg_dump.exe')
|
||||
if (existsSync(cand)) return cand
|
||||
}
|
||||
} catch { /* no install dir */ }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
const pgDump = findPgDump()
|
||||
if (pgDump === null) {
|
||||
console.error('pg_dump not found. Install the PostgreSQL client tools, or add pg_dump to PATH.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const OUT = process.env.HQ_BACKUP_DIR || join(process.cwd(), 'backups')
|
||||
const KEEP = Math.max(1, Number(process.env.HQ_BACKUP_KEEP || 14))
|
||||
mkdirSync(OUT, { recursive: true })
|
||||
|
||||
const d = new Date()
|
||||
const p2 = (n) => String(n).padStart(2, '0')
|
||||
const ts = `${d.getFullYear()}${p2(d.getMonth() + 1)}${p2(d.getDate())}-${p2(d.getHours())}${p2(d.getMinutes())}${p2(d.getSeconds())}`
|
||||
const file = join(OUT, `hq-${ts}.dump`)
|
||||
|
||||
// -Fc = custom, compressed, restorable with pg_restore. --no-owner keeps it portable.
|
||||
try {
|
||||
execFileSync(pgDump, ['-Fc', '--no-owner', '-f', file, url], { stdio: ['ignore', 'inherit', 'inherit'] })
|
||||
} catch (e) {
|
||||
console.error(`pg_dump failed: ${e instanceof Error ? e.message : String(e)}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const kb = (statSync(file).size / 1024).toFixed(1)
|
||||
console.log(`✓ backup written: ${file} (${kb} KB)`)
|
||||
|
||||
// Rotation: keep the newest KEEP hq-YYYYMMDD-HHMMSS.dump files.
|
||||
const dumps = readdirSync(OUT).filter((f) => /^hq-\d{8}-\d{6}\.dump$/.test(f)).sort()
|
||||
for (const f of dumps.slice(0, Math.max(0, dumps.length - KEEP))) {
|
||||
unlinkSync(join(OUT, f))
|
||||
console.log(` rotated out: ${f}`)
|
||||
}
|
||||
console.log(`retained ${Math.min(dumps.length, KEEP)} backup(s) in ${OUT} (HQ_BACKUP_KEEP=${KEEP})`)
|
||||
Loading…
Reference in New Issue