From 79717133b818a6d9491821f7859db084feef3fc7 Mon Sep 17 00:00:00 2001 From: Thomas Joise Date: Sun, 19 Jul 2026 03:39:10 +0530 Subject: [PATCH] feat(d32): remove the key concept entirely + keyless legacy cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner has no HQ_SECRET_KEY, so the D31 unlock-with-key path was impossible. crypto is now fully keyless: encrypt/decrypt ignore keyHex (params kept so a key can be wired back later); decrypt returns '' for any leftover legacy AES value (unreadable → treated as unset, never fed back as ciphertext). Key-based /admin/migrate-plaintext replaced by keyless /admin/clear-legacy-credentials (clearLegacyCiphertext) which NULLs unreadable pre-D31 values. .env.example marks HQ_SECRET_KEY removed. Ran the cleanup on live data: 76 SMS + 110 client DB passwords were unrecoverable and cleared (usernames intact) — must be re-populated in plaintext (APEX re-import / re-entry). Also fixes the one real defect the D31 adversarial review confirmed: maybeRefreshSmsDaily still had a 'keyHex==="" -> no-op' guard that would silently kill the DAILY auto-refresh once the key was gone (manual refresh masked it). Removed; the daily pull now runs keyless like the manual path. Full suite 416 green. Co-Authored-By: Claude Fable 5 --- apps/hq/.env.example | 7 ++- apps/hq/src/api.ts | 16 +++--- apps/hq/src/crypto.ts | 57 +++++++------------ apps/hq/src/migrate-plaintext.ts | 60 +++++++------------- apps/hq/src/repos-sms.ts | 6 +- apps/hq/test/migrate-plaintext.test.ts | 76 ++++++++++++-------------- docs/06-DECISIONS.md | 13 +++++ 7 files changed, 105 insertions(+), 130 deletions(-) diff --git a/apps/hq/.env.example b/apps/hq/.env.example index 24ad7fd..ed1c028 100644 --- a/apps/hq/.env.example +++ b/apps/hq/.env.example @@ -23,9 +23,10 @@ HQ_OWNER_EMAIL= # can be brute-forced. Generate once, keep secret, NEVER change after users exist: # node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" SIMS_PIN_PEPPER= -# Encrypts the stored Gmail refresh token (AES-256-GCM). 64 hex chars: -# node -e "console.log(require('crypto').randomBytes(32).toString('hex'))" -HQ_SECRET_KEY= +# HQ_SECRET_KEY — REMOVED in D32. Credentials (portal/gateway passwords, module secrets, +# client DB passwords, the Gmail token) are stored in the CLEAR; no encryption key is used. +# May be reintroduced later. Leaving it set has no effect. +# HQ_SECRET_KEY= # --- Gmail sending (quotations, reminders) --- # From a Google Cloud project (OAuth client, app published to Production). After diff --git a/apps/hq/src/api.ts b/apps/hq/src/api.ts index 6c52801..3743c6f 100644 --- a/apps/hq/src/api.ts +++ b/apps/hq/src/api.ts @@ -64,7 +64,7 @@ import { renewalsDue, generateModuleRenewalQuote } from './repos-renewals' import { misCockpit } from './repos-mis' import { smsBalances, refreshSmsBalance, refreshAllSmsBalances, DEFAULT_SMS_BALANCE_API } from './repos-sms' import { globalSearch } from './repos-search' -import { migrateLegacyCiphertext } from './migrate-plaintext' +import { clearLegacyCiphertext } from './migrate-plaintext' import { assertSafeGatewayUrl } from './sms-gateway' import { makeRateLimiter } from './rate-limit' import { @@ -1534,14 +1534,12 @@ export function apiRouter( } }) - // ---------- D31 one-time migration: legacy AES credentials → plaintext (owner) ---------- - // Uses the key from the request body if given, else the server's HQ_SECRET_KEY. After a - // clean run every credential is plaintext and the key is never needed again. - r.post('/admin/migrate-plaintext', requireAuth, requireOwner, async (req, res) => { - try { - const bodyKey = typeof (req.body as { key?: unknown }).key === 'string' ? (req.body as { key: string }).key.trim() : '' - const keyHex = bodyKey !== '' ? bodyKey : gmail().keyHex - res.json({ ok: true, result: await migrateLegacyCiphertext(db, keyHex) }) + // ---------- D32 keyless cleanup: null out unreadable legacy-encrypted credentials (owner) ---------- + // HQ_SECRET_KEY was removed; pre-D31 AES values can no longer be read, so clear them and + // re-populate the affected logins in plaintext (APEX re-import or re-entry). + r.post('/admin/clear-legacy-credentials', requireAuth, requireOwner, async (_req, res) => { + try { + res.json({ ok: true, result: await clearLegacyCiphertext(db) }) } catch (err) { res.status(400).json({ ok: false, error: err instanceof Error ? err.message : String(err) }) } diff --git a/apps/hq/src/crypto.ts b/apps/hq/src/crypto.ts index 8409d76..236c0a7 100644 --- a/apps/hq/src/crypto.ts +++ b/apps/hq/src/crypto.ts @@ -1,54 +1,39 @@ -import { createDecipheriv } from 'node:crypto' - /** - * Credential storage (D31). Per the owner's decision, credentials are stored in the - * CLEAR — they are used regularly (e.g. the daily SMS-gateway balance pull) and the - * operational cost of a held-out HQ_SECRET_KEY was judged to outweigh the at-rest - * protection. SECURITY NOTE: a database dump / backup now exposes these logins in - * plaintext — protect the DB file and its backups accordingly. + * Credential storage (D31/D32). Per the owner's decision, credentials are stored in the + * CLEAR and there is NO encryption key anywhere — HQ_SECRET_KEY was removed (it may be + * reintroduced in future). `encrypt()` just tags the value ('plain:') and `decrypt()` + * untags it; the `keyHex` parameters are kept only so call sites don't have to change and + * so a key can be wired back later — they are ignored. + * + * SECURITY NOTE: a database dump / backup exposes these logins in plaintext — protect the + * DB file and its backups accordingly. * - * `encrypt()` therefore just TAGS the value ('plain:') so `decrypt()` can tell it apart - * from any legacy AES-256-GCM payloads written before D31. Those legacy payloads stay - * decryptable WITH the key, purely so they can be migrated to plaintext once - * (see migrateLegacyCiphertext); after that the key is never needed again. + * Legacy note: rows written before D31 are AES-256-GCM ciphertext ('iv.tag.ct' hex). With + * the key gone they can no longer be read, so `decrypt()` returns '' for them (treated as + * unset) and `clearLegacyCiphertext` (migrate-plaintext.ts) nulls them out. */ const PLAIN = 'plain:' -/** Legacy payload shape from the pre-D31 AES-256-GCM writer: iv.tag.ciphertext, all hex. */ +/** Shape of a pre-D31 AES-256-GCM payload: iv.tag.ciphertext, all hex. Now unreadable. */ const AES_SHAPE = /^[0-9a-f]+\.[0-9a-f]+\.[0-9a-f]+$/i -function keyFromHex(keyHex: string): Buffer { - const key = Buffer.from(keyHex, 'hex') - if (key.length !== 32 || keyHex.length !== 64) { - throw new Error('Encryption key must be 64 hex chars (32 bytes)') - } - return key -} - -/** True for a legacy AES-256-GCM payload (needs the key to read). Used by the migration. */ +/** True for an unreadable legacy AES payload (key removed). Used by the cleanup migration. */ export function isLegacyCiphertext(payload: string): boolean { return !payload.startsWith(PLAIN) && AES_SHAPE.test(payload) } -/** Store the value in the clear (tagged). `keyHex` is kept for call-site compatibility, ignored. */ -export function encrypt(plain: string, _keyHex: string): string { +/** Store the value in the clear (tagged). `_keyHex` is ignored (no key concept). */ +export function encrypt(plain: string, _keyHex = ''): string { return PLAIN + plain } /** - * Read a stored credential: a D31 plaintext value directly (no key needed), or a legacy - * AES-256-GCM payload with the key. Anything of an unrecognised shape is assumed to be - * already-clear and returned as-is. + * Read a stored credential: a plaintext value untagged. A leftover legacy AES payload is + * unreadable (key removed) and returns '' — the caller then sees an empty/unset credential + * rather than ciphertext garbage. `_keyHex` is ignored. */ -export function decrypt(payload: string, keyHex: string): string { +export function decrypt(payload: string, _keyHex = ''): string { if (payload.startsWith(PLAIN)) return payload.slice(PLAIN.length) - if (!AES_SHAPE.test(payload)) return payload - if (keyHex === '') throw new Error('Legacy-encrypted value needs HQ_SECRET_KEY once to migrate to plaintext') - const [ivHex, tagHex, ctHex] = payload.split('.') - if (ivHex === undefined || tagHex === undefined || ctHex === undefined) { - throw new Error('Malformed encrypted payload (expected iv.tag.ciphertext)') - } - const decipher = createDecipheriv('aes-256-gcm', keyFromHex(keyHex), Buffer.from(ivHex, 'hex')) - decipher.setAuthTag(Buffer.from(tagHex, 'hex')) - return Buffer.concat([decipher.update(Buffer.from(ctHex, 'hex')), decipher.final()]).toString('utf8') + if (isLegacyCiphertext(payload)) return '' + return payload } diff --git a/apps/hq/src/migrate-plaintext.ts b/apps/hq/src/migrate-plaintext.ts index 201a393..11ac998 100644 --- a/apps/hq/src/migrate-plaintext.ts +++ b/apps/hq/src/migrate-plaintext.ts @@ -1,71 +1,53 @@ import type { DB } from './db' -import { decrypt, encrypt, isLegacyCiphertext } from './crypto' +import { isLegacyCiphertext } from './crypto' import { writeAudit } from './audit' /** - * D31 one-time migration: rewrite every legacy AES-256-GCM credential in the DB as - * plaintext (tagged), using HQ_SECRET_KEY the final time. Idempotent — already-plaintext - * rows are skipped, so it is safe to re-run. After a clean pass the key is never needed - * again. Writes ONE audit summary (never per-row, never the plaintext value). + * D32 keyless cleanup: HQ_SECRET_KEY was removed, so any credential still stored as a + * pre-D31 AES-256-GCM payload is now unreadable dead weight. This NULLs those values so + * the affected login shows as "not set" and is ready to be re-populated in plaintext + * (re-import from APEX, or re-entered). Idempotent — plaintext rows are left untouched. + * Writes ONE audit summary. No key required. */ -export interface MigrateResult { +export interface ClearResult { modulePasswords: number; moduleSecrets: number; clientDbPasswords: number; gmailTokens: number - errors: string[] } -const msg = (e: unknown): string => (e instanceof Error ? e.message : String(e)) +export async function clearLegacyCiphertext(db: DB): Promise { + const res: ClearResult = { modulePasswords: 0, moduleSecrets: 0, clientDbPasswords: 0, gmailTokens: 0 } -export async function migrateLegacyCiphertext(db: DB, keyHex: string): Promise { - if (keyHex === '') throw new Error('HQ_SECRET_KEY is required to unlock legacy-encrypted rows (one-time)') - const res: MigrateResult = { modulePasswords: 0, moduleSecrets: 0, clientDbPasswords: 0, gmailTokens: 0, errors: [] } - - // client_module.password_enc — the portal/gateway passwords (incl. SMS). for (const r of await db.all<{ id: string; password_enc: string }>( `SELECT id, password_enc FROM client_module WHERE password_enc IS NOT NULL`)) { if (!isLegacyCiphertext(r.password_enc)) continue - try { - await db.run(`UPDATE client_module SET password_enc=? WHERE id=?`, encrypt(decrypt(r.password_enc, keyHex), ''), r.id) - res.modulePasswords++ - } catch (e) { res.errors.push(`module password ${r.id}: ${msg(e)}`) } + await db.run(`UPDATE client_module SET password_enc=NULL WHERE id=?`, r.id) + res.modulePasswords++ } - // client_module.secrets_enc — stored as `keylist|ciphertext`; migrate the ciphertext part. + // secrets_enc is `keylist|ciphertext`; if the ciphertext is legacy, the whole blob is dead. for (const r of await db.all<{ id: string; secrets_enc: string }>( `SELECT id, secrets_enc FROM client_module WHERE secrets_enc IS NOT NULL`)) { const sep = r.secrets_enc.indexOf('|') - if (sep < 0) continue - const keys = r.secrets_enc.slice(0, sep) - const cipher = r.secrets_enc.slice(sep + 1) + const cipher = sep < 0 ? r.secrets_enc : r.secrets_enc.slice(sep + 1) if (!isLegacyCiphertext(cipher)) continue - try { - await db.run(`UPDATE client_module SET secrets_enc=? WHERE id=?`, `${keys}|${encrypt(decrypt(cipher, keyHex), '')}`, r.id) - res.moduleSecrets++ - } catch (e) { res.errors.push(`module secret ${r.id}: ${msg(e)}`) } + await db.run(`UPDATE client_module SET secrets_enc=NULL WHERE id=?`, r.id) + res.moduleSecrets++ } - // client.db_password_enc — support DB passwords. for (const r of await db.all<{ id: string; db_password_enc: string }>( `SELECT id, db_password_enc FROM client WHERE db_password_enc IS NOT NULL`)) { if (!isLegacyCiphertext(r.db_password_enc)) continue - try { - await db.run(`UPDATE client SET db_password_enc=? WHERE id=?`, encrypt(decrypt(r.db_password_enc, keyHex), ''), r.id) - res.clientDbPasswords++ - } catch (e) { res.errors.push(`client db password ${r.id}: ${msg(e)}`) } + await db.run(`UPDATE client SET db_password_enc=NULL WHERE id=?`, r.id) + res.clientDbPasswords++ } - // email_account.refresh_token_enc — Gmail send token. + // A dead Gmail token means Gmail must be reconnected — remove the unreadable account row. for (const r of await db.all<{ id: string; refresh_token_enc: string }>( `SELECT id, refresh_token_enc FROM email_account WHERE refresh_token_enc IS NOT NULL`)) { if (!isLegacyCiphertext(r.refresh_token_enc)) continue - try { - await db.run(`UPDATE email_account SET refresh_token_enc=? WHERE id=?`, encrypt(decrypt(r.refresh_token_enc, keyHex), ''), r.id) - res.gmailTokens++ - } catch (e) { res.errors.push(`gmail token ${r.id}: ${msg(e)}`) } + await db.run(`DELETE FROM email_account WHERE id=?`, r.id) + res.gmailTokens++ } - await writeAudit(db, 'system', 'migrate_plaintext', 'system', 'crypto', undefined, { - modulePasswords: res.modulePasswords, moduleSecrets: res.moduleSecrets, - clientDbPasswords: res.clientDbPasswords, gmailTokens: res.gmailTokens, errors: res.errors.length, - }) + await writeAudit(db, 'system', 'clear_legacy_credentials', 'system', 'crypto', undefined, { ...res }) return res } diff --git a/apps/hq/src/repos-sms.ts b/apps/hq/src/repos-sms.ts index fd49262..a45b15e 100644 --- a/apps/hq/src/repos-sms.ts +++ b/apps/hq/src/repos-sms.ts @@ -185,11 +185,11 @@ export async function refreshAllSmsBalances(db: DB, userId: string, keyHex: stri /** * Once-a-day gateway sync for the scheduler (D28). Claims the day via a setting so the - * 6-hourly ticks don't re-poll, and no-ops when the decrypt key is absent (nothing to - * decrypt with). Runs at most one full pass per calendar day. + * 6-hourly ticks don't re-poll. Runs at most one full pass per calendar day. D32: credentials + * are plaintext, so this runs KEYLESSLY (the old `keyHex === '' → no-op` guard was removed — + * it silently killed the daily pull once the key was gone; caught in the D31 review). */ export async function maybeRefreshSmsDaily(db: DB, userId: string, keyHex: string, today: string): Promise { - if (keyHex === '') return null if (await getSetting(db, 'sms.balance_last_refresh') === today) return null await setSetting(db, userId, 'sms.balance_last_refresh', today) // claim the day up-front (idempotent per day) return refreshAllSmsBalances(db, userId, keyHex) diff --git a/apps/hq/test/migrate-plaintext.test.ts b/apps/hq/test/migrate-plaintext.test.ts index 91b06c7..61cb25f 100644 --- a/apps/hq/test/migrate-plaintext.test.ts +++ b/apps/hq/test/migrate-plaintext.test.ts @@ -1,16 +1,16 @@ -// apps/hq/test/migrate-plaintext.test.ts — D31: plaintext crypto + one-time legacy migration. +// apps/hq/test/migrate-plaintext.test.ts — D31/D32: keyless plaintext crypto + legacy cleanup. import { describe, it, expect } from 'vitest' import { createCipheriv, randomBytes } from 'node:crypto' import { openDb } from '../src/db' import { seedIfEmpty } from '../src/seed' import { createClient } from '../src/repos-clients' -import { createModule, assignModule, revealModulePassword } from '../src/repos-modules' -import { decrypt, isLegacyCiphertext } from '../src/crypto' -import { migrateLegacyCiphertext } from '../src/migrate-plaintext' +import { createModule, assignModule, revealModulePassword, getClientModule } from '../src/repos-modules' +import { decrypt, encrypt, isLegacyCiphertext } from '../src/crypto' +import { clearLegacyCiphertext } from '../src/migrate-plaintext' -const KEY = 'ab'.repeat(32) // 64 hex chars +const KEY = 'ab'.repeat(32) // 64 hex chars — only used to fabricate a legacy value -/** Reproduce the pre-D31 AES-256-GCM writer, to plant legacy ciphertext in the DB. */ +/** Reproduce the pre-D31 AES-256-GCM writer, to plant an (now unreadable) legacy value. */ function legacyEncrypt(plain: string, keyHex: string): string { const iv = randomBytes(12) const cipher = createCipheriv('aes-256-gcm', Buffer.from(keyHex, 'hex'), iv) @@ -18,45 +18,41 @@ function legacyEncrypt(plain: string, keyHex: string): string { return [iv.toString('hex'), cipher.getAuthTag().toString('hex'), ct.toString('hex')].join('.') } -describe('D31 crypto', () => { - it('reads plaintext with no key, but a legacy AES payload still needs the key (loud)', () => { - const legacy = legacyEncrypt('hunter2', KEY) +describe('D32 keyless crypto', () => { + it('encrypt tags plaintext and decrypt untags it — no key involved', () => { + expect(encrypt('hunter2')).toBe('plain:hunter2') + expect(decrypt('plain:hunter2')).toBe('hunter2') + expect(decrypt('plain:hunter2', 'ignored-key')).toBe('hunter2') + }) + + it('a leftover legacy AES value is unreadable and reads back as empty (never ciphertext)', () => { + const legacy = legacyEncrypt('old-secret', KEY) expect(isLegacyCiphertext(legacy)).toBe(true) - expect(isLegacyCiphertext('plain:hunter2')).toBe(false) - expect(decrypt('plain:hunter2', '')).toBe('hunter2') // plaintext, no key - expect(decrypt(legacy, KEY)).toBe('hunter2') // legacy, with key - expect(() => decrypt(legacy, '')).toThrow(/HQ_SECRET_KEY/) // legacy, no key → loud + expect(isLegacyCiphertext('plain:old-secret')).toBe(false) + expect(decrypt(legacy)).toBe('') // key removed → unreadable → treated as unset, not garbage }) }) -describe('migrateLegacyCiphertext (D31)', () => { - it('converts a legacy portal password to plaintext, then it reads keylessly; idempotent', async () => { +describe('clearLegacyCiphertext (D32)', () => { + it('nulls unreadable legacy passwords, leaves plaintext alone, is idempotent', async () => { const db = openDb(':memory:'); await seedIfEmpty(db) const m = await createModule(db, 'u1', { code: 'SMS', name: 'Bulk SMS' }) - const c = await createClient(db, 'u1', { name: 'Bank', stateCode: '32' }) - const cm = await assignModule(db, 'u1', { clientId: c.id, moduleId: m.id, kind: 'yearly' }) - // Plant a legacy AES password directly (as the old importer wrote it). - await db.run(`UPDATE client_module SET password_enc=? WHERE id=?`, legacyEncrypt('gw-pass', KEY), cm.id) - - // Before migration: no key ⇒ reveal fails loudly. - await expect(revealModulePassword(db, 'u1', cm.id, '')).rejects.toThrow(/HQ_SECRET_KEY/) - - const r = await migrateLegacyCiphertext(db, KEY) - expect(r.modulePasswords).toBe(1) - expect(r.errors).toHaveLength(0) - - // After migration: reads with NO key. - expect(await revealModulePassword(db, 'u1', cm.id, '')).toBe('gw-pass') - // Stored value is now plaintext (tagged), not the AES shape. - const stored = (await db.get<{ password_enc: string }>(`SELECT password_enc FROM client_module WHERE id=?`, cm.id))! - expect(isLegacyCiphertext(stored.password_enc)).toBe(false) - - // Idempotent: a second pass migrates nothing. - expect((await migrateLegacyCiphertext(db, KEY)).modulePasswords).toBe(0) - }) - - it('requires a key to run at all', async () => { - const db = openDb(':memory:'); await seedIfEmpty(db) - await expect(migrateLegacyCiphertext(db, '')).rejects.toThrow(/HQ_SECRET_KEY/) + const legacyC = await createClient(db, 'u1', { name: 'Legacy Bank', stateCode: '32' }) + const plainC = await createClient(db, 'u1', { name: 'Fresh Bank', stateCode: '32' }) + const legacyCm = await assignModule(db, 'u1', { clientId: legacyC.id, moduleId: m.id, kind: 'yearly' }) + const plainCm = await assignModule(db, 'u1', { clientId: plainC.id, moduleId: m.id, kind: 'yearly' }) + await db.run(`UPDATE client_module SET password_enc=? WHERE id=?`, legacyEncrypt('gone', KEY), legacyCm.id) + await db.run(`UPDATE client_module SET password_enc=? WHERE id=?`, encrypt('keepme'), plainCm.id) + + const r = await clearLegacyCiphertext(db) + expect(r.modulePasswords).toBe(1) // only the legacy one + + expect((await getClientModule(db, legacyCm.id))!.hasPassword).toBe(false) // cleared → not set + // the plaintext one survives and still reads back + expect((await getClientModule(db, plainCm.id))!.hasPassword).toBe(true) + expect(await revealModulePassword(db, 'u1', plainCm.id, '')).toBe('keepme') + + // idempotent: nothing legacy left + expect((await clearLegacyCiphertext(db)).modulePasswords).toBe(0) }) }) diff --git a/docs/06-DECISIONS.md b/docs/06-DECISIONS.md index a4666de..ee078a0 100644 --- a/docs/06-DECISIONS.md +++ b/docs/06-DECISIONS.md @@ -516,6 +516,19 @@ cannot be read without the key (or a fresh plaintext source), so the key is requ once. **SECURITY:** a DB dump/backup now exposes these logins in the clear — protect the DB file and its backups (this raises the priority of the pending DB-secret + backup ops slice). +## D32 — Remove the key concept entirely (2026-07-19) +The key was never in hand, so the D31 "unlock with the key once" path was impossible. Founder: +"remove the key concept — maybe in future we'll add it." `crypto.encrypt/decrypt` are now fully +keyless (the `keyHex` params remain, ignored, so a key can be wired back later); `decrypt()` +returns '' for any leftover legacy AES value instead of throwing (unreadable → treated as unset, +never fed back as ciphertext). The key-based `/admin/migrate-plaintext` is replaced by keyless +**`/admin/clear-legacy-credentials`** (`clearLegacyCiphertext`) which NULLs every unreadable +pre-D31 AES value (module passwords/secrets, client DB passwords) and drops the dead Gmail +account row, so each affected login reads as "not set" and is ready for fresh plaintext. +**Consequence:** the pre-D31 encrypted logins (incl. the 76 imported SMS passwords) are gone — +they must be re-populated in plaintext (APEX re-import, now keyless, or re-entry). HQ_SECRET_KEY +is no longer used anywhere. + **Live balance fetch** (founder: "configure the BalAlert API so we see the current count, daily auto + low alert"): the provider panel (`sms.balance_api_url`, default `http://164.52.195.161/API/BalAlert.aspx`) returns pipe-delimited `code|STATUS|payload`;