feat(hq): aws cost explorer ingestion — sigv4 signer and monthly tag-grouped pull

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

apps/hq/src/aws-costs.ts

@@ -0,0 +1,124 @@
+import { createHash, createHmac } from 'node:crypto'
+import type { DB } from './db'
+import type { Fetcher } from './gmail'
+import { getSetting, setSetting } from './repos-reminders'
+import { previousMonth, upsertAwsUsage } from './repos-aws'
+
+/**
+ * AWS Cost Explorer ingestion — SigV4 signed with node:crypto (no SDK), fetch
+ * injected (Fetcher) so tests stub the network. GetCostAndUsage is grouped by
+ * the cost-allocation tag 'client'; each tag value maps to a client by code and
+ * upserts one aws_usage row per (client, month). Untagged/unknown tag values are
+ * returned in the result, never dropped. Credentials arrive as a parameter —
+ * the caller (server wiring / route) reads them from env, never this module.
+ */
+
+export interface AwsCreds { accessKeyId: string; secretAccessKey: string }
+export interface SignedRequest { url: string; method: 'POST'; headers: Record<string, string>; body: string }
+
+const REGION = 'us-east-1' // Cost Explorer is a global service pinned to us-east-1
+const SERVICE = 'ce'
+const TARGET = 'AWSInsightsIndexService.GetCostAndUsage'
+
+function sha256Hex(data: string): string {
+  return createHash('sha256').update(data, 'utf8').digest('hex')
+}
+function hmac(key: Buffer | string, data: string): Buffer {
+  return createHmac('sha256', key).update(data, 'utf8').digest()
+}
+
+/** Full SigV4: canonical request → string-to-sign → signing key → Authorization header. */
+export function signAwsRequest(args: {
+  creds: AwsCreds; region: string; service: string; target: string; body: string; now: Date
+}): SignedRequest {
+  const { creds, region, service, target, body, now } = args
+  const host = `${service}.${region}.amazonaws.com`
+  const amzDate = now.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') // 20260201T000000Z
+  const dateStamp = amzDate.slice(0, 8) // 20260201
+  const contentType = 'application/x-amz-json-1.1'
+
+  // Canonical request — headers lowercased and sorted, body SHA-256 hashed.
+  const payloadHash = sha256Hex(body)
+  const canonicalHeaders =
+    `content-type:${contentType}\n` + `host:${host}\n` +
+    `x-amz-date:${amzDate}\n` + `x-amz-target:${target}\n`
+  const signedHeaders = 'content-type;host;x-amz-date;x-amz-target'
+  const canonicalRequest = ['POST', '/', '', canonicalHeaders, signedHeaders, payloadHash].join('\n')
+
+  // String to sign.
+  const scope = `${dateStamp}/${region}/${service}/aws4_request`
+  const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, sha256Hex(canonicalRequest)].join('\n')
+
+  // Derived signing key + signature.
+  const kDate = hmac(`AWS4${creds.secretAccessKey}`, dateStamp)
+  const kRegion = hmac(kDate, region)
+  const kService = hmac(kRegion, service)
+  const kSigning = hmac(kService, 'aws4_request')
+  const signature = createHmac('sha256', kSigning).update(stringToSign, 'utf8').digest('hex')
+
+  const authorization =
+    `AWS4-HMAC-SHA256 Credential=${creds.accessKeyId}/${scope}, ` +
+    `SignedHeaders=${signedHeaders}, Signature=${signature}`
+
+  return {
+    url: `https://${host}/`, method: 'POST', body,
+    // 'host' is set by undici on the wire too; keeping it here documents what was signed.
+    headers: { 'content-type': contentType, host, 'x-amz-date': amzDate, 'x-amz-target': target, authorization },
+  }
+}
+
+export interface AwsCostDeps { f: Fetcher; creds: AwsCreds; now?: () => string }
+export interface PullResult { upserted: number; unknownTags: { tag: string; costPaise: number }[]; totalPaise: number }
+
+interface CeGroup { Keys?: string[]; Metrics?: { UnblendedCost?: { Amount?: string; Unit?: string } } }
+interface CeResponse { ResultsByTime?: { Groups?: CeGroup[] }[] }
+
+/** '2026-06' → { start: '2026-06-01', end: '2026-07-01' } (End is exclusive in Cost Explorer). */
+function monthBounds(month: string): { start: string; end: string } {
+  const [y, m] = month.split('-').map(Number)
+  return { start: `${month}-01`, end: new Date(Date.UTC(y!, m!, 1)).toISOString().slice(0, 10) }
+}
+
+export async function pullMonthlyCosts(db: DB, deps: AwsCostDeps, month: string): Promise<PullResult> {
+  const { start, end } = monthBounds(month)
+  const body = JSON.stringify({
+    TimePeriod: { Start: start, End: end },
+    Granularity: 'MONTHLY',
+    Metrics: ['UnblendedCost'],
+    GroupBy: [{ Type: 'TAG', Key: 'client' }],
+  })
+  const now = deps.now !== undefined ? new Date(deps.now()) : new Date()
+  const signed = signAwsRequest({ creds: deps.creds, region: REGION, service: SERVICE, target: TARGET, body, now })
+  const res = await deps.f(signed.url, { method: 'POST', headers: signed.headers, body: signed.body })
+  const json = (await res.json().catch(() => ({}))) as CeResponse
+  if (!res.ok) throw new Error(`Cost Explorer request failed: HTTP ${res.status}`)
+
+  const groups = json.ResultsByTime?.[0]?.Groups ?? []
+  const unknownTags: { tag: string; costPaise: number }[] = []
+  let upserted = 0
+  let totalPaise = 0
+  for (const g of groups) {
+    const cost = g.Metrics?.UnblendedCost
+    const unit = cost?.Unit ?? 'INR'
+    if (unit !== 'INR') throw new Error(`Cost Explorer returned ${unit}, expected INR — HQ-3a assumes INR billing`)
+    const tag = (g.Keys?.[0] ?? '').split('$')[1] ?? '' // 'client$acme' → 'acme'; 'client$' → ''
+    const costPaise = Math.round(Number(cost?.Amount ?? '0') * 100)
+    totalPaise += costPaise
+    const client = tag === ''
+      ? undefined
+      : db.prepare(`SELECT id FROM client WHERE lower(code)=lower(?)`).get(tag) as { id: string } | undefined
+    if (client === undefined) { unknownTags.push({ tag, costPaise }); continue }
+    upsertAwsUsage(db, 'system', { clientId: client.id, month, costPaise, source: 'auto' })
+    upserted += 1
+  }
+  return { upserted, unknownTags, totalPaise }
+}
+
+/** Gate the pull to once per calendar month; the 6-hourly tick calls this. */
+export async function maybePullAwsCosts(db: DB, deps: AwsCostDeps, today: string): Promise<PullResult | null> {
+  const target = previousMonth(today)
+  if (getSetting(db, 'aws.last_pull_month') === target) return null
+  const out = await pullMonthlyCosts(db, deps, target)
+  setSetting(db, 'system', 'aws.last_pull_month', target)
+  return out
+}

apps/hq/test/aws-costs.test.ts

@@ -0,0 +1,81 @@
+// apps/hq/test/aws-costs.test.ts
+import { describe, it, expect } from 'vitest'
+import { openDb } from '../src/db'
+import { seedIfEmpty } from '../src/seed'
+import { createClient } from '../src/repos-clients'
+import { getSetting } from '../src/repos-reminders'
+import { previousMonth } from '../src/repos-aws'
+import { signAwsRequest, pullMonthlyCosts, maybePullAwsCosts, type AwsCostDeps } from '../src/aws-costs'
+
+const CREDS = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' }
+
+describe('signAwsRequest (SigV4)', () => {
+  it('produces a deterministic, well-formed authorization header', () => {
+    const now = new Date('2026-02-01T00:00:00.000Z')
+    const a = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 'AWSInsightsIndexService.GetCostAndUsage', body: '{}', now })
+    const b = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 'AWSInsightsIndexService.GetCostAndUsage', body: '{}', now })
+    expect(a.headers['authorization']).toBe(b.headers['authorization']) // deterministic for a fixed clock
+    expect(a.headers['authorization']).toMatch(
+      /^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\/20260201\/us-east-1\/ce\/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=[0-9a-f]{64}$/,
+    )
+    expect(a.headers['x-amz-date']).toBe('20260201T000000Z')
+    expect(a.url).toBe('https://ce.us-east-1.amazonaws.com/')
+  })
+  it('changes the signature when the secret changes', () => {
+    const now = new Date('2026-02-01T00:00:00.000Z')
+    const a = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 't', body: '{}', now })
+    const b = signAwsRequest({ creds: { ...CREDS, secretAccessKey: 'other' }, region: 'us-east-1', service: 'ce', target: 't', body: '{}', now })
+    expect(a.headers['authorization']).not.toBe(b.headers['authorization'])
+  })
+})
+
+const CE_RESPONSE = {
+  ResultsByTime: [{
+    Groups: [
+      { Keys: ['client$ACME'], Metrics: { UnblendedCost: { Amount: '1234.56', Unit: 'INR' } } },
+      { Keys: ['client$GHOST'], Metrics: { UnblendedCost: { Amount: '10.00', Unit: 'INR' } } },
+      { Keys: ['client$'], Metrics: { UnblendedCost: { Amount: '5.00', Unit: 'INR' } } },
+    ],
+  }],
+}
+
+function deps(response: unknown, capture?: (url: string, init?: RequestInit) => void): AwsCostDeps {
+  const f = (async (url: string, init?: RequestInit) => {
+    capture?.(String(url), init)
+    return new Response(JSON.stringify(response), { status: 200 })
+  }) as unknown as typeof fetch
+  return { f, creds: CREDS, now: () => '2026-07-01T00:00:00Z' }
+}
+
+describe('pullMonthlyCosts', () => {
+  it('maps client tags to clients, upserts cost rows, and collects unknown tags', async () => {
+    const db = openDb(':memory:'); seedIfEmpty(db)
+    const acme = createClient(db, 'u1', { name: 'Acme', code: 'ACME', stateCode: '32' })
+    let sentUrl = ''
+    const out = await pullMonthlyCosts(db, deps(CE_RESPONSE, (u) => { sentUrl = u }), '2026-06')
+    expect(sentUrl).toBe('https://ce.us-east-1.amazonaws.com/')
+    expect(out.upserted).toBe(1) // only ACME matched
+    expect(out.unknownTags.map((u) => u.tag).sort()).toEqual(['', 'GHOST']) // ghost client + untagged
+    expect(out.unknownTags.find((u) => u.tag === 'GHOST')?.costPaise).toBe(10_00)
+    const row = db.prepare(`SELECT cost_paise, source FROM aws_usage WHERE client_id=?`).get(acme.id)
+    expect(row).toMatchObject({ cost_paise: 123456, source: 'auto' })
+  })
+  it('throws on a non-INR billing currency rather than mis-converting', async () => {
+    const db = openDb(':memory:'); seedIfEmpty(db)
+    const usd = { ResultsByTime: [{ Groups: [{ Keys: ['client$ACME'], Metrics: { UnblendedCost: { Amount: '10.00', Unit: 'USD' } } }] }] }
+    await expect(pullMonthlyCosts(db, deps(usd), '2026-06')).rejects.toThrow(/INR/i)
+  })
+})
+
+describe('maybePullAwsCosts', () => {
+  it('pulls the previous month once per calendar month', async () => {
+    const db = openDb(':memory:'); seedIfEmpty(db)
+    createClient(db, 'u1', { name: 'Acme', code: 'ACME', stateCode: '32' })
+    expect(previousMonth('2026-07-10')).toBe('2026-06')
+    const first = await maybePullAwsCosts(db, deps(CE_RESPONSE), '2026-07-10')
+    expect(first).not.toBeNull()
+    expect(getSetting(db, 'aws.last_pull_month')).toBe('2026-06')
+    const second = await maybePullAwsCosts(db, deps(CE_RESPONSE), '2026-07-20')
+    expect(second).toBeNull() // already pulled 2026-06 this run
+  })
+})