feat(hq): aws cost explorer ingestion — sigv4 signer and monthly tag-grouped pull
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>feat/client-detail-redesign
Thomas Joise
2 weeks ago
parent
9d2eedc366
commit
51732df5e5
@ -0,0 +1,124 @@
|
||||
import { createHash, createHmac } from 'node:crypto'
|
||||
import type { DB } from './db'
|
||||
import type { Fetcher } from './gmail'
|
||||
import { getSetting, setSetting } from './repos-reminders'
|
||||
import { previousMonth, upsertAwsUsage } from './repos-aws'
|
||||
|
||||
/**
|
||||
* AWS Cost Explorer ingestion — SigV4 signed with node:crypto (no SDK), fetch
|
||||
* injected (Fetcher) so tests stub the network. GetCostAndUsage is grouped by
|
||||
* the cost-allocation tag 'client'; each tag value maps to a client by code and
|
||||
* upserts one aws_usage row per (client, month). Untagged/unknown tag values are
|
||||
* returned in the result, never dropped. Credentials arrive as a parameter —
|
||||
* the caller (server wiring / route) reads them from env, never this module.
|
||||
*/
|
||||
|
||||
export interface AwsCreds { accessKeyId: string; secretAccessKey: string }
|
||||
export interface SignedRequest { url: string; method: 'POST'; headers: Record<string, string>; body: string }
|
||||
|
||||
const REGION = 'us-east-1' // Cost Explorer is a global service pinned to us-east-1
|
||||
const SERVICE = 'ce'
|
||||
const TARGET = 'AWSInsightsIndexService.GetCostAndUsage'
|
||||
|
||||
function sha256Hex(data: string): string {
|
||||
return createHash('sha256').update(data, 'utf8').digest('hex')
|
||||
}
|
||||
function hmac(key: Buffer | string, data: string): Buffer {
|
||||
return createHmac('sha256', key).update(data, 'utf8').digest()
|
||||
}
|
||||
|
||||
/** Full SigV4: canonical request → string-to-sign → signing key → Authorization header. */
|
||||
export function signAwsRequest(args: {
|
||||
creds: AwsCreds; region: string; service: string; target: string; body: string; now: Date
|
||||
}): SignedRequest {
|
||||
const { creds, region, service, target, body, now } = args
|
||||
const host = `${service}.${region}.amazonaws.com`
|
||||
const amzDate = now.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') // 20260201T000000Z
|
||||
const dateStamp = amzDate.slice(0, 8) // 20260201
|
||||
const contentType = 'application/x-amz-json-1.1'
|
||||
|
||||
// Canonical request — headers lowercased and sorted, body SHA-256 hashed.
|
||||
const payloadHash = sha256Hex(body)
|
||||
const canonicalHeaders =
|
||||
`content-type:${contentType}\n` + `host:${host}\n` +
|
||||
`x-amz-date:${amzDate}\n` + `x-amz-target:${target}\n`
|
||||
const signedHeaders = 'content-type;host;x-amz-date;x-amz-target'
|
||||
const canonicalRequest = ['POST', '/', '', canonicalHeaders, signedHeaders, payloadHash].join('\n')
|
||||
|
||||
// String to sign.
|
||||
const scope = `${dateStamp}/${region}/${service}/aws4_request`
|
||||
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, sha256Hex(canonicalRequest)].join('\n')
|
||||
|
||||
// Derived signing key + signature.
|
||||
const kDate = hmac(`AWS4${creds.secretAccessKey}`, dateStamp)
|
||||
const kRegion = hmac(kDate, region)
|
||||
const kService = hmac(kRegion, service)
|
||||
const kSigning = hmac(kService, 'aws4_request')
|
||||
const signature = createHmac('sha256', kSigning).update(stringToSign, 'utf8').digest('hex')
|
||||
|
||||
const authorization =
|
||||
`AWS4-HMAC-SHA256 Credential=${creds.accessKeyId}/${scope}, ` +
|
||||
`SignedHeaders=${signedHeaders}, Signature=${signature}`
|
||||
|
||||
return {
|
||||
url: `https://${host}/`, method: 'POST', body,
|
||||
// 'host' is set by undici on the wire too; keeping it here documents what was signed.
|
||||
headers: { 'content-type': contentType, host, 'x-amz-date': amzDate, 'x-amz-target': target, authorization },
|
||||
}
|
||||
}
|
||||
|
||||
export interface AwsCostDeps { f: Fetcher; creds: AwsCreds; now?: () => string }
|
||||
export interface PullResult { upserted: number; unknownTags: { tag: string; costPaise: number }[]; totalPaise: number }
|
||||
|
||||
interface CeGroup { Keys?: string[]; Metrics?: { UnblendedCost?: { Amount?: string; Unit?: string } } }
|
||||
interface CeResponse { ResultsByTime?: { Groups?: CeGroup[] }[] }
|
||||
|
||||
/** '2026-06' → { start: '2026-06-01', end: '2026-07-01' } (End is exclusive in Cost Explorer). */
|
||||
function monthBounds(month: string): { start: string; end: string } {
|
||||
const [y, m] = month.split('-').map(Number)
|
||||
return { start: `${month}-01`, end: new Date(Date.UTC(y!, m!, 1)).toISOString().slice(0, 10) }
|
||||
}
|
||||
|
||||
export async function pullMonthlyCosts(db: DB, deps: AwsCostDeps, month: string): Promise<PullResult> {
|
||||
const { start, end } = monthBounds(month)
|
||||
const body = JSON.stringify({
|
||||
TimePeriod: { Start: start, End: end },
|
||||
Granularity: 'MONTHLY',
|
||||
Metrics: ['UnblendedCost'],
|
||||
GroupBy: [{ Type: 'TAG', Key: 'client' }],
|
||||
})
|
||||
const now = deps.now !== undefined ? new Date(deps.now()) : new Date()
|
||||
const signed = signAwsRequest({ creds: deps.creds, region: REGION, service: SERVICE, target: TARGET, body, now })
|
||||
const res = await deps.f(signed.url, { method: 'POST', headers: signed.headers, body: signed.body })
|
||||
const json = (await res.json().catch(() => ({}))) as CeResponse
|
||||
if (!res.ok) throw new Error(`Cost Explorer request failed: HTTP ${res.status}`)
|
||||
|
||||
const groups = json.ResultsByTime?.[0]?.Groups ?? []
|
||||
const unknownTags: { tag: string; costPaise: number }[] = []
|
||||
let upserted = 0
|
||||
let totalPaise = 0
|
||||
for (const g of groups) {
|
||||
const cost = g.Metrics?.UnblendedCost
|
||||
const unit = cost?.Unit ?? 'INR'
|
||||
if (unit !== 'INR') throw new Error(`Cost Explorer returned ${unit}, expected INR — HQ-3a assumes INR billing`)
|
||||
const tag = (g.Keys?.[0] ?? '').split('$')[1] ?? '' // 'client$acme' → 'acme'; 'client$' → ''
|
||||
const costPaise = Math.round(Number(cost?.Amount ?? '0') * 100)
|
||||
totalPaise += costPaise
|
||||
const client = tag === ''
|
||||
? undefined
|
||||
: db.prepare(`SELECT id FROM client WHERE lower(code)=lower(?)`).get(tag) as { id: string } | undefined
|
||||
if (client === undefined) { unknownTags.push({ tag, costPaise }); continue }
|
||||
upsertAwsUsage(db, 'system', { clientId: client.id, month, costPaise, source: 'auto' })
|
||||
upserted += 1
|
||||
}
|
||||
return { upserted, unknownTags, totalPaise }
|
||||
}
|
||||
|
||||
/** Gate the pull to once per calendar month; the 6-hourly tick calls this. */
|
||||
export async function maybePullAwsCosts(db: DB, deps: AwsCostDeps, today: string): Promise<PullResult | null> {
|
||||
const target = previousMonth(today)
|
||||
if (getSetting(db, 'aws.last_pull_month') === target) return null
|
||||
const out = await pullMonthlyCosts(db, deps, target)
|
||||
setSetting(db, 'system', 'aws.last_pull_month', target)
|
||||
return out
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
// apps/hq/test/aws-costs.test.ts
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { openDb } from '../src/db'
|
||||
import { seedIfEmpty } from '../src/seed'
|
||||
import { createClient } from '../src/repos-clients'
|
||||
import { getSetting } from '../src/repos-reminders'
|
||||
import { previousMonth } from '../src/repos-aws'
|
||||
import { signAwsRequest, pullMonthlyCosts, maybePullAwsCosts, type AwsCostDeps } from '../src/aws-costs'
|
||||
|
||||
const CREDS = { accessKeyId: 'AKIDEXAMPLE', secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' }
|
||||
|
||||
describe('signAwsRequest (SigV4)', () => {
|
||||
it('produces a deterministic, well-formed authorization header', () => {
|
||||
const now = new Date('2026-02-01T00:00:00.000Z')
|
||||
const a = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 'AWSInsightsIndexService.GetCostAndUsage', body: '{}', now })
|
||||
const b = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 'AWSInsightsIndexService.GetCostAndUsage', body: '{}', now })
|
||||
expect(a.headers['authorization']).toBe(b.headers['authorization']) // deterministic for a fixed clock
|
||||
expect(a.headers['authorization']).toMatch(
|
||||
/^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\/20260201\/us-east-1\/ce\/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=[0-9a-f]{64}$/,
|
||||
)
|
||||
expect(a.headers['x-amz-date']).toBe('20260201T000000Z')
|
||||
expect(a.url).toBe('https://ce.us-east-1.amazonaws.com/')
|
||||
})
|
||||
it('changes the signature when the secret changes', () => {
|
||||
const now = new Date('2026-02-01T00:00:00.000Z')
|
||||
const a = signAwsRequest({ creds: CREDS, region: 'us-east-1', service: 'ce', target: 't', body: '{}', now })
|
||||
const b = signAwsRequest({ creds: { ...CREDS, secretAccessKey: 'other' }, region: 'us-east-1', service: 'ce', target: 't', body: '{}', now })
|
||||
expect(a.headers['authorization']).not.toBe(b.headers['authorization'])
|
||||
})
|
||||
})
|
||||
|
||||
const CE_RESPONSE = {
|
||||
ResultsByTime: [{
|
||||
Groups: [
|
||||
{ Keys: ['client$ACME'], Metrics: { UnblendedCost: { Amount: '1234.56', Unit: 'INR' } } },
|
||||
{ Keys: ['client$GHOST'], Metrics: { UnblendedCost: { Amount: '10.00', Unit: 'INR' } } },
|
||||
{ Keys: ['client$'], Metrics: { UnblendedCost: { Amount: '5.00', Unit: 'INR' } } },
|
||||
],
|
||||
}],
|
||||
}
|
||||
|
||||
function deps(response: unknown, capture?: (url: string, init?: RequestInit) => void): AwsCostDeps {
|
||||
const f = (async (url: string, init?: RequestInit) => {
|
||||
capture?.(String(url), init)
|
||||
return new Response(JSON.stringify(response), { status: 200 })
|
||||
}) as unknown as typeof fetch
|
||||
return { f, creds: CREDS, now: () => '2026-07-01T00:00:00Z' }
|
||||
}
|
||||
|
||||
describe('pullMonthlyCosts', () => {
|
||||
it('maps client tags to clients, upserts cost rows, and collects unknown tags', async () => {
|
||||
const db = openDb(':memory:'); seedIfEmpty(db)
|
||||
const acme = createClient(db, 'u1', { name: 'Acme', code: 'ACME', stateCode: '32' })
|
||||
let sentUrl = ''
|
||||
const out = await pullMonthlyCosts(db, deps(CE_RESPONSE, (u) => { sentUrl = u }), '2026-06')
|
||||
expect(sentUrl).toBe('https://ce.us-east-1.amazonaws.com/')
|
||||
expect(out.upserted).toBe(1) // only ACME matched
|
||||
expect(out.unknownTags.map((u) => u.tag).sort()).toEqual(['', 'GHOST']) // ghost client + untagged
|
||||
expect(out.unknownTags.find((u) => u.tag === 'GHOST')?.costPaise).toBe(10_00)
|
||||
const row = db.prepare(`SELECT cost_paise, source FROM aws_usage WHERE client_id=?`).get(acme.id)
|
||||
expect(row).toMatchObject({ cost_paise: 123456, source: 'auto' })
|
||||
})
|
||||
it('throws on a non-INR billing currency rather than mis-converting', async () => {
|
||||
const db = openDb(':memory:'); seedIfEmpty(db)
|
||||
const usd = { ResultsByTime: [{ Groups: [{ Keys: ['client$ACME'], Metrics: { UnblendedCost: { Amount: '10.00', Unit: 'USD' } } }] }] }
|
||||
await expect(pullMonthlyCosts(db, deps(usd), '2026-06')).rejects.toThrow(/INR/i)
|
||||
})
|
||||
})
|
||||
|
||||
describe('maybePullAwsCosts', () => {
|
||||
it('pulls the previous month once per calendar month', async () => {
|
||||
const db = openDb(':memory:'); seedIfEmpty(db)
|
||||
createClient(db, 'u1', { name: 'Acme', code: 'ACME', stateCode: '32' })
|
||||
expect(previousMonth('2026-07-10')).toBe('2026-06')
|
||||
const first = await maybePullAwsCosts(db, deps(CE_RESPONSE), '2026-07-10')
|
||||
expect(first).not.toBeNull()
|
||||
expect(getSetting(db, 'aws.last_pull_month')).toBe('2026-06')
|
||||
const second = await maybePullAwsCosts(db, deps(CE_RESPONSE), '2026-07-20')
|
||||
expect(second).toBeNull() // already pulled 2026-06 this run
|
||||
})
|
||||
})
|
||||
Loading…
Reference in New Issue