pipework 0.251.0 → 0.252.1

package/src/infra/alerts/handle.ts

@@ -0,0 +1,778 @@
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { fitting } from '../../request/di/index.js'
+import { getContext } from '../../request/context/store.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import type { Queue } from '../../async/jobs/queue.js'
+import type { Handler } from '../../request/di/types.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import { auditAlert } from './audit.js'
+import { ALERT_SEVERITIES, ALERT_STATUSES } from './schema.js'
+import { encodeCursor, decodeCursor } from '../../request/fixture/paginate.js'
+import { createOutbox } from '../outbox/core.js'
+import type { AlertDelivery, AlertPayload, AlertSink } from './sinks.js'
+import type { AlertSource } from './sources.js'
+import { redactJson, redactText } from './redact.js'
+import { recordBeat, ALERTS_TICK_LOOP, DEFAULT_LOOP_EVERY_MS } from './beat.js'
+import { describeError } from '../../core/logging/error-chain.js'
+import { ValidationError } from '../../core/errors/index.js'
+
+export type AlertSeverity = 'info' | 'warning' | 'error' | 'critical'
+
+export interface AlertSubject {
+  readonly tenantId?: string
+  readonly entity?: string
+  readonly entityId?: string
+}
+
+export interface AlertsConfig {
+  /** Where alerts are sent. At least one; names must be unique. */
+  readonly sinks: readonly AlertSink[]
+  /** The queue the alerts job runs on; its worker surface must run `handle.job`. */
+  readonly queue: Pick<Queue, 'enqueue'>
+  /** Job type of `handle.job`; default `'pipework.alerts'`. */
+  readonly jobType?: string
+  /** Send again while an alert stays open and is raised again after this long since the last send. Default `null`: never. */
+  readonly resendAfterMs?: number | null
+  /** Resolve an open alert nobody raised for this long, on the job's tick. Default `null`: only `resolve`. */
+  readonly resolveAfterQuietMs?: number | null
+  /** Delivery attempts before a row is given up on; default 10. */
+  readonly maxAttempts?: number
+  /** First retry delay, doubling per attempt up to one hour; default 5000. */
+  readonly backoffMs?: number
+  /** How long an occurrence is kept; the tick deletes older ones. Default 30 days. */
+  readonly occurrenceRetentionMs?: number
+  /**
+   * Send alerts of these severities in one periodic `digest` delivery per sink instead of one at a time. A raise or
+   * resolve of such an alert queues nothing (so `resendAfterMs` has no effect on it); each tick at least `everyMs`
+   * after the last digest sends one naming those that changed since (the first, those changed in the last `everyMs`). `severities` default `['warning']`;
+   * `error` and `critical` always go one at a time. Default: no digest.
+   */
+  readonly digest?: { readonly everyMs: number; readonly severities?: readonly AlertSeverity[] }
+  /** The environment label on every delivery; default the raising Flow's `config.environment`. */
+  readonly environment?: string
+  readonly sources?: readonly AlertSource[]
+  /** Database holding the alert tables; default the default database. */
+  readonly database?: string
+  /** How often you schedule the tick; used to tell when it has stopped. Default 60_000. */
+  readonly tickEveryMs?: number
+  /**
+   * Email addresses in an alert's message and in the strings of its context become `[email]` before the alert is
+   * stored, recorded or sent. Each pattern here is replaced by `[redacted]` too (matched globally).
+   */
+  readonly redact?: readonly RegExp[]
+}
+
+export type AlertStatus = 'open' | 'acknowledged' | 'resolved'
+export type JsonValue = string | number | boolean | null | readonly JsonValue[] | { readonly [k: string]: JsonValue }
+
+export interface RaiseOptions {
+  readonly severity: AlertSeverity
+  readonly message: string
+  readonly subject?: AlertSubject
+  /** Which team most likely fixes it. Latest raise wins. */
+  readonly owner?: string
+  /** One line: what is safe to do. Latest raise wins. */
+  readonly runbook?: string
+  /** Ids and counts, stored on the alert (latest raise wins) and on the occurrence. A JSON object. */
+  readonly context?: Readonly<Record<string, JsonValue>>
+  /** Overrides the handle's `resendAfterMs` for this alert; `null`: this alert is never re-sent. */
+  readonly resendAfterMs?: number | null
+  /** Overrides the handle's `resolveAfterQuietMs` for this alert; `null`: only `resolve` closes it. */
+  readonly resolveAfterQuietMs?: number | null
+}
+
+export interface OwnerChange { readonly from: string | null; readonly to: string; readonly by: string; readonly at: Date }
+
+export interface Alert {
+  readonly key: string
+  readonly status: AlertStatus
+  readonly severity: AlertSeverity
+  readonly message: string
+  readonly subject: AlertSubject | null
+  readonly owner: string | null
+  /** Each change of owner made by acknowledge, oldest first. Kept across reopens. */
+  readonly ownerHistory: readonly OwnerChange[]
+  readonly runbook: string | null
+  readonly context: Readonly<Record<string, JsonValue>> | null
+  /** Raises since first seen, across reopens. */
+  readonly count: number
+  /** Times a raise came after a resolve. */
+  readonly reopenCount: number
+  readonly firstSeenAt: Date
+  readonly lastSeenAt: Date
+  readonly lastSentAt: Date | null
+  readonly acknowledgedAt: Date | null
+  readonly acknowledgedBy: string | null
+  readonly resolvedAt: Date | null
+  readonly resolvedBy: string | null
+  readonly resolveNote: string | null
+  /** The release the resolve names (e.g. the fix's version). */
+  readonly resolvedRelease: string | null
+  readonly reopenedAt: Date | null
+  readonly issueUrl: string | null
+  /** Latest ten traces that raised it, newest first. */
+  readonly traceIds: readonly string[]
+}
+
+export interface AlertOccurrence {
+  readonly id: string
+  readonly key: string
+  readonly at: Date
+  readonly severity: AlertSeverity
+  readonly message: string
+  readonly context: Readonly<Record<string, JsonValue>> | null
+  readonly traceId: string | null
+}
+
+export interface Page<T> { readonly items: T[]; readonly next: string | null }
+
+export interface AlertTotals {
+  /** Every severity, by status; zero counts included for all three statuses and four severities. */
+  readonly bySeverity: Record<AlertSeverity, Record<AlertStatus, number>>
+}
+
+export interface DeliveryHealth {
+  /** Undelivered, not given up. */
+  readonly pending: number
+  /** Given up (failed_at set). */
+  readonly failed: number
+  readonly lastDeliveredAt: Date | null
+  readonly lastFailedAt: Date | null
+  readonly bySink: readonly {
+    readonly sink: string; readonly pending: number; readonly failed: number
+    readonly lastDeliveredAt: Date | null; readonly lastFailedAt: Date | null; readonly lastError: string | null
+  }[]
+}
+
+export interface ResolveOptions {
+  readonly note?: string
+  readonly release?: string
+  readonly by?: string
+  /** Queue a `resolve` delivery; default true. */
+  readonly notify?: boolean
+}
+
+export interface DeliveryRow {
+  readonly id: string
+  readonly key: string
+  readonly kind: AlertPayload['kind']
+  readonly sink: string
+  readonly payload: AlertPayload
+  readonly attempts: number
+  readonly nextAttemptAt: Date
+  readonly deliveredAt: Date | null
+  readonly failedAt: Date | null
+  readonly lastError: string | null
+}
+
+export interface TickResult {
+  /** Alerts resolved because nobody raised them for their quiet period, or because a source cleared them. */
+  readonly resolved: number
+  readonly delivered: number
+  /** Attempts that failed and will be retried. */
+  readonly retried: number
+  /** Deliveries given up on. */
+  readonly failed: number
+}
+
+export interface Alerts {
+  /**
+   * Raises the condition `key`. New: opens it and sends once. After a resolve: reopens it (count continues,
+   * `reopenCount` + 1) and sends `reopen`. Open: groups it (count, last seen, trace id) and sends nothing, unless
+   * `resendAfterMs` has passed since the last send. Acknowledged: groups, never sends. Every raise writes an
+   * occurrence. Needs a Flow.
+   */
+  raise(key: string, options: RaiseOptions): Promise<{ status: 'opened' | 'grouped' | 'resent' | 'reopened' }>
+  /** Resolves an open or acknowledged alert and sends `resolve` once unless `notify` is false. Otherwise: false, nothing sent. */
+  resolve(key: string, options?: ResolveOptions): Promise<boolean>
+  /**
+   * open -> acknowledged. Not open: false. No delivery. `owner` replaces the stored owner and appends to `ownerHistory`
+   * (nothing is appended when it is the current owner). An acknowledged alert given a different `owner` changes the owner
+   * only, keeps who acknowledged it and when, and returns true; without `owner`, or the same one, it returns false.
+   */
+  acknowledge(key: string, options: { by: string; note?: string; owner?: string }): Promise<boolean>
+  /** Sets issueUrl on an alert in any status. Unknown key: false. No delivery. Must be an http(s) URL. */
+  link(key: string, url: string): Promise<boolean>
+  /** Newest `lastSeenAt` first, then key. `limit` default 50, at most 200. */
+  list(options?: {
+    status?: AlertStatus | readonly AlertStatus[]; severity?: AlertSeverity | readonly AlertSeverity[]
+    /** The current owner is one of these. */
+    owner?: readonly string[]; limit?: number; cursor?: string
+  }): Promise<Page<Alert>>
+  /** One alert's raises, newest first. `limit` default 50, at most 200. */
+  occurrences(key: string, options?: { limit?: number; cursor?: string }): Promise<Page<AlertOccurrence>>
+  totals(): Promise<AlertTotals>
+  health(): Promise<DeliveryHealth>
+  /** The job to add to a worker surface's `jobs` (and schedule, e.g. every minute): auto-resolve, sources, deliveries. */
+  readonly job: Handler<Record<string, unknown>, unknown>
+  /** What one run of `job` does; callable from a test. Needs a Flow. */
+  tick(): Promise<TickResult>
+  open(): Promise<Alert[]>
+  get(key: string): Promise<Alert | null>
+  deliveries(options?: { key?: string }): Promise<DeliveryRow[]>
+  /**
+   * Puts the deliveries that were given up (and never delivered) back on the queue, optionally only those of `sink`
+   * and/or that failed at or after `failedSince`; the next `tick` sends them. Delivery ids stay the same, so a receiver
+   * that dedupes on `delivery.id` takes each once. Writes a `requeue` audit row per delivery naming `by` (who asked:
+   * an operator or key name, required).
+   */
+  requeueFailed(options: { by: string; sink?: string; failedSince?: Date }): Promise<{ requeued: number }>
+}
+
+const SEVERITIES = new Set<string>(ALERT_SEVERITIES)
+const STATUSES = new Set<string>(ALERT_STATUSES)
+const DEFAULT_RETENTION_MS = 30 * 24 * 3_600_000
+const RETENTION_BATCH = 10_000
+const DEFAULT_LIMIT = 50
+const MAX_LIMIT = 200
+/** A row's time as text, UTC to the microsecond; a JS Date holds only milliseconds. */
+const exact = (column: string) => sql.raw(`to_char(${column} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')`)
+
+function toAlert(row: Record<string, unknown>): Alert {
+  const date = (v: unknown): Date => new Date(v as string)
+  const dateOrNull = (v: unknown): Date | null => (v === null || v === undefined ? null : date(v))
+  return {
+    key: row['key'] as string,
+    status: row['status'] as Alert['status'],
+    severity: row['severity'] as AlertSeverity,
+    message: row['message'] as string,
+    subject: (row['subject'] ?? null) as AlertSubject | null,
+    owner: (row['owner'] ?? null) as string | null,
+    ownerHistory: ((row['owner_history'] ?? []) as { from: string | null; to: string; by: string; at: string }[])
+      .map(h => ({ from: h.from, to: h.to, by: h.by, at: date(h.at) })),
+    runbook: (row['runbook'] ?? null) as string | null,
+    context: (row['context'] ?? null) as Alert['context'],
+    count: Number(row['count']),
+    reopenCount: Number(row['reopen_count']),
+    firstSeenAt: date(row['first_seen_at']),
+    lastSeenAt: date(row['last_seen_at']),
+    lastSentAt: dateOrNull(row['last_sent_at']),
+    acknowledgedAt: dateOrNull(row['acknowledged_at']),
+    acknowledgedBy: (row['acknowledged_by'] ?? null) as string | null,
+    resolvedAt: dateOrNull(row['resolved_at']),
+    resolvedBy: (row['resolved_by'] ?? null) as string | null,
+    resolveNote: (row['resolve_note'] ?? null) as string | null,
+    resolvedRelease: (row['resolved_release'] ?? null) as string | null,
+    reopenedAt: dateOrNull(row['reopened_at']),
+    issueUrl: (row['issue_url'] ?? null) as string | null,
+    traceIds: (row['trace_ids'] ?? []) as string[],
+  }
+}
+
+function toDeliveryRow(row: Record<string, unknown>): DeliveryRow {
+  return {
+    id: row['id'] as string,
+    key: row['alert_key'] as string,
+    kind: row['kind'] as DeliveryRow['kind'],
+    sink: row['sink'] as string,
+    payload: row['payload'] as AlertPayload,
+    attempts: Number(row['attempts']),
+    nextAttemptAt: new Date(row['next_attempt_at'] as string),
+    deliveredAt: row['delivered_at'] === null ? null : new Date(row['delivered_at'] as string),
+    failedAt: row['failed_at'] === null ? null : new Date(row['failed_at'] as string),
+    lastError: (row['last_error'] ?? null) as string | null,
+  }
+}
+
+function toOccurrence(row: Record<string, unknown>): AlertOccurrence {
+  return {
+    id: row['id'] as string,
+    key: row['alert_key'] as string,
+    at: new Date(row['at'] as string),
+    severity: row['severity'] as AlertSeverity,
+    message: row['message'] as string,
+    context: (row['context'] ?? null) as AlertOccurrence['context'],
+    traceId: (row['trace_id'] ?? null) as string | null,
+  }
+}
+
+function checkLimit(fn: string, limit: number | undefined): number {
+  if (limit === undefined) return DEFAULT_LIMIT
+  if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) {
+    throw new Error(`[pipework] alerts.${fn}: limit must be a whole number from 1 to ${String(MAX_LIMIT)}, got ${String(limit)}.`)
+  }
+  return limit
+}
+
+/** Reads `[time, tiebreak]` from a cursor this module encoded; anything else is refused by name. */
+function readCursor(fn: string, cursor: string): [string, string] {
+  const bad = (): Error => new Error(`[pipework] alerts.${fn}: cursor is not one this handle returned. Pass the \`next\` value unchanged.`)
+  let parts: unknown[]
+  try { parts = decodeCursor(cursor) } catch { throw bad() }
+  const [at, tie] = parts
+  if (parts.length !== 2 || typeof at !== 'string' || typeof tie !== 'string' || Number.isNaN(new Date(at).getTime())) throw bad()
+  return [at, tie]
+}
+
+function pickList<T extends string>(fn: string, name: string, value: T | readonly T[] | undefined, allowed: Set<string>): T[] | null {
+  if (value === undefined) return null
+  const list = typeof value === 'string' ? [value] : [...value]
+  for (const v of list) {
+    if (!allowed.has(v)) throw new Error(`[pipework] alerts.${fn}: ${name} must be one of ${[...allowed].join(', ')}, got "${String(v)}".`)
+  }
+  return list
+}
+
+function pgTextArray(values: readonly string[]): string {
+  return `{${values.map(v => `"${v.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`).join(',')}}`
+}
+
+export function createAlerts(config: AlertsConfig): Alerts {
+  if (config.sinks.length === 0) throw new Error('[pipework] alerts.create needs at least one sink.')
+  const sinks = new Map<string, AlertSink>()
+  for (const s of config.sinks) {
+    if (sinks.has(s.name)) throw new Error(`[pipework] alerts.create: two sinks are named "${s.name}"; give each its own name.`)
+    sinks.set(s.name, s)
+  }
+  const sinkNames = pgTextArray([...sinks.keys()])
+  const digestEveryMs = config.digest?.everyMs ?? null
+  if (config.digest !== undefined) {
+    if (!Number.isInteger(config.digest.everyMs) || config.digest.everyMs < 1000) {
+      throw new Error(`[pipework] alerts.create: digest.everyMs must be a whole number of at least 1000, got ${String(config.digest.everyMs)}.`)
+    }
+    for (const v of config.digest.severities ?? []) {
+      if (v === 'error' || v === 'critical') throw new Error(`[pipework] alerts.create: digest.severities cannot include "${v}"; errors and critical go one at a time.`)
+      if (!SEVERITIES.has(v)) throw new Error(`[pipework] alerts.create: digest.severities must be from info, warning, got "${String(v)}".`)
+    }
+    if (config.digest.severities?.length === 0) throw new Error('[pipework] alerts.create: digest.severities needs at least one severity, or leave it out for warning.')
+  }
+  const digestSeverities = pgTextArray(config.digest === undefined ? [] : [...new Set(config.digest.severities ?? ['warning'])])
+  const jobType = config.jobType ?? 'pipework.alerts'
+  const maxAttempts = config.maxAttempts ?? 10
+  const backoffMs = config.backoffMs ?? 5000
+  const defaultResend = config.resendAfterMs ?? null
+  const defaultQuiet = config.resolveAfterQuietMs ?? null
+  const sources = config.sources ?? []
+  const database = config.database
+  const redactExtra = config.redact ?? []
+  const log = getBaseLogger()
+
+  const db = () => pipeSystem(database)
+
+  const retentionMs = config.occurrenceRetentionMs ?? DEFAULT_RETENTION_MS
+  const environmentOf = (): string => config.environment ?? getContext()?.instance.config.environment ?? 'unknown'
+
+  // `null` is stored as -1 ("never"); undefined as NULL ("the handle's default").
+  function storedMs(name: string, value: number | null | undefined): number | null {
+    if (value === undefined) return null
+    if (value === null) return -1
+    if (!Number.isInteger(value) || value < 0) throw new Error(`[pipework] alerts.raise: ${name} must be a non-negative integer or null, got ${String(value)}.`)
+    return value
+  }
+
+  // The alert and its deliveries move in one statement, so a crash cannot leave one without the other.
+  const payloadSql = (kind: ReturnType<typeof sql>, environment: string) => sql`jsonb_build_object(
+    'kind', ${kind}, 'key', up.key, 'severity', up.severity, 'message', up.message, 'subject', up.subject,
+    'owner', up.owner, 'runbook', up.runbook, 'context', up.context,
+    'count', up.count, 'reopenCount', up.reopen_count, 'firstSeenAt', up.first_seen_at, 'lastSeenAt', up.last_seen_at,
+    'resolvedAt', up.resolved_at, 'acknowledgedAt', up.acknowledged_at, 'acknowledgedBy', up.acknowledged_by,
+    'resolvedBy', up.resolved_by, 'resolveNote', up.resolve_note, 'resolvedRelease', up.resolved_release,
+    'issueUrl', up.issue_url, 'environment', ${environment}::text, 'traceIds', to_jsonb(up.trace_ids))`
+
+  async function enqueueJob(): Promise<void> {
+    await config.queue.enqueue(db(), { type: jobType, payload: {}, continueFlow: true })
+  }
+
+  // Inside a tick the deliveries are sent by that same tick, so it does not enqueue another.
+  async function raise(key: string, options: RaiseOptions, enqueue = true): Promise<{ status: 'opened' | 'grouped' | 'resent' | 'reopened' }> {
+    if (key.trim() === '') throw new Error('[pipework] alerts.raise needs a non-empty key.')
+    if (!SEVERITIES.has(options.severity)) throw new Error(`[pipework] alerts.raise: severity must be info, warning, error or critical, got "${String(options.severity)}".`)
+    const resendStored = storedMs('resendAfterMs', options.resendAfterMs)
+    const quietStored = storedMs('resolveAfterQuietMs', options.resolveAfterQuietMs)
+    const traceId = getContext()?.trace?.id ?? null
+    const traceArray = traceId === null ? '{}' : pgTextArray([traceId])
+    const subject = options.subject === undefined ? null : JSON.stringify(options.subject)
+    const context = options.context === undefined ? null : JSON.stringify(redactJson(options.context, redactExtra))
+    const message = redactText(options.message, redactExtra)
+    const environment = environmentOf()
+    // `last_seen_at` never moves back. A raise sends exactly when this statement sets `last_sent_at`, and then sets
+    // `last_seen_at` to the same value; a raise that does not send never leaves them equal, so RETURNING can tell the two apart.
+    const seenSql = sql`GREATEST(a.last_seen_at, EXCLUDED.last_seen_at)`
+    const sendsSql = sql`(a.status <> 'acknowledged' AND (a.status = 'resolved' OR a.last_sent_at IS NULL
+            OR (NULLIF(COALESCE(EXCLUDED.resend_after_ms, a.resend_after_ms, ${defaultResend}::int), -1) IS NOT NULL
+                AND EXCLUDED.last_sent_at - a.last_sent_at
+                    >= NULLIF(COALESCE(EXCLUDED.resend_after_ms, a.resend_after_ms, ${defaultResend}::int), -1) * interval '1 millisecond')))`
+    const kindSql = sql`CASE WHEN up.inserted THEN 'raise' WHEN up.reopened THEN 'reopen' ELSE 'repeat' END`
+    const rows = await db().execute(sql`
+      WITH up AS (
+        INSERT INTO pipework_alerts AS a
+          (key, status, severity, message, subject, owner, runbook, context, count, first_seen_at, last_seen_at, last_sent_at,
+           trace_ids, resend_after_ms, resolve_after_quiet_ms, updated_at)
+        SELECT ${key}, 'open', ${options.severity}, ${message}, ${subject}::jsonb,
+               ${options.owner ?? null}::text, ${options.runbook ?? null}::text, ${context}::jsonb, 1, c.ts, c.ts, c.ts,
+               ${traceArray}::text[], ${resendStored}::int, ${quietStored}::int, c.ts
+        FROM (SELECT clock_timestamp() AS ts) c
+        ON CONFLICT (key) DO UPDATE SET
+          status = CASE WHEN a.status = 'acknowledged' THEN 'acknowledged' ELSE 'open' END,
+          severity = EXCLUDED.severity,
+          message = EXCLUDED.message,
+          subject = EXCLUDED.subject,
+          owner = CASE WHEN jsonb_array_length(a.owner_history) > 0 THEN a.owner ELSE EXCLUDED.owner END,
+          runbook = EXCLUDED.runbook,
+          context = EXCLUDED.context,
+          count = a.count + 1,
+          reopen_count = CASE WHEN a.status = 'resolved' THEN a.reopen_count + 1 ELSE a.reopen_count END,
+          reopened_at = CASE WHEN a.status = 'resolved' THEN ${seenSql} ELSE a.reopened_at END,
+          acknowledged_at = CASE WHEN a.status = 'resolved' THEN NULL ELSE a.acknowledged_at END,
+          acknowledged_by = CASE WHEN a.status = 'resolved' THEN NULL ELSE a.acknowledged_by END,
+          last_seen_at = CASE WHEN ${sendsSql} OR ${seenSql} IS DISTINCT FROM a.last_sent_at THEN ${seenSql}
+                              ELSE ${seenSql} + interval '1 microsecond' END,
+          resolved_at = NULL,
+          trace_ids = CASE
+            WHEN a.status = 'resolved' THEN EXCLUDED.trace_ids
+            WHEN ${traceId}::text IS NULL THEN a.trace_ids
+            ELSE (EXCLUDED.trace_ids || array_remove(a.trace_ids, ${traceId}::text))[1:10] END,
+          resend_after_ms = COALESCE(EXCLUDED.resend_after_ms, CASE WHEN a.status = 'resolved' THEN NULL ELSE a.resend_after_ms END),
+          resolve_after_quiet_ms = COALESCE(EXCLUDED.resolve_after_quiet_ms, CASE WHEN a.status = 'resolved' THEN NULL ELSE a.resolve_after_quiet_ms END),
+          last_sent_at = CASE WHEN ${sendsSql} THEN ${seenSql} ELSE a.last_sent_at END,
+          updated_at = EXCLUDED.updated_at
+        RETURNING a.*, (a.last_sent_at = a.last_seen_at) AS sent, (xmax = 0) AS inserted,
+                  (a.reopened_at IS NOT NULL AND a.reopened_at = a.last_seen_at) AS reopened
+      ),
+      occ AS (
+        INSERT INTO pipework_alert_occurrences (alert_key, at, severity, message, context, trace_id)
+        SELECT up.key, up.last_seen_at, up.severity, up.message, up.context, ${traceId}::text FROM up
+      ),
+      dels AS (
+        INSERT INTO pipework_alert_deliveries (alert_key, kind, sink, payload, created_at, next_attempt_at)
+        SELECT up.key, ${kindSql}, s.sink,
+               ${payloadSql(kindSql, environment)}, clock_timestamp(), clock_timestamp()
+        FROM up CROSS JOIN unnest(${sinkNames}::text[]) AS s(sink)
+        WHERE up.sent AND NOT up.severity = ANY(${digestSeverities}::text[])
+        RETURNING id
+      )
+      SELECT up.*, (SELECT count(*) FROM dels)::int AS queued FROM up`)
+    const row = rows[0]!
+    const alert = toAlert(row)
+    const sent = row['sent'] === true
+    const status = !sent ? 'grouped' : row['inserted'] === true ? 'opened' : row['reopened'] === true ? 'reopened' : 'resent'
+    await auditAlert(status === 'opened' ? 'raise' : status === 'reopened' ? 'reopen' : 'group', key, {
+      status, severity: alert.severity, count: alert.count, message: alert.message,
+    }, database)
+    if (sent && enqueue && Number(row['queued']) > 0) await enqueueJob()
+    return { status }
+  }
+
+  async function resolveKey(key: string, onlyIfQuiet: boolean, enqueue = true, options: ResolveOptions = {}): Promise<boolean> {
+    const notify = options.notify ?? true
+    const environment = environmentOf()
+    const rows = await db().execute(sql`
+      WITH up AS (
+        UPDATE pipework_alerts SET status = 'resolved', resolved_at = clock_timestamp(), updated_at = clock_timestamp(),
+          resolved_by = ${onlyIfQuiet ? sql`'quiet'` : sql`${options.by ?? null}::text`},
+          resolve_note = ${onlyIfQuiet ? sql`'quiet for ' || COALESCE(resolve_after_quiet_ms, ${defaultQuiet}::int) || ' ms'` : sql`${options.note ?? null}::text`},
+          resolved_release = ${options.release ?? null}::text
+        WHERE key = ${key} AND status IN ('open', 'acknowledged')
+          AND (NOT ${onlyIfQuiet} OR (resolve_after_quiet_ms IS DISTINCT FROM -1 AND last_seen_at < clock_timestamp()
+               - COALESCE(resolve_after_quiet_ms, ${defaultQuiet}::int) * interval '1 millisecond'))
+        RETURNING *
+      ),
+      dels AS (
+        INSERT INTO pipework_alert_deliveries (alert_key, kind, sink, payload, created_at, next_attempt_at)
+        SELECT up.key, 'resolve', s.sink, ${payloadSql(sql`'resolve'`, environment)}, clock_timestamp(), clock_timestamp()
+        FROM up CROSS JOIN unnest(${sinkNames}::text[]) AS s(sink)
+        WHERE ${notify}::boolean AND NOT up.severity = ANY(${digestSeverities}::text[])
+        RETURNING id
+      )
+      SELECT up.*, (SELECT count(*) FROM dels)::int AS queued FROM up`)
+    if (rows.length === 0) return false
+    const alert = toAlert(rows[0]!)
+    await auditAlert('resolve', key, { count: alert.count, auto: onlyIfQuiet, ...(onlyIfQuiet ? { by: 'quiet' } : options.by !== undefined ? { by: options.by } : {}) }, database)
+    if (enqueue && notify && Number(rows[0]!['queued']) > 0) await enqueueJob()
+    return true
+  }
+
+  async function acknowledge(key: string, options: { by: string; note?: string; owner?: string }): Promise<boolean> {
+    if (options.by.trim() === '') throw new Error('[pipework] alerts.acknowledge needs a non-empty by.')
+    if (options.owner !== undefined && options.owner.trim() === '') throw new Error('[pipework] alerts.acknowledge needs a non-empty owner when one is given.')
+    const owner = options.owner?.trim() ?? null
+    const rows = await db().execute(sql`
+      UPDATE pipework_alerts SET
+        status = 'acknowledged',
+        acknowledged_at = CASE WHEN status = 'open' THEN clock_timestamp() ELSE acknowledged_at END,
+        acknowledged_by = CASE WHEN status = 'open' THEN ${options.by} ELSE acknowledged_by END,
+        owner = COALESCE(${owner}::text, owner),
+        owner_history = CASE WHEN ${owner}::text IS NOT NULL AND ${owner}::text IS DISTINCT FROM owner
+          THEN owner_history || jsonb_build_array(jsonb_build_object('from', owner, 'to', ${owner}::text, 'by', ${options.by}::text,
+            'at', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')))
+          ELSE owner_history END,
+        updated_at = clock_timestamp()
+      WHERE key = ${key}
+        AND (status = 'open' OR (status = 'acknowledged' AND ${owner}::text IS NOT NULL AND ${owner}::text IS DISTINCT FROM owner))
+      RETURNING key`)
+    if (rows.length === 0) return false
+    await auditAlert('acknowledge', key, {
+      by: options.by, ...(options.note !== undefined ? { note: options.note } : {}), ...(owner !== null ? { owner } : {}),
+    }, database)
+    return true
+  }
+
+  async function link(key: string, url: string): Promise<boolean> {
+    let parsed: URL
+    try { parsed = new URL(url) } catch { throw new Error(`[pipework] alerts.link: url must be an http or https URL, got "${url}".`) }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+      throw new Error(`[pipework] alerts.link: url must be an http or https URL, got "${url}".`)
+    }
+    const rows = await db().execute(sql`UPDATE pipework_alerts SET issue_url = ${url}, updated_at = clock_timestamp() WHERE key = ${key} RETURNING key`)
+    if (rows.length === 0) return false
+    await auditAlert('link', key, { url }, database)
+    return true
+  }
+
+  async function list(options: Parameters<Alerts['list']>[0] = {}): Promise<Page<Alert>> {
+    const statuses = pickList('list', 'status', options.status, STATUSES)
+    const severities = pickList('list', 'severity', options.severity, SEVERITIES)
+    const owners = options.owner ?? null
+    const limit = checkLimit('list', options.limit)
+    const cursor = options.cursor === undefined ? null : readCursor('list', options.cursor)
+    const rows = await db().execute(sql`
+      SELECT *, ${exact('last_seen_at')} AS exact_at FROM pipework_alerts
+      WHERE (${statuses === null}::boolean OR status = ANY(${pgTextArray(statuses ?? [])}::text[]))
+        AND (${severities === null}::boolean OR severity = ANY(${pgTextArray(severities ?? [])}::text[]))
+        AND (${owners === null}::boolean OR owner = ANY(${pgTextArray([...(owners ?? [])])}::text[]))
+        AND (${cursor === null}::boolean OR last_seen_at < ${cursor?.[0] ?? null}::timestamptz
+             OR (last_seen_at = ${cursor?.[0] ?? null}::timestamptz AND key > ${cursor?.[1] ?? null}::text))
+      ORDER BY last_seen_at DESC, key ASC
+      LIMIT ${limit + 1}`)
+    const page = rows.slice(0, limit)
+    const last = page[page.length - 1]
+    return {
+      items: page.map(toAlert),
+      next: rows.length > limit && last !== undefined ? encodeCursor([last['exact_at'], last['key']]) : null,
+    }
+  }
+
+  async function occurrences(key: string, options: { limit?: number; cursor?: string } = {}): Promise<Page<AlertOccurrence>> {
+    const limit = checkLimit('occurrences', options.limit)
+    const cursor = options.cursor === undefined ? null : readCursor('occurrences', options.cursor)
+    const rows = await db().execute(sql`
+      SELECT *, ${exact('at')} AS exact_at FROM pipework_alert_occurrences
+      WHERE alert_key = ${key}
+        AND (${cursor === null}::boolean OR at < ${cursor?.[0] ?? null}::timestamptz
+             OR (at = ${cursor?.[0] ?? null}::timestamptz AND id < ${cursor?.[1] ?? null}::uuid))
+      ORDER BY at DESC, id DESC
+      LIMIT ${limit + 1}`)
+    const page = rows.slice(0, limit)
+    const last = page[page.length - 1]
+    return {
+      items: page.map(toOccurrence),
+      next: rows.length > limit && last !== undefined ? encodeCursor([last['exact_at'], last['id']]) : null,
+    }
+  }
+
+  async function totals(): Promise<AlertTotals> {
+    const rows = await db().execute(sql`SELECT severity, status, count(*)::int AS n FROM pipework_alerts GROUP BY severity, status`)
+    const bySeverity = {} as Record<AlertSeverity, Record<AlertStatus, number>>
+    for (const sev of ALERT_SEVERITIES) bySeverity[sev] = { open: 0, acknowledged: 0, resolved: 0 }
+    for (const row of rows) bySeverity[row['severity'] as AlertSeverity][row['status'] as AlertStatus] = Number(row['n'])
+    return { bySeverity }
+  }
+
+  async function health(): Promise<DeliveryHealth> {
+    const rows = await db().execute(sql`
+      SELECT sink,
+        count(*) FILTER (WHERE delivered_at IS NULL AND failed_at IS NULL)::int AS pending,
+        count(*) FILTER (WHERE failed_at IS NOT NULL)::int AS failed,
+        max(delivered_at) AS last_delivered_at,
+        max(failed_at) AS last_failed_at,
+        (array_agg(last_error ORDER BY COALESCE(failed_at, next_attempt_at) DESC) FILTER (WHERE last_error IS NOT NULL))[1] AS last_error
+      FROM pipework_alert_deliveries GROUP BY sink`)
+    const toDate = (v: unknown): Date | null => (v === null || v === undefined ? null : new Date(v as string))
+    const bySink = new Map<string, DeliveryHealth['bySink'][number]>()
+    for (const name of sinks.keys()) {
+      bySink.set(name, { sink: name, pending: 0, failed: 0, lastDeliveredAt: null, lastFailedAt: null, lastError: null })
+    }
+    let pending = 0
+    let failed = 0
+    let lastDeliveredAt: Date | null = null
+    let lastFailedAt: Date | null = null
+    const later = (a: Date | null, b: Date | null): Date | null => (a === null ? b : b === null ? a : a > b ? a : b)
+    for (const row of rows) {
+      const entry = {
+        sink: row['sink'] as string, pending: Number(row['pending']), failed: Number(row['failed']),
+        lastDeliveredAt: toDate(row['last_delivered_at']), lastFailedAt: toDate(row['last_failed_at']),
+        lastError: (row['last_error'] ?? null) as string | null,
+      }
+      bySink.set(entry.sink, entry)
+      pending += entry.pending
+      failed += entry.failed
+      lastDeliveredAt = later(lastDeliveredAt, entry.lastDeliveredAt)
+      lastFailedAt = later(lastFailedAt, entry.lastFailedAt)
+    }
+    return { pending, failed, lastDeliveredAt, lastFailedAt, bySink: [...bySink.values()].sort((x, y) => x.sink.localeCompare(y.sink)) }
+  }
+
+  async function pruneOccurrences(): Promise<void> {
+    await db().execute(sql`
+      DELETE FROM pipework_alert_occurrences WHERE id IN (
+        SELECT id FROM pipework_alert_occurrences
+        WHERE at < clock_timestamp() - ${retentionMs}::bigint * interval '1 millisecond' LIMIT ${RETENTION_BATCH})`)
+    await db().execute(sql`
+      DELETE FROM pipework_faults WHERE id IN (
+        SELECT id FROM pipework_faults
+        WHERE recorded_at < clock_timestamp() - ${retentionMs}::bigint * interval '1 millisecond' LIMIT ${RETENTION_BATCH})`)
+  }
+
+  async function autoResolve(): Promise<number> {
+    if (defaultQuiet === null) {
+      const any = await db().execute(sql`SELECT 1 FROM pipework_alerts WHERE status IN ('open', 'acknowledged') AND resolve_after_quiet_ms IS NOT NULL AND resolve_after_quiet_ms <> -1 LIMIT 1`)
+      if (any.length === 0) return 0
+    }
+    const due = await db().execute(sql`
+      SELECT key FROM pipework_alerts
+      WHERE status IN ('open', 'acknowledged')
+        AND resolve_after_quiet_ms IS DISTINCT FROM -1
+        AND last_seen_at < clock_timestamp() - COALESCE(resolve_after_quiet_ms, ${defaultQuiet}::int) * interval '1 millisecond'`)
+    let resolved = 0
+    for (const row of due) if (await resolveKey(row['key'] as string, true, false)) resolved++
+    return resolved
+  }
+
+  async function runSources(): Promise<number> {
+    let resolved = 0
+    for (const source of sources) {
+      let conditions
+      try {
+        conditions = await source.evaluate()
+      } catch (err) {
+        log.error({ err, source: source.name }, 'alert source failed')
+        try {
+          await raise(`alert-source-failed:${source.name}`, {
+            severity: 'error',
+            message: describeError(err).message,
+          }, false)
+        } catch (raiseErr) {
+          log.error({ err: raiseErr, source: source.name }, 'alert source failure could not be raised')
+        }
+        continue
+      }
+      if (await resolveKey(`alert-source-failed:${source.name}`, false, false)) resolved++
+      for (const c of conditions) {
+        await raise(c.key, {
+          severity: c.severity,
+          message: c.message,
+          ...(c.subject !== undefined ? { subject: c.subject } : {}),
+          ...(c.owner !== undefined ? { owner: c.owner } : {}),
+          ...(c.runbook !== undefined ? { runbook: c.runbook } : {}),
+          ...(c.context !== undefined ? { context: c.context } : {}),
+        }, false)
+      }
+      if (source.owns === undefined) continue
+      const live = new Set(conditions.map(c => c.key))
+      const opened = await db().execute(sql`SELECT key FROM pipework_alerts WHERE status IN ('open', 'acknowledged')`)
+      for (const row of opened) {
+        const key = row['key'] as string
+        if (source.owns(key) && !live.has(key) && await resolveKey(key, false, false)) resolved++
+      }
+    }
+    return resolved
+  }
+
+  const DIGEST_KEY = 'pipework.alerts.digest'
+
+  // One statement claims the window (only the handle that moves the cursor wins) and queues the digest.
+  async function sendDigest(): Promise<void> {
+    if (digestEveryMs === null) return
+    const environment = environmentOf()
+    const rows = await db().execute(sql`
+      WITH old AS (SELECT cursor_at FROM pipework_alert_source_state WHERE name = ${DIGEST_KEY}),
+      claim AS (
+        INSERT INTO pipework_alert_source_state AS s (name, cursor_at) VALUES (${DIGEST_KEY}, statement_timestamp())
+        ON CONFLICT (name) DO UPDATE SET cursor_at = EXCLUDED.cursor_at
+          WHERE s.cursor_at <= clock_timestamp() - ${digestEveryMs}::bigint * interval '1 millisecond'
+        RETURNING cursor_at AS until
+      ),
+      changed AS (
+        SELECT a.* FROM pipework_alerts a, claim
+        WHERE a.severity = ANY(${digestSeverities}::text[])
+          AND a.updated_at > COALESCE((SELECT cursor_at FROM old), statement_timestamp() - ${digestEveryMs}::bigint * interval '1 millisecond')
+      ),
+      dels AS (
+        INSERT INTO pipework_alert_deliveries (alert_key, kind, sink, payload, created_at, next_attempt_at)
+        SELECT ${DIGEST_KEY}, 'digest', s.sink,
+               jsonb_build_object('kind', 'digest', 'key', ${DIGEST_KEY}::text, 'environment', ${environment}::text,
+                 'since', (SELECT cursor_at FROM old), 'until', claim.until,
+                 'alerts', (SELECT jsonb_agg(jsonb_build_object('key', c.key, 'severity', c.severity, 'status', c.status,
+                   'message', c.message, 'owner', c.owner, 'count', c.count, 'reopenCount', c.reopen_count,
+                   'lastSeenAt', c.last_seen_at) ORDER BY c.key) FROM changed c)),
+               clock_timestamp(), clock_timestamp()
+        FROM claim CROSS JOIN unnest(${sinkNames}::text[]) AS s(sink)
+        WHERE EXISTS (SELECT 1 FROM changed)
+        RETURNING id
+      )
+      SELECT (SELECT count(*) FROM claim)::int AS claimed, (SELECT count(*) FROM dels)::int AS queued,
+             (SELECT coalesce(array_agg(key ORDER BY key), '{}') FROM changed) AS keys`)
+    const row = rows[0]!
+    if (Number(row['queued']) > 0) await auditAlert('digest', DIGEST_KEY, { keys: row['keys'] }, database)
+  }
+
+  const outbox = createOutbox<AlertDelivery>({
+    table: 'pipework_alert_deliveries', groupColumn: 'alert_key', database, sinks, maxAttempts, backoffMs,
+    label: 'alert delivery given up',
+    toDelivery: (row, attempt) => ({
+      id: row['id'] as string, kind: row['kind'] as AlertDelivery['kind'], key: row['alert_key'] as string,
+      sink: row['sink'] as string, payload: row['payload'] as AlertPayload, attempt,
+    }),
+    onDelivered: (row, attempt) => auditAlert('deliver', row['alert_key'] as string, { sink: row['sink'], kind: row['kind'], attempt }, database),
+    onGivenUp: (row, attempt, error) => auditAlert('deliver-failed', row['alert_key'] as string, { sink: row['sink'], kind: row['kind'], attempt, error }, database),
+  })
+
+  async function runTick(): Promise<TickResult> {
+    let resolved = await autoResolve()
+    resolved += await runSources()
+    await pruneOccurrences()
+    await sendDigest()
+    return { resolved, ...await outbox.drain() }
+  }
+
+  const flowTick = __flowStep({ ref: 'pipework/alerts:tick' }, runTick)
+  const tickEveryMs = config.tickEveryMs ?? DEFAULT_LOOP_EVERY_MS
+  const tickStep = async (): Promise<TickResult> => {
+    try {
+      const result = await flowTick()
+      await recordBeat(db, ALERTS_TICK_LOOP, { ok: true, everyMs: tickEveryMs })
+      return result
+    } catch (err) {
+      await recordBeat(db, ALERTS_TICK_LOOP, { ok: false, error: describeError(err).message, everyMs: tickEveryMs })
+      throw err
+    }
+  }
+  const job = (database !== undefined ? fitting.use(database) : fitting.use()).job(jobType).fit(async () => tickStep())
+
+  return {
+    raise: (key, options) => raise(key, options),
+    resolve: (key, options) => resolveKey(key, false, true, options),
+    acknowledge,
+    link,
+    list,
+    occurrences,
+    totals,
+    health,
+    job: job as unknown as Handler<Record<string, unknown>, unknown>,
+    tick: () => tickStep(),
+    async requeueFailed(options) {
+      if (typeof options.by !== 'string' || options.by.trim() === '') throw new ValidationError({ by: ['must be a non-empty string'] }, 'alerts.requeueFailed needs by, who is requeueing.')
+      const by = options.by.trim()
+      const requeued = await outbox.requeue({ ...(options.sink !== undefined ? { sink: options.sink } : {}), ...(options.failedSince !== undefined ? { failedSince: options.failedSince } : {}) })
+      for (const row of requeued) {
+        await auditAlert('requeue', row['alert_key'] as string, { sink: row['sink'], kind: row['kind'], by, attempts: Number(row['prev_attempts']), error: row['last_error'] ?? null }, database)
+      }
+      return { requeued: requeued.length }
+    },
+    async open() {
+      const rows = await db().execute(sql`SELECT * FROM pipework_alerts WHERE status = 'open' ORDER BY first_seen_at, key`)
+      return rows.map(toAlert)
+    },
+    async get(key) {
+      const rows = await db().execute(sql`SELECT * FROM pipework_alerts WHERE key = ${key}`)
+      return rows[0] === undefined ? null : toAlert(rows[0])
+    },
+    async deliveries(options = {}) {
+      const rows = options.key === undefined
+        ? await db().execute(sql`SELECT * FROM pipework_alert_deliveries ORDER BY created_at, id`)
+        : await db().execute(sql`SELECT * FROM pipework_alert_deliveries WHERE alert_key = ${options.key} ORDER BY created_at, id`)
+      return rows.map(toDeliveryRow)
+    },
+  }
+}