pipework 0.252.0 → 0.253.0

package/src/infra/alerts/http.ts

@@ -0,0 +1,192 @@
+import { createHash, timingSafeEqual } from 'node:crypto'
+import { Builder } from '../../request/di/builder.js'
+import type { Handler } from '../../request/di/types.js'
+import { schema } from '../../data/schema/namespace.js'
+import { PipeworkError, NotFoundError, UnauthorizedError } from '../../core/errors/index.js'
+import type { Alerts, AlertSeverity, AlertStatus } from './handle.js'
+
+export interface AlertsHttpOptions {
+  /** Route prefix, e.g. '/ops/alerts'. No trailing slash. */
+  readonly path: string
+  /** Service credentials, name -> secret. A request sends `Authorization: Bearer <secret>`; the matching name is the default `by`. At least one; secrets at least 32 chars. */
+  readonly credentials: Readonly<Record<string, string>>
+}
+
+export class BadRequestError extends PipeworkError {
+  readonly code = 'BAD_REQUEST' as const
+  readonly httpStatus = 400 as const
+}
+
+const STATUSES: readonly AlertStatus[] = ['open', 'acknowledged', 'resolved']
+const SEVERITIES: readonly AlertSeverity[] = ['info', 'warning', 'error', 'critical']
+
+const sha = (s: string): Buffer => createHash('sha256').update(s).digest()
+
+function authenticate(credentials: readonly (readonly [string, Buffer])[], header: string | undefined): string {
+  const match = header !== undefined ? /^Bearer (.+)$/.exec(header) : null
+  const given = sha(match?.[1] ?? '')
+  let name: string | null = null
+  for (const [credName, digest] of credentials) {
+    if (timingSafeEqual(given, digest) && match !== null && name === null) name = credName
+  }
+  if (name === null) throw new UnauthorizedError('A valid service credential is required: send Authorization: Bearer <secret>.')
+  return name
+}
+
+function bad(field: string, problem: string): never {
+  throw new BadRequestError(`Invalid ${field}: ${problem}.`, { field })
+}
+
+function optString(source: unknown, field: string): string | undefined {
+  const value = (source as Record<string, unknown> | null | undefined)?.[field]
+  if (value === undefined) return undefined
+  if (typeof value !== 'string') bad(field, 'must be a string')
+  return value
+}
+
+function body(raw: unknown): Record<string, unknown> {
+  if (raw === undefined || raw === null) return {}
+  if (typeof raw !== 'object' || Array.isArray(raw)) bad('body', 'must be a JSON object')
+  return raw as Record<string, unknown>
+}
+
+function csv<T extends string>(raw: string | undefined, field: string, allowed: readonly T[]): T[] | undefined {
+  if (raw === undefined || raw === '') return undefined
+  const out: T[] = []
+  for (const part of raw.split(',')) {
+    if (!(allowed as readonly string[]).includes(part)) bad(field, `"${part}" is not one of ${allowed.join(', ')}`)
+    out.push(part as T)
+  }
+  return out
+}
+
+function ownerList(raw: string | undefined): string[] | undefined {
+  if (raw === undefined || raw === '') return undefined
+  const parts = raw.split(',').map(p => p.trim())
+  if (parts.some(p => p === '')) bad('owner', 'must be a comma list of non-empty names')
+  return parts
+}
+
+function limitOf(raw: string | undefined): number | undefined {
+  if (raw === undefined) return undefined
+  if (!/^[1-9][0-9]*$/.test(raw)) bad('limit', 'must be a positive integer')
+  return Number(raw)
+}
+
+function pageQuery(query: unknown): { limit?: number; cursor?: string } {
+  const limit = optString(query, 'limit')
+  const cursor = optString(query, 'cursor')
+  const out: { limit?: number; cursor?: string } = {}
+  const n = limitOf(limit)
+  if (n !== undefined) out.limit = n
+  if (cursor !== undefined) out.cursor = cursor
+  return out
+}
+
+function validateOptions(options: AlertsHttpOptions): void {
+  if (!options.path.startsWith('/') || options.path.length < 2 || options.path.endsWith('/')) {
+    throw new Error(`alerts.http: path "${options.path}" must start with "/" and have no trailing slash, e.g. "/ops/alerts".`)
+  }
+  const entries = Object.entries(options.credentials)
+  if (entries.length === 0) {
+    throw new Error('alerts.http: credentials needs at least one entry, name -> secret (the secret is at least 32 characters).')
+  }
+  for (const [name, secret] of entries) {
+    if (typeof secret !== 'string' || secret.length < 32) {
+      throw new Error(`alerts.http: the secret for credential "${name}" is under 32 characters; use a long random value.`)
+    }
+  }
+}
+
+/**
+ * Routes for an operations agent, under `options.path` (all JSON, no session, no tenant):
+ * GET list (`status`, `severity`, `owner` comma lists, `limit`, `cursor`; `owner` filters on the current owner), GET `/totals`, GET `/health`,
+ * GET `/:key` (an alert keyed `totals` or `health` is not reachable here), GET `/:key/occurrences`, and POST `/:key/acknowledge` (optional `owner` in the body replaces the alert's owner and is kept in its `ownerHistory`), `/:key/resolve`, `/:key/link`.
+ * `:key` is URL-encoded. Every request needs `Authorization: Bearer <secret>` matching one of
+ * `credentials`; the credential's name is the default `by` for acknowledge and resolve.
+ * Reads cross tenants (the handle reads through `pipe.system()`). Resolve never notifies:
+ * the agent does not hear its own change. Dates are ISO strings. A bad value is 400 naming the field.
+ */
+export function alertsHttp(handle: Alerts, options: AlertsHttpOptions): Handler<Record<string, unknown>, unknown>[] {
+  validateOptions(options)
+  const credentials = Object.entries(options.credentials).map(([name, secret]) => [name, sha(secret)] as const)
+  const base = options.path
+  const any = schema.check.unknown()
+
+  const get = (path: string) => new Builder().public().request().query(any).params(any).route('GET', path)
+  const post = (path: string) => new Builder().public().request().query(any).params(any).input(any).route('POST', path)
+
+  // Dates inside the results serialize through JSON.stringify as ISO strings.
+  const keyOf = (params: unknown): string => String((params as { key: string }).key)
+  const known = async (key: string): Promise<void> => {
+    if ((await handle.get(key)) === null) throw new NotFoundError(`Alert "${key}"`)
+  }
+
+  return [
+    get(base).fit(async ({ request, query }) => {
+      authenticate(credentials, request.headers['authorization'])
+      const q: { status?: AlertStatus[]; severity?: AlertSeverity[]; owner?: string[]; limit?: number; cursor?: string } = pageQuery(query)
+      const status = csv(optString(query, 'status'), 'status', STATUSES)
+      const severity = csv(optString(query, 'severity'), 'severity', SEVERITIES)
+      if (status !== undefined) q.status = status
+      if (severity !== undefined) q.severity = severity
+      const owner = ownerList(optString(query, 'owner'))
+      if (owner !== undefined) q.owner = owner
+      return handle.list(q)
+    }),
+    get(`${base}/totals`).fit(async ({ request }) => {
+      authenticate(credentials, request.headers['authorization'])
+      return handle.totals()
+    }),
+    get(`${base}/health`).fit(async ({ request }) => {
+      authenticate(credentials, request.headers['authorization'])
+      return handle.health()
+    }),
+    get(`${base}/:key`).fit(async ({ request, params }) => {
+      authenticate(credentials, request.headers['authorization'])
+      const found = await handle.get(keyOf(params))
+      if (found === null) throw new NotFoundError(`Alert "${keyOf(params)}"`)
+      return found
+    }),
+    get(`${base}/:key/occurrences`).fit(async ({ request, params, query }) => {
+      authenticate(credentials, request.headers['authorization'])
+      const page = pageQuery(query)
+      await known(keyOf(params))
+      return handle.occurrences(keyOf(params), page)
+    }),
+    post(`${base}/:key/acknowledge`).fit(async ({ request, params, input }) => {
+      const name = authenticate(credentials, request.headers['authorization'])
+      const b = body(input)
+      const by = optString(b, 'by') ?? name
+      const note = optString(b, 'note')
+      const owner = optString(b, 'owner')
+      if (owner !== undefined && owner.trim() === '') bad('owner', 'must not be empty')
+      return {
+        acknowledged: await handle.acknowledge(keyOf(params), {
+          by, ...(note === undefined ? {} : { note }), ...(owner === undefined ? {} : { owner }),
+        }),
+      }
+    }),
+    post(`${base}/:key/resolve`).fit(async ({ request, params, input }) => {
+      const name = authenticate(credentials, request.headers['authorization'])
+      const b = body(input)
+      const by = optString(b, 'by') ?? name
+      const note = optString(b, 'note')
+      const release = optString(b, 'release')
+      return {
+        resolved: await handle.resolve(keyOf(params), {
+          by, notify: false,
+          ...(note === undefined ? {} : { note }),
+          ...(release === undefined ? {} : { release }),
+        }),
+      }
+    }),
+    post(`${base}/:key/link`).fit(async ({ request, params, input }) => {
+      authenticate(credentials, request.headers['authorization'])
+      const url = optString(body(input), 'url')
+      if (url === undefined) bad('url', 'is required')
+      if (!/^https?:\/\/[^\s]+$/i.test(url)) bad('url', 'must be an http or https URL')
+      return { linked: await handle.link(keyOf(params), url) }
+    }),
+  ]
+}

package/src/infra/alerts/index.ts

@@ -0,0 +1,12 @@
+export { createAlerts } from './handle.js'
+export type {
+  Alerts, AlertsConfig, Alert, OwnerChange, AlertSeverity, AlertSubject, RaiseOptions, DeliveryRow, TickResult,
+  AlertStatus, JsonValue, AlertOccurrence, Page, AlertTotals, DeliveryHealth, ResolveOptions,
+} from './handle.js'
+export type { AlertSink, AlertDelivery, AlertPayload, AlertEventPayload, AlertDigestPayload, AlertWebhookConfig, MemoryAlertSink } from './sinks.js'
+export type { AlertSource, AlertCondition } from './sources.js'
+export { alertsHttp } from './http.js'
+export type { AlertsHttpOptions } from './http.js'
+export { clientFaults } from './client-faults.js'
+export type { ClientFaultsOptions, ClientFaultReport } from './client-faults.js'
+export type { RequestFaultsOptions } from './request-faults.js'

package/src/infra/alerts/infra-sources.ts

@@ -0,0 +1,180 @@
+// Sources over the failures that mail, storage, access links and sign-in locks record but nothing used to read.
+// Each is a composition of `querySource` and `newRowsSource`; a source is opt-in, so its tables exist only
+// when the blueprint key named in its JSDoc is set.
+
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { common, positive, text, FIFTEEN_MIN, HOUR, type Basics } from './source-options.js'
+import { composeSources, newRowsSource, querySource, type AlertSource } from './sources.js'
+
+const nullable = (v: unknown): string | null => v === null || v === undefined ? null : String(v)
+
+/**
+ * Needs blueprint `email: {}`. Raises `email-failed:outbound` and `email-failed:inbound` for messages marked
+ * `failed` (`failed_at`) since the last tick (clears by the handle's quiet period); `email-stuck:sending` while
+ * a message has been `sending` more than `stuckAfterMs` (default 15 min); `email-stuck:queued` while one has been
+ * `queued` more than `queuedAfterMs` (default 1 h), which means nothing drains the send queue;
+ * `email-inbound-stuck:received` while an inbound message has been `received` more than `receivedAfterMs`
+ * (default 15 min) without being handled; and `email-raw-undrained:all` while raw inbound messages are more than
+ * `sweepGraceMs` (default 1 h) past `raw_expires_at` and not deleted, which means the raw sweep is not running.
+ * Addresses and subjects are never put in a key, message or context.
+ */
+export function emailSource(options: Basics & { stuckAfterMs?: number; queuedAfterMs?: number; receivedAfterMs?: number; sweepGraceMs?: number; settleMs?: number } = {}): AlertSource {
+  const who = 'alerts.source.email'
+  const stuckAfter = positive(options.stuckAfterMs ?? FIFTEEN_MIN, 'stuckAfterMs', who)
+  const queuedAfter = positive(options.queuedAfterMs ?? HOUR, 'queuedAfterMs', who)
+  const receivedAfter = positive(options.receivedAfterMs ?? FIFTEEN_MIN, 'receivedAfterMs', who)
+  const grace = positive(options.sweepGraceMs ?? HOUR, 'sweepGraceMs', who)
+  const failedRows = (direction: 'outbound' | 'inbound', table: string): AlertSource => newRowsSource({
+    ...common(options),
+    name: 'email-failed',
+    stateKey: `email-failed:${direction}`,
+    table,
+    at: 'failed_at',
+    groupBy: sql.raw(`'${direction}'`),
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    message: g => `${String(g.count)} ${direction} email(s) failed; last error: ${text(g.last['error'])}`,
+    context: g => ({ direction, count: g.count, lastError: nullable(g.last['error']) }),
+  })
+  const stuck = querySource({
+    ...common(options),
+    name: 'email-stuck',
+    query: sql`
+      SELECT 'sending' AS kind, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(sending_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_email_messages
+      WHERE status = 'sending' AND sending_at < clock_timestamp() - ${stuckAfter} * interval '1 millisecond'
+      HAVING count(*) > 0
+      UNION ALL
+      SELECT 'queued', count(*)::int, (extract(epoch FROM clock_timestamp() - min(created_at)) * 1000)::bigint
+      FROM pipework_email_messages
+      WHERE status = 'queued' AND created_at < clock_timestamp() - ${queuedAfter} * interval '1 millisecond'
+      HAVING count(*) > 0`,
+    id: r => String(r['kind']),
+    message: r => String(r['kind']) === 'sending'
+      ? `${String(r['n'])} email(s) stuck sending for more than ${String(stuckAfter)} ms (oldest ${String(r['oldest_ms'])} ms)`
+      : `${String(r['n'])} email(s) queued for more than ${String(queuedAfter)} ms (oldest ${String(r['oldest_ms'])} ms); is the send queue being drained?`,
+    context: r => ({ count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  const received = querySource({
+    ...common(options),
+    name: 'email-inbound-stuck',
+    query: sql`
+      SELECT count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(received_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_email_inbound
+      WHERE status = 'received' AND received_at < clock_timestamp() - ${receivedAfter} * interval '1 millisecond'
+      HAVING count(*) > 0`,
+    id: () => 'received',
+    message: r => `${String(r['n'])} inbound email(s) received more than ${String(receivedAfter)} ms ago and not handled (oldest ${String(r['oldest_ms'])} ms)`,
+    context: r => ({ count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  const raw = querySource({
+    ...common(options),
+    name: 'email-raw-undrained',
+    query: sql`
+      SELECT count(*)::int AS n FROM pipework_email_inbound
+      WHERE raw_deleted_at IS NULL AND raw_expires_at < clock_timestamp() - ${grace} * interval '1 millisecond'
+      HAVING count(*) > 0`,
+    id: () => 'all',
+    message: r => `${String(r['n'])} raw inbound message(s) past their retention and not deleted; is the inbound sweep scheduled?`,
+    context: r => ({ count: Number(r['n']) }),
+  })
+  return composeSources('email', [
+    failedRows('outbound', 'pipework_email_messages'), failedRows('inbound', 'pipework_email_inbound'), stuck, received, raw,
+  ])
+}
+
+/**
+ * Needs blueprint `storage: {}`. Raises `storage-expiry-undrained:all` while `storage_file` rows are more than
+ * `sweepGraceMs` (default 1 h) past `expires_at` and still exist (`sweep()` deletes them), which means the
+ * storage sweep is not scheduled.
+ */
+export function storageSource(options: Basics & { sweepGraceMs?: number } = {}): AlertSource {
+  const grace = positive(options.sweepGraceMs ?? HOUR, 'sweepGraceMs', 'alerts.source.storage')
+  return querySource({
+    ...common(options),
+    name: 'storage-expiry-undrained',
+    query: sql`
+      SELECT count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(expires_at)) * 1000)::bigint AS oldest_ms
+      FROM storage_file
+      WHERE expires_at IS NOT NULL AND expires_at < clock_timestamp() - ${grace} * interval '1 millisecond'
+      HAVING count(*) > 0`,
+    id: () => 'all',
+    message: r => `${String(r['n'])} expired file(s) not swept (oldest ${String(r['oldest_ms'])} ms past expiry); is the storage sweep scheduled?`,
+    context: r => ({ count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+}
+
+/**
+ * Needs blueprint `accessLinks: {}`. Raises `access-link-abandoned:<tenantId>` while a tenant has uses still
+ * `pending` more than `graceMs` (default 15 min) after `reserved_until`: the reservation was taken and never
+ * completed or released.
+ */
+export function accessLinksSource(options: Basics & { graceMs?: number } = {}): AlertSource {
+  const grace = positive(options.graceMs ?? FIFTEEN_MIN, 'graceMs', 'alerts.source.accessLinks')
+  return querySource({
+    ...common(options),
+    name: 'access-link-abandoned',
+    query: sql`
+      SELECT tenant_id, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(reserved_until)) * 1000)::bigint AS oldest_ms
+      FROM access_link_use
+      WHERE status = 'pending' AND reserved_until < clock_timestamp() - ${grace} * interval '1 millisecond'
+      GROUP BY tenant_id ORDER BY tenant_id`,
+    id: r => String(r['tenant_id']),
+    message: r => `${String(r['n'])} access link use(s) for ${String(r['tenant_id'])} reserved and never finished (oldest ${String(r['oldest_ms'])} ms past reservation)`,
+    subject: r => ({ tenantId: String(r['tenant_id']) }),
+    context: r => ({ count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+}
+
+const LOCKOUT_TABLES = ['pipework_signin_attempts', 'pipework_guard_attempts'] as const
+type LockoutTable = typeof LOCKOUT_TABLES[number]
+
+/**
+ * Raises `auth-lockout:<table>` (default severity `warning`) while rows of that table have `locked_until` in the
+ * future, with the count in the message and context. One condition per table, never per account: no account,
+ * address or key is named. `pipework_signin_attempts` needs blueprint `passwords: {}`, `pipework_guard_attempts`
+ * needs `guards: {}`. `tables` overrides; when omitted, the tables that exist are resolved once, on the first
+ * evaluate, and it throws if neither does.
+ */
+export function lockoutsSource(options: Basics & { tables?: readonly LockoutTable[] } = {}): AlertSource {
+  if (options.tables !== undefined) {
+    if (options.tables.length === 0) throw new Error('[pipework] alerts.source.lockouts tables must not be empty; omit it to watch the lockout tables that exist.')
+    for (const t of options.tables) {
+      if (!LOCKOUT_TABLES.includes(t)) throw new Error(`[pipework] alerts.source.lockouts table must be one of ${LOCKOUT_TABLES.join(', ')}, got ${String(t)}.`)
+    }
+  }
+  const build = (tables: readonly LockoutTable[]): AlertSource => {
+    const selects = [...new Set(tables)].map(t => `SELECT '${t}' AS tbl, count(*)::int AS n FROM ${t} WHERE locked_until > clock_timestamp() HAVING count(*) > 0`)
+    return querySource({
+      ...common(options),
+      severity: options.severity ?? 'warning',
+      name: 'auth-lockout',
+      query: sql.raw(selects.join(' UNION ALL ')),
+      id: r => String(r['tbl']),
+      message: r => `${String(r['n'])} lockout(s) active on ${String(r['tbl'])}`,
+      context: r => ({ table: String(r['tbl']), count: Number(r['n']) }),
+    })
+  }
+  if (options.tables !== undefined) return build(options.tables)
+  let resolved: Promise<AlertSource> | undefined
+  const resolve = async (): Promise<AlertSource> => {
+    const rows = await pipeSystem(options.database).execute(sql`
+      SELECT to_regclass('pipework_signin_attempts') IS NOT NULL AS a, to_regclass('pipework_guard_attempts') IS NOT NULL AS b`)
+    const found = LOCKOUT_TABLES.filter((_, i) => rows[0]?.[i === 0 ? 'a' : 'b'] === true)
+    if (found.length === 0) {
+      throw new Error('[pipework] alerts.source.lockouts found neither pipework_signin_attempts nor pipework_guard_attempts; enable `passwords: {}` and/or `guards: {}` in the blueprint, or drop this source.')
+    }
+    return build(found)
+  }
+  return {
+    name: 'auth-lockout',
+    owns: key => key.startsWith('auth-lockout:'),
+    async evaluate() {
+      if (resolved === undefined) {
+        resolved = resolve()
+        resolved.catch(() => { resolved = undefined })
+      }
+      return (await resolved).evaluate()
+    },
+  }
+}

package/src/infra/alerts/namespace.ts

@@ -0,0 +1,81 @@
+import { createAlerts } from './handle.js'
+import { alertsHttp } from './http.js'
+import { clientFaults } from './client-faults.js'
+import { webhookSink, memorySink } from './sinks.js'
+import { requestFaultsHook } from './request-faults.js'
+import { deadLetterSource, longRunningSource, backlogSource, healthSource, quotaExhaustedSource, unitFaultSource, loopStalledSource, loopCheck, querySource, newRowsSource } from './sources.js'
+import { recordFault, recordFaultFor, useFaultTarget, faultsSource } from './faults.js'
+import { recordBeatFor } from './beat.js'
+import { deliveryFailedSource, webhookFailedSource, creditsSource } from './subsystem-sources.js'
+import { emailSource, storageSource, accessLinksSource, lockoutsSource } from './infra-sources.js'
+import { schedulesSource, stepsSource, pipelinesSource, tracesSource } from './async-sources.js'
+
+/** Alerts namespace — raise and resolve keyed conditions, grouped, delivered to sinks through an outbox, traced. */
+export const alerts = {
+  /** Creates an alerts handle. */
+  create: createAlerts,
+  /** Routes for an operations agent: list, get, totals, delivery health, acknowledge, resolve, link; bearer credentials. */
+  http: alertsHttp,
+  /** One POST route that records failures reported by browser code as a trace step and an alert; answers 204, no session needed. Send with `reportClientFault` from `pipework/client`. */
+  clientFaults,
+  /** An `onInternalError` hook for the HTTP server: raises `request-fault:<METHOD> <route>` for each internal 500. */
+  requestFaults: requestFaultsHook,
+  /** A sink that POSTs the alert as signed JSON to a URL. */
+  webhook: webhookSink,
+  /** A sink that keeps deliveries in `.delivered`, for tests. */
+  memory: memorySink,
+  /** A `/ready` health check named `loops`: unhealthy while a background loop (the tick included) has not run or succeeded within its allowance. */
+  loopCheck,
+  /** Records a failure your code swallowed (logged and carried on) so `alerts.source.faults` can raise it; `void alerts.recordFault('my-kind', message, context?)`. Best effort, never throws, writes outside the caller's transaction. Does nothing without `alerts` in the blueprint. */
+  recordFault,
+  /** `recordFault` for code that holds the manifold and may run outside any Flow: `void alerts.recordFaultFor(manifold, kind, message, context?)`. */
+  recordFaultFor,
+  /** Registers the manifold (or a `DB`, or a function returning one) that `alerts.recordFault` writes on outside any Flow; call once at boot in a process that is not a pipework surface, and keep the release for shutdown. A manifold without `alerts` registers nothing. */
+  useFaultTarget,
+  /** Records one iteration of a background loop of your own for `alerts.source.loopStalled`: `void alerts.recordBeat(manifold, 'my-loop', { ok, error?, everyMs })`. Works outside any Flow; does nothing without `alerts`. */
+  recordBeat: recordBeatFor,
+  source: {
+    /** Raises `job-dead-letter:<jobType>` for jobs that ran out of attempts. */
+    deadLetter: deadLetterSource,
+    /** Raises `job-long-running:<jobType>` while a job has been claimed too long. */
+    longRunning: longRunningSource,
+    /** Raises `queue-backlog:<jobType>` while too many jobs wait or the oldest has waited too long. */
+    backlog: backlogSource,
+    /** Raises `quota-exhausted:<key>` while an upstream quota is at or under its floor or refused a call. */
+    quotaExhausted: quotaExhaustedSource,
+    /** Raises `health:<name>` while a health check fails. */
+    health: healthSource,
+    /** Raises `unit-fault:<unit>` while a systemd unit has stopped abnormally within `window` (rows written by `pipework alerts unit-fault`). */
+    unitFault: unitFaultSource,
+    /** Raises `${name}:${id}` for each row a query returns while the condition holds, and resolves what it stops returning. */
+    query: querySource,
+    /** Raises `${name}:${id}` per group of rows recorded in a table since the last tick (a cursor, settled against late commits). */
+    newRows: newRowsSource,
+    /** Raises `delivery-failed:<table>:<sink>` for outbox rows that gave up and `delivery-stuck:<table>:<sink>` for rows pending too long. */
+    deliveryFailed: deliveryFailedSource,
+    /** Raises `webhook-failed:<endpointId>` for outbound webhook deliveries that gave up and `webhook-stalled:<endpointId>` for ones overdue for a retry. */
+    webhookFailed: webhookFailedSource,
+    /** Raises `credit-overdraft:<tenantId>`, `credit-held:<tenantId>` and `credit-expiry-undrained:all` over the credits tables. */
+    credits: creditsSource,
+    /** Raises `email-failed:<direction>`, `email-stuck:<sending|queued>`, `email-inbound-stuck:received` and `email-raw-undrained:all` over the mail tables. */
+    email: emailSource,
+    /** Raises `storage-expiry-undrained:all` while expired files are not swept. */
+    storage: storageSource,
+    /** Raises `access-link-abandoned:<tenantId>` while link uses stay reserved and never finish. */
+    accessLinks: accessLinksSource,
+    /** Raises `auth-lockout:<table>` (warning) while sign-in or guard lockouts are active; counts only, no accounts. */
+    lockouts: lockoutsSource,
+    /** Raises `schedule-overdue:<name>` while a cron schedule's `next_run_at` is long past, which means no scheduler is firing it. */
+    schedules: schedulesSource,
+    /** Raises `step-run-failed:<graph>`, `step-run-unnotified:<graph>` and `step-task-waiting:<graph>` and `step-run-stalled:<graph>` over the step graph tables. */
+    steps: stepsSource,
+    /** Raises `pipeline-failed:<pipeline>` for failed runs `pipeline-waiting:<pipeline>` for runs waiting too long and `pipeline-stalled:<pipeline>` for runs still running too long. */
+    pipelines: pipelinesSource,
+    /** Raises `trace-expired:<entryKind>` for traces force-closed by the expiry sweep and `trace-expiry-undrained:all` while expired open traces are not being swept. */
+    traces: tracesSource,
+    /** Raises `fault:<kind>` per kind of swallowed failure recorded by `recordFault` since the last tick (the pipework subsystems record audit-write, quota-record, credits-callback, trace-steps-lost, queue-* and scheduler faults). `kinds` limits it. */
+    faults: faultsSource,
+    /** Raises `loop-stalled:<name>` while a background loop (scheduler, reaper, claim loop, support drain) has not run or succeeded within its allowance; the tick itself is covered by `pipework alerts stale`. */
+    loopStalled: loopStalledSource,
+  },
+} as const

package/src/infra/alerts/redact.ts

@@ -0,0 +1,21 @@
+// An email address: local part, "@", and a domain with at least one dot.
+const EMAIL = /[A-Za-z0-9._%+-]+@(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+/gi
+
+const global = (p: RegExp): RegExp => (p.flags.includes('g') ? p : new RegExp(p.source, `${p.flags}g`))
+
+/** Email addresses become `[email]`, then each extra pattern becomes `[redacted]`. The rest of the text stays. */
+export function redactText(text: string, extra: readonly RegExp[] = []): string {
+  let out = text.replace(EMAIL, '[email]')
+  for (const p of extra) out = out.replace(global(p), '[redacted]')
+  return out
+}
+
+/** `redactText` on every string inside a JSON value, at any depth. */
+export function redactJson<T>(value: T, extra: readonly RegExp[] = []): T {
+  if (typeof value === 'string') return redactText(value, extra) as T
+  if (Array.isArray(value)) return value.map(v => redactJson(v, extra)) as T
+  if (value !== null && typeof value === 'object') {
+    return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, redactJson(v, extra)])) as T
+  }
+  return value
+}

package/src/infra/alerts/request-faults.ts

@@ -0,0 +1,41 @@
+import type { Alerts, AlertSeverity } from './handle.js'
+import { describeError } from '../../core/logging/error-chain.js'
+import type { InternalErrorFault } from '../../request/http/types.js'
+
+export interface RequestFaultsOptions {
+  readonly owner?: string
+  readonly runbook?: string
+  /** The owner and runbook for one fault, over `owner` and `runbook`: an undefined field or result keeps them. */
+  readonly assign?: (fault: InternalErrorFault) => { readonly owner?: string; readonly runbook?: string } | undefined
+  /** Default `'error'`. */
+  readonly severity?: AlertSeverity
+}
+
+/**
+ * An `onInternalError` hook for the HTTP server: raises `request-fault:<METHOD> <route pattern>`
+ * with the error message (its causes' text folded in) and a context of method, route, requestId and traceId, and the cause chain when there is one; the subject is the
+ * request's tenant when it has one. Repeats of one route group into one alert.
+ */
+export function requestFaultsHook(handle: Alerts, options: RequestFaultsOptions = {}): (fault: InternalErrorFault) => Promise<void> {
+  return async (fault) => {
+    const { message: detail, chain } = describeError(fault.error)
+    const assigned = options.assign?.(fault)
+    const owner = assigned?.owner ?? options.owner
+    const runbook = assigned?.runbook ?? options.runbook
+    await handle.raise(`request-fault:${fault.method} ${fault.route}`, {
+      severity: options.severity ?? 'error',
+      message: `${fault.method} ${fault.route}: ${detail}`,
+      ...(fault.tenantId !== null ? { subject: { tenantId: fault.tenantId } } : {}),
+      ...(owner !== undefined ? { owner } : {}),
+      ...(runbook !== undefined ? { runbook } : {}),
+      context: {
+        method: fault.method,
+        route: fault.route,
+        requestId: fault.requestId,
+        traceId: fault.traceId,
+        message: detail,
+        ...(chain.length > 0 ? { cause: chain.map((l) => ({ ...l })) } : {}),
+      },
+    })
+  }
+}

package/src/infra/alerts/schema.ts

@@ -0,0 +1,124 @@
+// The tables behind alerts: one row per alert key, one per raise (occurrences), the outbox of deliveries (one row per
+// event and sink), and the cursor a source keeps between evaluations. Not tenant-scoped.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import { schema } from '../../data/schema/namespace.js'
+import { index } from '../../data/db/idx/index.js'
+import { sql } from '../../data/db/sql.js'
+
+export const ALERT_STATUSES = ['open', 'acknowledged', 'resolved'] as const
+export const ALERT_SEVERITIES = ['info', 'warning', 'error', 'critical'] as const
+export const ALERT_DELIVERY_KINDS = ['raise', 'repeat', 'resolve', 'reopen', 'digest'] as const
+
+export const alerts = define('pipework_alerts', {
+  key: field.text().primaryKey(),
+  status: field.enum(ALERT_STATUSES).default('open'),
+  severity: field.enum(ALERT_SEVERITIES),
+  message: field.text(),
+  subject: field.jsonb(schema.check.json()).nullable(),
+  owner: field.text().nullable(),
+  ownerHistory: field.jsonb(schema.check.json()).default([]),
+  runbook: field.text().nullable(),
+  context: field.jsonb(schema.check.json()).nullable(),
+  count: field.integer().default(1),
+  firstSeenAt: field.timestamp().default('now'),
+  lastSeenAt: field.timestamp().default('now'),
+  lastSentAt: field.timestamp().nullable(),
+  resolvedAt: field.timestamp().nullable(),
+  reopenCount: field.integer().default(0),
+  reopenedAt: field.timestamp().nullable(),
+  acknowledgedAt: field.timestamp().nullable(),
+  acknowledgedBy: field.text().nullable(),
+  resolvedBy: field.text().nullable(),
+  resolveNote: field.text().nullable(),
+  resolvedRelease: field.text().nullable(),
+  issueUrl: field.text().nullable(),
+  traceIds: field.custom(schema.col.text('trace_ids').array(), schema.check.array(schema.check.string())).default('{}'),
+  resendAfterMs: field.integer().nullable(),
+  resolveAfterQuietMs: field.integer().nullable(),
+  updatedAt: field.timestamp().default('now'),
+}, {
+  indexes: (t) => ({
+    byStatus: index('idx_pipework_alerts_status').on(t['status']),
+  }),
+})
+
+// One row per raise, kept for `occurrenceRetentionMs`.
+export const alertOccurrences = define('pipework_alert_occurrences', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  alertKey: field.text(),
+  at: field.timestamp().default('now'),
+  severity: field.enum(ALERT_SEVERITIES),
+  message: field.text(),
+  context: field.jsonb(schema.check.json()).nullable(),
+  traceId: field.text().nullable(),
+}, {
+  indexes: (t) => ({
+    byKey: index('idx_pipework_alert_occurrences_key').on(t['alertKey'], t['at'], t['id']),
+    byAt: index('idx_pipework_alert_occurrences_at').on(t['at']),
+  }),
+})
+
+export const alertDeliveries = define('pipework_alert_deliveries', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  alertKey: field.text(),
+  kind: field.enum(ALERT_DELIVERY_KINDS),
+  sink: field.text(),
+  payload: field.jsonb(schema.check.json()),
+  attempts: field.integer().default(0),
+  nextAttemptAt: field.timestamp().default('now'),
+  deliveredAt: field.timestamp().nullable(),
+  failedAt: field.timestamp().nullable(),
+  lastError: field.text().nullable(),
+  createdAt: field.timestamp().default('now'),
+}, {
+  indexes: (t) => ({
+    due: index('idx_pipework_alert_deliveries_due')
+      .on(t['nextAttemptAt'])
+      .where(sql.raw('delivered_at IS NULL AND failed_at IS NULL')),
+    byKey: index('idx_pipework_alert_deliveries_key').on(t['alertKey']),
+  }),
+})
+
+export const alertSourceState = define('pipework_alert_source_state', {
+  name: field.text().primaryKey(),
+  cursorAt: field.timestamp(),
+})
+
+// One row per abnormal stop of a systemd unit, written by `pipework alerts unit-fault` from the unit's ExecStopPost.
+export const unitFaults = define('pipework_unit_faults', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  unit: field.text(),
+  result: field.text(),
+  exitCode: field.text().nullable(),
+  exitStatus: field.text().nullable(),
+  at: field.timestamp().default('now'),
+}, {
+  indexes: (t) => ({
+    byUnit: index('idx_pipework_unit_faults_unit_at').on(t['unit'], t['at']),
+  }),
+})
+
+// One row per background loop, upserted at the end of each iteration; `pipework alerts stale`, the loop-stalled source and the loops check read it.
+export const loopBeats = define('pipework_loop_beats', {
+  name: field.text().primaryKey(),
+  lastAt: field.timestamp(),
+  lastOkAt: field.timestamp().nullable(),
+  lastError: field.text().nullable(),
+  expectEveryMs: field.integer(),
+})
+
+// One row per failure that was swallowed and only logged, written by `recordFault`; the faults source reads it. Not tenant-scoped.
+export const faults = define('pipework_faults', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  kind: field.text(),
+  message: field.text(),
+  context: field.jsonb(schema.check.json()).nullable(),
+  tenantId: field.text().nullable(),
+  recordedAt: field.timestamp().default('now'),
+}, {
+  indexes: (t) => ({
+    byRecordedAt: index('idx_pipework_faults_recorded_at').on(t['recordedAt']),
+  }),
+})