pipework 0.252.0 → 0.253.0

package/src/infra/alerts/sinks.ts

@@ -0,0 +1,125 @@
+import { signPayload } from '../webhook/sign.js'
+import type { WebhookSignatureOptions } from '../webhook/sign.js'
+import type { Sink, SinkDelivery } from '../outbox/core.js'
+
+/** One alert's event, delivered at once. */
+export interface AlertEventPayload {
+  readonly kind: 'raise' | 'repeat' | 'resolve' | 'reopen'
+  readonly key: string
+  readonly severity: 'info' | 'warning' | 'error' | 'critical'
+  readonly message: string
+  readonly subject: { readonly tenantId?: string; readonly entity?: string; readonly entityId?: string } | null
+  readonly owner: string | null
+  readonly runbook: string | null
+  readonly context: Readonly<Record<string, unknown>> | null
+  readonly count: number
+  /** Times a raise came after a resolve. */
+  readonly reopenCount: number
+  readonly firstSeenAt: string
+  readonly lastSeenAt: string
+  readonly resolvedAt: string | null
+  readonly acknowledgedAt: string | null
+  readonly acknowledgedBy: string | null
+  /** On a `reopen`, those of the resolve it reopens. */
+  readonly resolvedBy: string | null
+  readonly resolveNote: string | null
+  readonly resolvedRelease: string | null
+  readonly issueUrl: string | null
+  readonly environment: string
+  /** The latest traces that raised it, newest first. */
+  readonly traceIds: readonly string[]
+}
+
+/** The alerts of the digest severities that changed in one window, sent once per sink. */
+export interface AlertDigestPayload {
+  readonly kind: 'digest'
+  readonly key: 'pipework.alerts.digest'
+  readonly environment: string
+  /** The previous digest's time; null on the first. */
+  readonly since: string | null
+  readonly until: string
+  readonly alerts: readonly {
+    readonly key: string
+    readonly severity: 'info' | 'warning' | 'error' | 'critical'
+    readonly status: 'open' | 'acknowledged' | 'resolved'
+    readonly message: string
+    readonly owner: string | null
+    readonly count: number
+    readonly reopenCount: number
+    readonly lastSeenAt: string
+  }[]
+}
+
+/** What a sink receives, and what the webhook sink POSTs as its body; tell them apart by `kind`. */
+export type AlertPayload = AlertEventPayload | AlertDigestPayload
+
+export type AlertDelivery = SinkDelivery<AlertPayload>
+
+/** A place alerts are sent; see `Sink`. */
+export type AlertSink = Sink<AlertPayload>
+
+export interface AlertWebhookConfig {
+  /** Sink name; default `'webhook'`. Two sinks on one handle need distinct names. */
+  readonly name?: string
+  readonly url: string
+  readonly secret: string
+  /** Request timeout; default 10 000, at most 20 000 (the handle gives a send 10 s and leases the row for 30 s). */
+  readonly timeoutMs?: number
+  readonly signing?: WebhookSignatureOptions
+}
+
+/**
+ * POSTs the payload as JSON with `X-Webhook-Id`, `X-Webhook-Timestamp` and `X-Webhook-Signature`
+ * (`X-Webhook-Id` is the delivery id, the same on every attempt; HMAC over `<id>.<timestamp>.<body>`, as `webhook.sign` makes them). Any non-2xx status fails the attempt.
+ */
+export function webhookSink<P extends { readonly kind: string } = AlertPayload>(config: AlertWebhookConfig): Sink<P> {
+  const timeoutMs = config.timeoutMs ?? 10_000
+  if (timeoutMs > 20_000) {
+    throw new Error(`[pipework] alerts.webhook: timeoutMs ${String(timeoutMs)} is over 20000. A delivery is leased for 30 s while it is sent; a longer send could be picked up and sent twice.`)
+  }
+  return {
+    name: config.name ?? 'webhook',
+    async send(delivery) {
+      const signed = signPayload(delivery.payload, config.secret, { ...config.signing, messageId: delivery.id })
+      const controller = new AbortController()
+      const timer = setTimeout(() => controller.abort(), timeoutMs)
+      try {
+        const response = await fetch(config.url, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'X-Webhook-Id': signed.messageId,
+            'X-Webhook-Timestamp': signed.timestamp.toString(),
+            'X-Webhook-Signature': signed.signature,
+          },
+          body: signed.body,
+          signal: controller.signal,
+        })
+        await response.text().catch(() => '')
+        if (response.status < 200 || response.status >= 300) throw new Error(`webhook answered HTTP ${String(response.status)}`)
+      } finally {
+        clearTimeout(timer)
+      }
+    },
+  }
+}
+
+export interface MemoryAlertSink<P extends { readonly kind: string } = AlertPayload> extends Sink<P> {
+  /** Every delivery received, in order. */
+  readonly delivered: SinkDelivery<P>[]
+  clear(): void
+}
+
+/** A sink that keeps what it receives, for tests. Generic over the payload so it fits `support.create` (`memorySink<SupportEventPayload>()`, or inferred where a support sink is expected). */
+export function memorySink<P extends { readonly kind: string } = AlertPayload>(options?: { name?: string }): MemoryAlertSink<P>
+export function memorySink(options?: { name?: string }): MemoryAlertSink
+export function memorySink(options: { name?: string } = {}): MemoryAlertSink<{ readonly kind: string }> {
+  const delivered: SinkDelivery<{ readonly kind: string }>[] = []
+  return {
+    name: options.name ?? 'memory',
+    delivered,
+    // eslint-disable-next-line @typescript-eslint/require-await -- the sink contract is async
+    async send(delivery) { delivered.push(delivery) },
+    clear() { delivered.length = 0 },
+  }
+}

package/src/infra/alerts/source-options.ts

@@ -0,0 +1,26 @@
+// Options and helpers shared by the alert sources built on `querySource` and `newRowsSource`.
+
+import type { AlertSeverity } from './handle.js'
+
+export interface Basics { owner?: string; runbook?: string; severity?: AlertSeverity; database?: string }
+
+export const FIFTEEN_MIN = 15 * 60_000
+export const HOUR = 3_600_000
+
+export function positive(value: number, label: string, who: string): number {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
+    throw new Error(`[pipework] ${who} ${label} must be a positive number of milliseconds, got ${String(value)}.`)
+  }
+  return Math.trunc(value)
+}
+
+export function common(options: Basics): Basics {
+  return {
+    ...(options.owner !== undefined ? { owner: options.owner } : {}),
+    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+    ...(options.severity !== undefined ? { severity: options.severity } : {}),
+    ...(options.database !== undefined ? { database: options.database } : {}),
+  }
+}
+
+export const text = (v: unknown): string => v === null || v === undefined ? 'none' : String(v)

package/src/infra/alerts/sources.ts

@@ -0,0 +1,503 @@
+import { createHash } from 'node:crypto'
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { assertValidIdentifier, quoteIdentifier } from '../../data/db/identifiers.js'
+import type { HealthCheck } from '../../core/lifecycle/types.js'
+import type { SQL } from '../../data/db/index.js'
+import type { Manifold } from '../../core/pipework.js'
+import type { AlertSeverity, JsonValue } from './handle.js'
+import { ALERTS_TICK_LOOP, LOOP_BEATS_QUERY, staleLoops } from './beat.js'
+import { getContext } from '../../request/context/store.js'
+import { describeError } from '../../core/logging/error-chain.js'
+
+/** A condition a source finds true right now. */
+export interface AlertCondition {
+  readonly key: string
+  readonly severity: AlertSeverity
+  readonly message: string
+  readonly subject?: { tenantId?: string; entity?: string; entityId?: string }
+  /** Passed to `alerts.raise` with the condition: who answers it, where the runbook is, and structured detail. */
+  readonly owner?: string
+  readonly runbook?: string
+  readonly context?: Readonly<Record<string, JsonValue>>
+}
+
+/**
+ * Evaluated on every tick. Each returned condition is raised (so an ongoing one groups). When
+ * `owns` is given, an open alert the source owns that it no longer returns is resolved.
+ */
+export interface AlertSource {
+  readonly name: string
+  owns?(key: string): boolean
+  evaluate(): Promise<AlertCondition[]>
+}
+
+interface TableOption { readonly table?: string; readonly database?: string }
+
+type Row = Record<string, unknown>
+
+interface SourceBasics { readonly severity?: AlertSeverity; readonly owner?: string; readonly runbook?: string; readonly database?: string }
+
+function assertSourceName(name: string, who: string): void {
+  if (typeof name !== 'string' || name === '' || name.includes(':')) {
+    throw new Error(`[pipework] ${who}: name must be a non-empty string without ':', got ${JSON.stringify(name)}.`)
+  }
+}
+
+function ownerFields(options: { owner?: string; runbook?: string }): { owner?: string; runbook?: string } {
+  return {
+    ...(options.owner !== undefined ? { owner: options.owner } : {}),
+    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+  }
+}
+
+/** One source from several: it evaluates each in turn and owns what any of them owns. */
+export function composeSources(name: string, parts: readonly AlertSource[]): AlertSource {
+  const owning = parts.filter(p => p.owns !== undefined)
+  return {
+    name,
+    ...(owning.length > 0 ? { owns: (key: string) => owning.some(p => p.owns!(key)) } : {}),
+    async evaluate() {
+      const out: AlertCondition[] = []
+      for (const part of parts) out.push(...await part.evaluate())
+      return out
+    },
+  }
+}
+
+export interface QuerySourceOptions extends SourceBasics {
+  /** Key prefix: conditions are keyed `${name}:${id}`. Non-empty, no ':'. */
+  readonly name: string
+  /** Returns one row per subject that is in trouble right now. */
+  readonly query: SQL
+  readonly id: (row: Row) => string
+  readonly message: (row: Row) => string
+  readonly context?: (row: Row) => Record<string, JsonValue>
+  readonly subject?: (row: Row) => AlertCondition['subject']
+}
+
+/**
+ * For a condition that is true right now: runs `query` (as the system, any tenant) on every tick and raises
+ * `${name}:${id}` for each row. An open alert under `${name}:` that a tick no longer returns is resolved.
+ */
+export function querySource(options: QuerySourceOptions): AlertSource {
+  assertSourceName(options.name, 'alerts.source.query')
+  const prefix = `${options.name}:`
+  const severity = options.severity ?? 'error'
+  return {
+    name: options.name,
+    owns: key => key.startsWith(prefix),
+    async evaluate() {
+      const rows = await pipeSystem(options.database).execute(options.query)
+      return rows.map((r: Row): AlertCondition => {
+        const subject = options.subject?.(r)
+        const context = options.context?.(r)
+        return {
+          key: `${prefix}${options.id(r)}`,
+          severity,
+          message: options.message(r),
+          ...(subject !== undefined ? { subject } : {}),
+          ...ownerFields(options),
+          ...(context !== undefined ? { context } : {}),
+        }
+      })
+    },
+  }
+}
+
+export interface NewRowsGroup { readonly id: string; readonly count: number; readonly last: Row; readonly sum: number }
+
+export interface NewRowsSourceOptions extends SourceBasics {
+  /** Key prefix: conditions are keyed `${name}:${id}`; also the cursor's state key unless `stateKey` is given. */
+  readonly name: string
+  /** The table holding the recorded failures (an identifier, not schema-qualified). */
+  readonly table: string
+  /** The timestamp column a row is stamped with when the failure is recorded. */
+  readonly at: string
+  /** Extra filter; the table is aliased `t`. */
+  readonly where?: SQL
+  /** A column of the table, or an SQL expression over alias `t`; one condition per distinct value. */
+  readonly groupBy: string | SQL
+  readonly message: (group: NewRowsGroup) => string
+  readonly context?: (group: NewRowsGroup) => Record<string, JsonValue>
+  readonly subject?: (group: NewRowsGroup) => AlertCondition['subject']
+  /** A numeric column to total per group, handed to the callbacks as `sum` (0 without it). */
+  readonly sum?: string
+  /** Rows are read once they are older than this many ms (default 30_000); see `newRowsSource`. */
+  readonly settleMs?: number
+  /** The cursor's key in `pipework_alert_source_state`; default `name`. */
+  readonly stateKey?: string
+}
+
+/**
+ * For a condition that is "new since the last tick": raises `${name}:${id}` for each group of rows in `table`
+ * stamped in `at` after the cursor. The cursor lives in `pipework_alert_source_state`, shared by every process;
+ * the first evaluation only stores it, so history is not replayed. Each evaluation reads rows with
+ * `cursor < at <= clock_timestamp() - settleMs` and moves the cursor to that upper bound, so a row stamped with
+ * its transaction's start time that commits within `settleMs` is still read. `settleMs` must exceed the longest
+ * transaction that writes the table. It has no `owns`: the condition clears by the handle's quiet period
+ * (`resolveAfterQuietMs` on `alerts.create`).
+ */
+export function newRowsSource(options: NewRowsSourceOptions): AlertSource {
+  assertSourceName(options.name, 'alerts.source.newRows')
+  const settleMs = options.settleMs ?? 30_000
+  if (typeof settleMs !== 'number' || !Number.isFinite(settleMs) || settleMs < 0) {
+    throw new Error(`[pipework] alerts.source.newRows settleMs must be a non-negative number of milliseconds, got ${String(settleMs)}.`)
+  }
+  const table = quoteIdentifier(options.table, 'newRows table')
+  const at = `t.${quoteIdentifier(options.at, 'newRows at column')}`
+  const group = typeof options.groupBy === 'string' ? sql.raw(`t.${quoteIdentifier(options.groupBy, 'newRows groupBy column')}`) : options.groupBy
+  const total = options.sum === undefined ? sql`0` : sql.raw(`sum(t.${quoteIdentifier(options.sum, 'newRows sum column')})`)
+  const where = options.where === undefined ? sql`` : sql`AND (${options.where})`
+  const stateKey = options.stateKey ?? options.name
+  const severity = options.severity ?? 'error'
+  const prefix = `${options.name}:`
+  return {
+    name: options.name,
+    async evaluate() {
+      const db = pipeSystem(options.database)
+      const first = await db.execute(sql`
+        INSERT INTO pipework_alert_source_state (name, cursor_at)
+        VALUES (${stateKey}, clock_timestamp() - ${settleMs} * interval '1 millisecond')
+        ON CONFLICT (name) DO NOTHING RETURNING name`)
+      if (first.length > 0) return []
+      const rows = await db.execute(sql`
+        WITH cur AS (
+          SELECT cursor_at, clock_timestamp() - ${settleMs} * interval '1 millisecond' AS upto
+          FROM pipework_alert_source_state WHERE name = ${stateKey} FOR UPDATE),
+        totals AS (
+          SELECT (${group})::text AS gid, count(*)::int AS n, (${total})::float8 AS total
+          FROM ${sql.raw(table)} t, cur
+          WHERE ${sql.raw(at)} > cur.cursor_at AND ${sql.raw(at)} <= cur.upto ${where}
+          GROUP BY 1),
+        newest AS (
+          SELECT DISTINCT ON (1) (${group})::text AS gid, to_jsonb(t) AS last
+          FROM ${sql.raw(table)} t, cur
+          WHERE ${sql.raw(at)} > cur.cursor_at AND ${sql.raw(at)} <= cur.upto ${where}
+          ORDER BY 1, ${sql.raw(at)} DESC),
+        found AS (
+          SELECT totals.gid AS id, n, total, last FROM totals JOIN newest ON newest.gid IS NOT DISTINCT FROM totals.gid),
+        moved AS (
+          UPDATE pipework_alert_source_state s SET cursor_at = GREATEST(s.cursor_at, cur.upto)
+          FROM cur WHERE s.name = ${stateKey} RETURNING 1)
+        SELECT id, n, total, last, (SELECT count(*) FROM moved) AS moved FROM found ORDER BY id`)
+      return rows.map((r: Row): AlertCondition => {
+        const last = (typeof r['last'] === 'string' ? JSON.parse(r['last']) : r['last']) as Row
+        const g: NewRowsGroup = { id: r['id'] === null ? '(none)' : String(r['id']), count: Number(r['n']), last, sum: Number(r['total']) }
+        const subject = options.subject?.(g)
+        const context = options.context?.(g)
+        return {
+          key: `${prefix}${g.id}`,
+          severity,
+          message: options.message(g),
+          ...(subject !== undefined ? { subject } : {}),
+          ...ownerFields(options),
+          ...(context !== undefined ? { context } : {}),
+        }
+      })
+    },
+  }
+}
+
+function jobsTable(table: string | undefined): string {
+  const name = table ?? 'pipework_jobs'
+  assertValidIdentifier(name, 'queue table')
+  return name
+}
+
+/**
+ * Raises `job-dead-letter:<jobType>` for jobs that exhausted their attempts since the last
+ * evaluation (any process: the position is kept in `pipework_alert_source_state`; the first
+ * evaluation starts from now, so history is not replayed). The condition clears only by the
+ * handle's quiet period, so set `resolveAfterQuietMs` on `alerts.create`. `types` limits it to those job
+ * types and `excludeTypes` leaves them out (not both, neither empty); each table and each filter keeps its own position.
+ * Jobs are read `settleMs` (default 30_000) after they dead-letter, so one that commits late is not missed.
+ * Conditions carry `owner`, `runbook` and a context of `count` and `lastError`.
+ */
+export function deadLetterSource(options: TableOption & { severity?: AlertSeverity; types?: readonly string[]; excludeTypes?: readonly string[]; owner?: string; runbook?: string; settleMs?: number } = {}): AlertSource {
+  if (options.types !== undefined && options.excludeTypes !== undefined) {
+    throw new Error('[pipework] alerts.source.deadLetter takes types or excludeTypes, not both.')
+  }
+  for (const opt of ['types', 'excludeTypes'] as const) {
+    if (options[opt] !== undefined && options[opt].length === 0) {
+      throw new Error(`[pipework] alerts.source.deadLetter ${opt} must not be empty; omit it to include every job type.`)
+    }
+  }
+  const jobs = jobsTable(options.table)
+  const name = 'job-dead-letter'
+  const mode = options.types !== undefined ? 'only' : options.excludeTypes !== undefined ? 'except' : null
+  const listed = [...new Set(options.types ?? options.excludeTypes ?? [])].sort()
+  const joined = listed.join(',')
+  const base = jobs === 'pipework_jobs' ? name : `${name}@${jobs}`
+  const stateKey = mode === null ? base
+    : `${base}:${mode}:${joined.length > 200 ? createHash('sha256').update(joined).digest('hex') : joined}`
+  const listJson = JSON.stringify(listed)
+  const typeFilter = mode === 'only' ? sql`AND t.job_type IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
+    : mode === 'except' ? sql`AND t.job_type NOT IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
+    : sql``
+  const lastError = (g: NewRowsGroup): string | null => {
+    const payload = g.last['error_payload']
+    const message = typeof payload === 'object' && payload !== null ? (payload as Row)['message'] : undefined
+    return message === null || message === undefined ? null : String(message)
+  }
+  return newRowsSource({
+    name,
+    stateKey,
+    table: jobs,
+    at: 'completed_at',
+    where: sql`t.status = 'dead_letter' ${typeFilter}`,
+    groupBy: 'job_type',
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    ...(options.severity !== undefined ? { severity: options.severity } : {}),
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    ...ownerFields(options),
+    message: g => `${String(g.count)} ${g.id} job(s) dead-lettered` + (lastError(g) !== null ? `; last error: ${lastError(g)!}` : ''),
+    subject: g => ({ entity: 'job', entityId: g.id }),
+    context: g => ({ count: g.count, lastError: lastError(g) }),
+  })
+}
+
+/** Raises `job-long-running:<jobType>` while a job of that type has been claimed for longer than `olderThanMs`; resolves when none is. */
+export function longRunningSource(options: TableOption & { olderThanMs: number; severity?: AlertSeverity }): AlertSource {
+  const jobs = quoteIdentifier(jobsTable(options.table), 'queue table')
+  return querySource({
+    name: 'job-long-running',
+    severity: options.severity ?? 'warning',
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    query: sql`
+      SELECT job_type, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(claimed_at)) * 1000)::bigint AS oldest_ms
+      FROM ${sql.raw(jobs)}
+      WHERE status IN ('claimed', 'running')
+        AND claimed_at < clock_timestamp() - ${options.olderThanMs} * interval '1 millisecond'
+      GROUP BY job_type ORDER BY job_type`,
+    id: r => String(r['job_type']),
+    message: r => `${String(r['n'])} ${String(r['job_type'])} job(s) running for more than ${String(options.olderThanMs)} ms (oldest ${String(r['oldest_ms'])} ms)`,
+    subject: r => ({ entity: 'job', entityId: String(r['job_type']) }),
+  })
+}
+
+/**
+ * Raises `queue-backlog:<jobType>` while more than `maxPending` jobs of a type are ready to claim, or the
+ * oldest has waited more than `maxOldestAgeMs`; resolves when the type is under both. Each condition carries
+ * `owner` and `runbook` and a context of `ready`, `running` and `oldestReadyWaitMs`.
+ */
+export function backlogSource(options: TableOption & { maxPending?: number; maxOldestAgeMs?: number; severity?: AlertSeverity; owner?: string; runbook?: string }): AlertSource {
+  if (options.maxPending === undefined && options.maxOldestAgeMs === undefined) {
+    throw new Error('[pipework] alerts.source.backlog needs maxPending, maxOldestAgeMs or both.')
+  }
+  const jobs = quoteIdentifier(jobsTable(options.table), 'queue table')
+  const maxPending = options.maxPending ?? null
+  const maxAge = options.maxOldestAgeMs ?? null
+  return querySource({
+    name: 'queue-backlog',
+    severity: options.severity ?? 'warning',
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    ...ownerFields(options),
+    query: sql`
+      SELECT job_type, n, running, oldest_ms FROM (
+        SELECT job_type,
+               (count(*) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp())))::int AS n,
+               (count(*) FILTER (WHERE status IN ('claimed', 'running')))::int AS running,
+               (extract(epoch FROM clock_timestamp() - min(queued_at) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp()))) * 1000)::bigint AS oldest_ms
+        FROM ${sql.raw(jobs)}
+        WHERE status IN ('pending', 'claimed', 'running')
+        GROUP BY job_type) p
+      WHERE (${maxPending}::int IS NOT NULL AND n > ${maxPending}::int)
+         OR (${maxAge}::bigint IS NOT NULL AND oldest_ms > ${maxAge}::bigint)
+      ORDER BY job_type`,
+    id: r => String(r['job_type']),
+    message: r => `${String(r['n'])} ${String(r['job_type'])} job(s) waiting (${String(r['running'])} running), the oldest for ${String(Number(r['oldest_ms']))} ms`,
+    subject: r => ({ entity: 'job', entityId: String(r['job_type']) }),
+    context: r => ({ ready: Number(r['n']), running: Number(r['running']), oldestReadyWaitMs: Number(r['oldest_ms']) }),
+  })
+}
+
+/** Raises `health:<name>` while a check reports unhealthy or throws; resolves when it is healthy again. Degraded is not a fault. */
+export function healthSource(checks: readonly HealthCheck[], options: { severity?: AlertSeverity } = {}): AlertSource {
+  const severity = options.severity ?? 'error'
+  const keys = new Set(checks.map(c => `health:${c.name}`))
+  return {
+    name: 'health',
+    owns: key => keys.has(key),
+    async evaluate() {
+      const found: AlertCondition[] = []
+      for (const check of checks) {
+        let message: string | null = null
+        try {
+          const result = await check.check()
+          if (result.status === 'unhealthy') message = `${check.name} is unhealthy${result.message !== undefined ? `: ${result.message}` : ''}`
+        } catch (err) {
+          message = `${check.name} check threw: ${describeError(err).message}`
+        }
+        if (message !== null) found.push({ key: `health:${check.name}`, severity, message, subject: { entity: 'health', entityId: check.name } })
+      }
+      return found
+    },
+  }
+}
+
+/**
+ * Raises `unit-fault:<unit>` for each systemd unit with an abnormal stop in the last `window` ms (rows written by
+ * `pipework alerts unit-fault`), with the count and last result; resolves once a unit has no fault in the window.
+ * `units` limits it to those units.
+ */
+export function unitFaultSource(options: { window: number; owner?: string; runbook?: string; severity?: AlertSeverity; units?: readonly string[]; database?: string }): AlertSource {
+  if (typeof options.window !== 'number' || !Number.isFinite(options.window) || options.window <= 0) {
+    throw new Error('[pipework] alerts.source.unitFault needs window to be a positive number of milliseconds.')
+  }
+  if (options.units !== undefined && options.units.length === 0) {
+    throw new Error('[pipework] alerts.source.unitFault units must not be empty; omit it to watch every unit.')
+  }
+  const units = options.units === undefined ? null : JSON.stringify(options.units)
+  const text = (v: unknown): string | null => v === null || v === undefined ? null : String(v)
+  return querySource({
+    name: 'unit-fault',
+    severity: options.severity ?? 'error',
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    ...ownerFields(options),
+    query: sql`
+      SELECT unit, count(*)::int AS n,
+             (array_agg(result ORDER BY at DESC))[1] AS last_result,
+             (array_agg(exit_code ORDER BY at DESC))[1] AS last_exit_code,
+             (array_agg(exit_status ORDER BY at DESC))[1] AS last_exit_status,
+             min(at) AS first_at, max(at) AS last_at,
+             ceil(extract(epoch FROM clock_timestamp() - min(at)) / 60)::int AS minutes
+      FROM pipework_unit_faults
+      WHERE at > clock_timestamp() - ${options.window} * interval '1 millisecond'
+        AND (${units}::jsonb IS NULL OR unit IN (SELECT jsonb_array_elements_text(${units}::jsonb)))
+      GROUP BY unit ORDER BY unit`,
+    id: r => String(r['unit']),
+    message: r => `${String(r['unit'])}: ${String(r['last_result'])} x${String(Number(r['n']))} in ${String(Math.max(1, Number(r['minutes'])))} min`,
+    subject: r => ({ entity: 'unit', entityId: String(r['unit']) }),
+    context: r => ({
+      count: Number(r['n']),
+      lastResult: String(r['last_result']),
+      lastExitCode: text(r['last_exit_code']),
+      lastExitStatus: text(r['last_exit_status']),
+      firstAt: new Date(r['first_at'] as string | Date).toISOString(),
+      lastAt: new Date(r['last_at'] as string | Date).toISOString(),
+    }),
+  })
+}
+
+const pad2 = (n: number): string => String(n).padStart(2, '0')
+
+function untilText(reset: Date, now: Date): string {
+  const day = reset.toISOString().slice(0, 10)
+  const hhmm = `${pad2(reset.getUTCHours())}:${pad2(reset.getUTCMinutes())}Z`
+  return day === now.toISOString().slice(0, 10) ? hhmm : `${day} ${hhmm}`
+}
+
+/**
+ * Raises `quota-exhausted:<key>` while an upstream quota row (needs `quota` configured) has remaining at or
+ * under `floor` (default 0) before its reset, or refused a call since it was last recorded; resolves when the
+ * row recovers or its reset passes. `keys` limits it to those keys (a non-empty array or a RegExp). Conditions
+ * carry `owner`, `runbook` and a context of `key`, `remaining`, `limit`, `resetAt` and `refused`.
+ */
+export function quotaExhaustedSource(options: { owner?: string; runbook?: string; keys?: readonly string[] | RegExp; floor?: number; severity?: AlertSeverity; database?: string } = {}): AlertSource {
+  const floor = options.floor ?? 0
+  if (!Number.isInteger(floor) || floor < 0) {
+    throw new Error(`[pipework] alerts.source.quotaExhausted floor must be a non-negative integer, got ${String(floor)}.`)
+  }
+  const keys = options.keys
+  if (Array.isArray(keys) && keys.length === 0) {
+    throw new Error('[pipework] alerts.source.quotaExhausted keys must not be empty; omit it to watch every key.')
+  }
+  const keyFilter = Array.isArray(keys)
+    ? sql`AND key IN (SELECT jsonb_array_elements_text(${JSON.stringify([...new Set(keys as readonly string[])])}::jsonb))`
+    : sql``
+  const base = querySource({
+    name: 'quota-exhausted',
+    severity: options.severity ?? 'error',
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    ...ownerFields(options),
+    query: sql`
+      SELECT key, remaining, "limit", reset_at,
+             (remaining > ${floor} AND refused_at IS NOT NULL AND refused_at >= updated_at) AS refused_only,
+             (refused_at IS NOT NULL AND refused_at >= updated_at) AS refused
+      FROM outbound_quota
+      WHERE reset_at > now() ${keyFilter}
+        AND (remaining <= ${floor} OR (refused_at IS NOT NULL AND refused_at >= updated_at))
+      ORDER BY key`,
+    id: r => String(r['key']),
+    message: r => {
+      const remaining = Number(r['remaining'])
+      const limit = r['limit'] === null ? null : Number(r['limit'])
+      const head = limit === null ? `${String(r['key'])} ${String(remaining)} remaining` : `${String(r['key'])} ${String(remaining)}/${String(limit)}`
+      return `${head} until ${untilText(new Date(r['reset_at'] as string | Date), new Date())}${r['refused_only'] === true ? '; a call was refused' : ''}`
+    },
+    subject: r => ({ entity: 'quota', entityId: String(r['key']) }),
+    context: r => ({
+      key: String(r['key']),
+      remaining: Number(r['remaining']),
+      limit: r['limit'] === null ? null : Number(r['limit']),
+      resetAt: new Date(r['reset_at'] as string | Date).toISOString(),
+      refused: r['refused'] === true,
+    }),
+  })
+  if (!(keys instanceof RegExp)) return base
+  return {
+    ...base,
+    async evaluate() {
+      return (await base.evaluate()).filter(c => {
+        keys.lastIndex = 0
+        return keys.test(c.key.slice('quota-exhausted:'.length))
+      })
+    },
+  }
+}
+
+/**
+ * Raises `loop-stalled:<name>` for each background loop (rows in `pipework_loop_beats`, written by the alerts tick,
+ * the job scheduler, reaper and claim loop, and the support events drain) that has not run, or not succeeded, within
+ * its allowance: `max(3 * its expected interval, 60 s)`, or `maxAge[name]` ms. `alerts.tick` is always left out, since
+ * a stopped tick cannot report itself; `pipework alerts stale` and `alerts.loopCheck` cover it from outside.
+ * `exclude` leaves other loops out. Conditions carry a context of `name`, `lastAt`, `lastOkAt`, `lastError`, `expectEveryMs` and `allowanceMs`.
+ */
+export function loopStalledSource(options: { maxAge?: Readonly<Record<string, number>>; owner?: string; runbook?: string; severity?: AlertSeverity; exclude?: readonly string[]; database?: string } = {}): AlertSource {
+  const severity = options.severity ?? 'error'
+  const exclude = [ALERTS_TICK_LOOP, ...(options.exclude ?? [])]
+  return {
+    name: 'loop-stalled',
+    owns: key => key.startsWith('loop-stalled:'),
+    async evaluate() {
+      const rows = await pipeSystem(options.database).execute(sql.raw(LOOP_BEATS_QUERY))
+      return staleLoops(rows, { ...(options.maxAge !== undefined ? { maxAge: options.maxAge } : {}), exclude }).map(l => ({
+        key: `loop-stalled:${l.name}`,
+        severity,
+        message: l.message,
+        subject: { entity: 'loop', entityId: l.name },
+        ...(options.owner !== undefined ? { owner: options.owner } : {}),
+        ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+        context: { name: l.name, lastAt: l.lastAt, lastOkAt: l.lastOkAt, lastError: l.lastError, expectEveryMs: l.expectEveryMs, allowanceMs: l.allowanceMs },
+      }))
+    },
+  }
+}
+
+/**
+ * A health check named `loops` for `/ready`: unhealthy while any included loop (default all, `alerts.tick` too) is
+ * stale, with the loops and their ages in the message. Inside a managed context it uses that instance; for
+ * `/ready` pass the `instance` (the route runs outside one).
+ */
+export function loopCheck(options: { maxAge?: Readonly<Record<string, number>>; include?: readonly string[]; database?: string; instance?: Manifold } = {}): HealthCheck {
+  return {
+    name: 'loops',
+    async check() {
+      const instance = options.instance ?? getContext()?.instance
+      if (instance === undefined) {
+        throw new Error('[pipework] alerts.loopCheck needs { instance } when it runs outside a managed context (as /ready does).')
+      }
+      const name = options.database ?? instance.config.defaultDatabaseName()
+      const db = instance.pool.getOrCreate(instance.config.database(name)).drizzle
+      const rows = await db.execute(sql.raw(LOOP_BEATS_QUERY))
+      if (rows.length === 0) return { status: 'unhealthy', message: 'no loop has ever recorded a beat' }
+      const stale = staleLoops(rows, {
+        ...(options.maxAge !== undefined ? { maxAge: options.maxAge } : {}),
+        ...(options.include !== undefined ? { include: options.include } : {}),
+      })
+      if (stale.length === 0) return { status: 'healthy' }
+      return { status: 'unhealthy', message: stale.map(l => l.message).join('; ') }
+    },
+  }
+}