pipework 0.252.0 → 0.253.0

package/src/infra/access-links/index.ts

@@ -0,0 +1,9 @@
+export { createAccessLinks } from './handle.js'
+export type {
+  AccessLinks, AccessLinksConfig, AccessLink, PublicLink, AccessLinkUse, AccessLinkTarget, AccessLinkListed, AccessLinkPage,
+  AccessLinkCheck, AccessLinkCheckArgs, LinkOptions, ResolveResult, BeginUseResult,
+  CompleteUseResult, AbandonUseResult,
+} from './handle.js'
+export type { CodeConfig } from './code.js'
+export type { RateLimitConfig } from './handle.js'
+export type { PersonCheck, PersonCheckConfig, PersonChallenge, PersonPass } from './person-check.js'

package/src/infra/access-links/namespace.ts

@@ -0,0 +1,30 @@
+import { createAccessLinks } from './handle.js'
+import { personCheck } from './person-check.js'
+
+/** Access links namespace — short revocable bearer links to a tenant's resource. */
+export const accessLinks = {
+  /** Creates an access-link handle: tenant-scoped management plus the public resolve / beginUse / completeUse / abandonUse. */
+  create: createAccessLinks,
+  /**
+   * A self-hosted person check for `requireChecks`: ALTCHA-format proof-of-work challenges bound to a link code
+   * and an expiry, HMAC-signed with `secret` (32 bytes or more, the same in every process). A solution is accepted
+   * once across all processes (`access_link_check_spent`, in the `accessLinks` tables) and yields a short-lived
+   * signed pass that is the proof for the link's check at `beginUse`. `personCheck.solve(challenge)` solves a
+   * challenge in code, for tests.
+   *
+   * The app writes two public routes: GET returns `check.challenge(code)` for the widget (`challengeurl`, or
+   * `challengejson`); POST gives `check.verify(code, payload)` the widget's payload and returns the pass, or refuses
+   * when it is `null`. The page carries the ALTCHA widget (MIT; serve its script yourself, no third party sees the
+   * visitor), which solves in the background and falls back to a keyboard and screen-reader checkbox. pipework
+   * does not ship the widget.
+   *
+   * ```ts
+   * const person = accessLinks.personCheck({ secret: config.linkSecret })
+   * const links = accessLinks.create({ permissions: ['view'], checks: { person: person.check } })
+   * await links.link({ target, permissions: ['view'], requireChecks: ['person'] })
+   * // public route: begin with the pass the visitor got from POST
+   * await links.beginUse(code, { kind: 'view', proof: { person: pass } })
+   * ```
+   */
+  personCheck,
+} as const

package/src/infra/access-links/person-check.ts

@@ -0,0 +1,177 @@
+import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto'
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import type { AccessLinkCheck } from './handle.js'
+
+const ALGORITHM = 'SHA-256'
+const MIN_SECRET_BYTES = 32
+const MAX_NUMBER_LIMIT = 10_000_000
+const PRUNE_BATCH = 100
+
+/** A proof-of-work challenge in the shape the ALTCHA widget takes. */
+export interface PersonChallenge {
+  readonly algorithm: 'SHA-256'
+  readonly challenge: string
+  readonly maxnumber: number
+  readonly salt: string
+  readonly signature: string
+}
+
+export interface PersonCheckConfig {
+  /** Signs challenges and passes; 32 bytes or more, the same in every process. */
+  readonly secret: string | Buffer
+  /** The largest number the browser may have to try; the work is half this on average. Default 100_000; 1 to 10_000_000. */
+  readonly maxNumber?: number
+  /** How long a challenge can be solved for. Default 10 minutes. */
+  readonly challengeMs?: number
+  /** How long a pass is good for. Default 30 minutes. */
+  readonly passMs?: number
+  /** Database holding `access_link_check_spent`; default the default database. */
+  readonly database?: string
+}
+
+export interface PersonPass {
+  readonly pass: string
+  readonly expiresAt: Date
+}
+
+export interface PersonCheck {
+  /** A challenge for one link's code, for the widget. */
+  challenge(code: string): PersonChallenge
+  /**
+   * Accepts a widget solution (its base64 payload) for `code` once, across processes, and returns a pass;
+   * `null` for anything else: malformed, tampered, expired, for another link, or already spent. Never throws on bad input.
+   * Needs a Flow (it records the spent solution through `pipe.system()`).
+   */
+  verify(code: string, solution: unknown): Promise<PersonPass | null>
+  /**
+   * True when `pass` is a pass issued for `code`, unexpired and correctly signed: the same answer `check` gives at
+   * `beginUse`. Use it on a public route right after `resolve(code)` to refuse a bad pass before answering anything
+   * about the link. Synchronous; no database, no Flow; never throws.
+   */
+  checkPass(code: string, pass: unknown): boolean
+  /** The `AccessLinkCheck` to register under a name in `accessLinks.create({ checks })`: the proof is a pass issued for this link. */
+  readonly check: AccessLinkCheck
+}
+
+function sha256Hex(data: string): string {
+  return createHash('sha256').update(data).digest('hex')
+}
+
+function sameBytes(a: Buffer, b: Buffer): boolean {
+  return a.length === b.length && a.length > 0 && timingSafeEqual(a, b)
+}
+
+function sameHex(a: string, b: string): boolean {
+  return sameBytes(Buffer.from(a, 'hex'), Buffer.from(b, 'hex'))
+}
+
+function positive(name: string, value: number): number {
+  if (!Number.isFinite(value) || value <= 0) throw new Error(`personCheck: ${name} must be a positive number`)
+  return value
+}
+
+function createPersonCheck(config: PersonCheckConfig): PersonCheck {
+  const secret = typeof config.secret === 'string' ? Buffer.from(config.secret, 'utf8') : config.secret
+  if (!Buffer.isBuffer(secret) || secret.length < MIN_SECRET_BYTES) {
+    throw new Error(`personCheck: secret must be ${String(MIN_SECRET_BYTES)} bytes or more, the same in every process (no random fallback: a per-process secret breaks passes across processes)`)
+  }
+  const maxNumber = config.maxNumber ?? 100_000
+  if (!Number.isInteger(maxNumber) || maxNumber < 1 || maxNumber > MAX_NUMBER_LIMIT) {
+    throw new Error(`personCheck: maxNumber must be an integer from 1 to ${String(MAX_NUMBER_LIMIT)}`)
+  }
+  const challengeMs = positive('challengeMs', config.challengeMs ?? 600_000)
+  const passMs = positive('passMs', config.passMs ?? 1_800_000)
+  const database = config.database
+
+  const key = (purpose: 'challenge' | 'pass'): Buffer =>
+    createHmac('sha256', secret).update(`pipework access links person check: ${purpose}`).digest()
+  const challengeKey = key('challenge')
+  const passKey = key('pass')
+
+  function challenge(code: string): PersonChallenge {
+    const expires = Math.floor((Date.now() + challengeMs) / 1000)
+    const salt = `${randomBytes(12).toString('hex')}?expires=${String(expires)}&link=${encodeURIComponent(code)}`
+    const secretNumber = randomBytes(4).readUInt32BE() % (maxNumber + 1)
+    const digest = sha256Hex(salt + String(secretNumber))
+    const signature = createHmac('sha256', challengeKey).update(digest).digest('hex')
+    return { algorithm: ALGORITHM, challenge: digest, maxnumber: maxNumber, salt, signature }
+  }
+
+  function issuePass(code: string): PersonPass {
+    const exp = Date.now() + passMs
+    const body = Buffer.from(JSON.stringify({ code, exp })).toString('base64url')
+    const mac = createHmac('sha256', passKey).update(body).digest('base64url')
+    return { pass: `${body}.${mac}`, expiresAt: new Date(exp) }
+  }
+
+  // The challenge's expiry in ms when `solution` solves a live challenge made for `code`; otherwise null.
+  function solvedUntil(code: string, solution: unknown): { id: string; untilMs: number } | null {
+    if (typeof solution !== 'string') return null
+    let payload: unknown
+    try {
+      payload = JSON.parse(Buffer.from(solution, 'base64').toString('utf8'))
+    } catch {
+      return null
+    }
+    if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) return null
+    const { algorithm, challenge: id, number, salt, signature } = payload as Record<string, unknown>
+    if (algorithm !== ALGORITHM || typeof id !== 'string' || typeof salt !== 'string' || typeof signature !== 'string') return null
+    if (typeof number !== 'number' || !Number.isSafeInteger(number) || number < 0) return null
+    const params = new URLSearchParams(salt.split('?')[1] ?? '')
+    const expires = Number(params.get('expires'))
+    if (!Number.isFinite(expires) || expires * 1000 <= Date.now() || params.get('link') !== code) return null
+    const expected = createHmac('sha256', challengeKey).update(id).digest('hex')
+    if (!sameHex(expected, signature)) return null
+    if (!sameHex(sha256Hex(salt + String(number)), id)) return null
+    return { id, untilMs: expires * 1000 }
+  }
+
+  async function verify(code: string, solution: unknown): Promise<PersonPass | null> {
+    const solved = solvedUntil(code, solution)
+    if (solved === null) return null
+    const db = pipeSystem(database)
+    const spent = await db.execute(sql`
+      INSERT INTO access_link_check_spent (challenge, expires_at)
+      VALUES (${solved.id}, ${new Date(solved.untilMs).toISOString()})
+      ON CONFLICT DO NOTHING RETURNING 1 AS spent`)
+    if (spent.length === 0) return null
+    // Kept an hour past expiry so clock skew between processes and the database never reopens a solution.
+    await db.execute(sql`
+      DELETE FROM access_link_check_spent WHERE ctid IN (
+        SELECT ctid FROM access_link_check_spent WHERE expires_at < now() - interval '1 hour' LIMIT ${PRUNE_BATCH})`)
+    return issuePass(code)
+  }
+
+  function checkPass(code: string, pass: unknown): boolean {
+    if (typeof pass !== 'string') return false
+    const parts = pass.split('.')
+    const [body, mac] = parts
+    if (parts.length !== 2 || body === undefined || mac === undefined) return false
+    const expected = createHmac('sha256', passKey).update(body).digest()
+    if (!sameBytes(Buffer.from(mac, 'base64url'), expected)) return false
+    try {
+      const claims = JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) as { code?: unknown; exp?: unknown }
+      return typeof claims.exp === 'number' && claims.exp > Date.now() && claims.code === code
+    } catch {
+      return false
+    }
+  }
+
+  const check: AccessLinkCheck = ({ link, proof }) => Promise.resolve(checkPass(link.code, proof))
+
+  return { challenge, verify, checkPass, check }
+}
+
+/** Solves a challenge the way the widget does and returns its base64 payload. For tests: it does the work the browser would. */
+function solvePersonChallenge(c: PersonChallenge): string {
+  for (let n = 0; n <= c.maxnumber; n++) {
+    if (sha256Hex(c.salt + String(n)) === c.challenge) {
+      const payload = { algorithm: c.algorithm, challenge: c.challenge, number: n, salt: c.salt, signature: c.signature, took: 0 }
+      return Buffer.from(JSON.stringify(payload)).toString('base64')
+    }
+  }
+  throw new Error('personCheck.solve: challenge has no solution')
+}
+
+export const personCheck = Object.assign(createPersonCheck, { solve: solvePersonChallenge })

package/src/infra/access-links/schema.ts

@@ -0,0 +1,71 @@
+// The four tables behind access links. `access_link_code` is deliberately NOT
+// tenant-scoped: it maps a code to its tenant so a request with no tenant can
+// find where to look, and holds nothing else. `access_link` and
+// `access_link_use` carry a `.tenant()` key, so generated migrations put them
+// under forced row-level security.
+
+import { define } from '../../data/domain/index.js'
+import { field } from '../../data/domain/field.js'
+import { schema } from '../../data/schema/namespace.js'
+import { index } from '../../data/db/idx/index.js'
+
+export const ACCESS_LINK_USE_STATUSES = ['pending', 'completed', 'abandoned'] as const
+
+export const accessLinkCode = define('access_link_code', {
+  code: field.text().primaryKey(),
+  tenantId: field.uuid(),
+  linkId: field.uuid(),
+  createdAt: field.timestamp().default('now'),
+})
+
+export const accessLink = define('access_link', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  tenantId: field.uuid().tenant(),
+  code: field.text().unique(),
+  targetType: field.text(),
+  targetId: field.text(),
+  permissions: field.jsonb(schema.check.json()),
+  note: field.text().nullable(),
+  createdBy: field.text().nullable(),
+  createdAt: field.timestamp().default('now'),
+  expiresAt: field.timestamp().nullable(),
+  maxUses: field.integer().nullable(),
+  requireChecks: field.jsonb(schema.check.json()).default([]),
+  checkData: field.jsonb(schema.check.json()).nullable(),
+  revokedAt: field.timestamp().nullable(),
+  revokedBy: field.text().nullable(),
+})
+
+export const accessLinkUse = define('access_link_use', {
+  id: field.uuid().defaultRandom().primaryKey(),
+  tenantId: field.uuid().tenant(),
+  linkId: field.uuid(),
+  kind: field.text(),
+  status: field.enum(ACCESS_LINK_USE_STATUSES),
+  startedAt: field.timestamp().default('now'),
+  reservedUntil: field.timestamp(),
+  completedAt: field.timestamp().nullable(),
+  client: field.text().nullable(),
+  location: field.text().nullable(),
+  detail: field.jsonb(schema.check.json()).nullable(),
+  claimedAt: field.timestamp().nullable(),
+}, {
+  indexes: (t) => ({
+    byLink: index('idx_access_link_use_link').on(t['linkId'], t['status']),
+  }),
+})
+
+// Person-check solutions already accepted, kept until the challenge has expired:
+// one solved challenge makes one pass, across processes. Not tenant-scoped.
+export const accessLinkCheckSpent = define('access_link_check_spent', {
+  challenge: field.text().primaryKey(),
+  expiresAt: field.timestamp(),
+})
+
+// Codes that matched no link, counted per client in a fixed window: the guessing
+// limit, shared by every process. Not tenant-scoped.
+export const accessLinkMiss = define('access_link_miss', {
+  client: field.text().primaryKey(),
+  count: field.integer().default(0),
+  windowStart: field.timestamp().default('now'),
+})

package/src/infra/alerts/async-sources.ts

@@ -0,0 +1,231 @@
+// Sources over the failures the async subsystems already record: cron schedules, step graphs, pipelines and
+// trace expiry. Each is a composition of `querySource` and `newRowsSource`.
+
+import { sql } from '../../data/db/sql.js'
+import { composeSources, newRowsSource, querySource, type AlertSource } from './sources.js'
+import { common, positive, text, FIFTEEN_MIN, HOUR, type Basics } from './source-options.js'
+
+const FIVE_MIN = 5 * 60_000
+const TRACE_SWEEP_GRACE = 26 * HOUR
+
+/** The `error` jsonb of a recorded failure as one line: its `message`, `error` or `step`/`error` pair, else the JSON. */
+function errorText(v: unknown): string | null {
+  if (v === null || v === undefined) return null
+  const parsed = typeof v === 'string' ? safeParse(v) : v
+  if (typeof parsed === 'string') return parsed
+  if (typeof parsed === 'object' && parsed !== null) {
+    const o = parsed as Record<string, unknown>
+    const inner = o['message'] ?? o['error']
+    if (typeof inner === 'string') return typeof o['step'] === 'string' ? `${o['step']}: ${inner}` : inner
+    if (typeof inner === 'object' && inner !== null) return errorText(inner)
+  }
+  return JSON.stringify(parsed)
+}
+
+function safeParse(s: string): unknown {
+  try { return JSON.parse(s) } catch { return s }
+}
+
+/**
+ * Raises `schedule-overdue:<schedule name>` while a cron schedule's `next_run_at` is more than `graceMs` (default 5 min)
+ * in the past. Every scheduler pass moves `next_run_at` into the future, whether it fires the job or skips it because
+ * the previous run is still pending, so an overdue row means no scheduler is running (or it cannot reach the row).
+ * Resolves when the scheduler catches up. Needs a worker surface that declares schedules (the `pipework_schedules` table; no blueprint key).
+ */
+export function schedulesSource(options: Basics & { graceMs?: number } = {}): AlertSource {
+  const grace = positive(options.graceMs ?? FIVE_MIN, 'graceMs', 'alerts.source.schedules')
+  return querySource({
+    ...common(options),
+    name: 'schedule-overdue',
+    query: sql`
+      SELECT name, next_run_at, last_run_at, (extract(epoch FROM clock_timestamp() - next_run_at) * 1000)::bigint AS overdue_ms
+      FROM pipework_schedules
+      WHERE next_run_at < clock_timestamp() - ${grace} * interval '1 millisecond'
+      ORDER BY name`,
+    id: r => String(r['name']),
+    message: r => `schedule ${String(r['name'])} is ${String(r['overdue_ms'])} ms past its next run; is the scheduler running?`,
+    subject: r => ({ entity: 'schedule', entityId: String(r['name']) }),
+    context: r => ({
+      schedule: String(r['name']),
+      overdueMs: Number(r['overdue_ms']),
+      nextRunAt: new Date(String(r['next_run_at'])).toISOString(),
+      lastRunAt: r['last_run_at'] === null ? null : new Date(String(r['last_run_at'])).toISOString(),
+    }),
+  })
+}
+
+/**
+ * Raises `step-run-failed:<graph>` for step runs that failed since the last tick (message with the count and the last
+ * error; context `count`, `lastError`, `lastRunId`, `lastTenantId`), `step-run-unnotified:<graph>` while runs finished
+ * more than `notifyAfterMs` (default 15 min) ago still have `notified = false` (the graph's `onDone` has not run), and
+ * `step-task-waiting:<graph>` while a task has waited for a person's decision more than `waitingAfterMs` (default 1 h);
+ * the wait is measured from its run's `created_at`, as tasks carry no timestamp, and `step-run-stalled:<graph>` while runs
+ * are still `running` more than `runningAfterMs` (default 1 h) after `created_at` with no task `waiting` (a `waiting` task is a
+ * person's decision and is covered above; `blocked` and `queued` tasks are work in flight, so a run stuck on them is stalled). A failed task always fails its run in the
+ * same transaction, so failed runs cover failed tasks. The failed condition clears by the handle's quiet period; the other two
+ * resolve when the rows drain. Needs a step graph defined in the app (the `pipework_step_run` and `pipework_step_task` tables; no blueprint key).
+ */
+export function stepsSource(options: Basics & { notifyAfterMs?: number; waitingAfterMs?: number; runningAfterMs?: number; settleMs?: number } = {}): AlertSource {
+  const notifyAfter = positive(options.notifyAfterMs ?? FIFTEEN_MIN, 'notifyAfterMs', 'alerts.source.steps')
+  const waitingAfter = positive(options.waitingAfterMs ?? HOUR, 'waitingAfterMs', 'alerts.source.steps')
+  const runningAfter = positive(options.runningAfterMs ?? HOUR, 'runningAfterMs', 'alerts.source.steps')
+  const failed = newRowsSource({
+    ...common(options),
+    name: 'step-run-failed',
+    table: 'pipework_step_run',
+    at: 'finished_at',
+    where: sql`t.status = 'failed'`,
+    groupBy: 'graph',
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    message: g => `${String(g.count)} run(s) of step graph ${g.id} failed; last error: ${text(errorText(g.last['error']))}`,
+    context: g => ({
+      graph: g.id,
+      count: g.count,
+      lastError: errorText(g.last['error']),
+      lastRunId: String(g.last['id']),
+      lastTenantId: String(g.last['tenant_id']),
+    }),
+  })
+  const unnotified = querySource({
+    ...common(options),
+    name: 'step-run-unnotified',
+    query: sql`
+      SELECT graph, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(finished_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_step_run
+      WHERE status = 'done' AND notified = false
+        AND finished_at < clock_timestamp() - ${notifyAfter} * interval '1 millisecond'
+      GROUP BY graph ORDER BY graph`,
+    id: r => String(r['graph']),
+    message: r => `${String(r['n'])} finished run(s) of step graph ${String(r['graph'])} not notified for more than ${String(notifyAfter)} ms (oldest ${String(r['oldest_ms'])} ms)`,
+    subject: r => ({ entity: 'step-graph', entityId: String(r['graph']) }),
+    context: r => ({ graph: String(r['graph']), count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  const waiting = querySource({
+    ...common(options),
+    name: 'step-task-waiting',
+    query: sql`
+      SELECT r.graph, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(r.created_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_step_task k JOIN pipework_step_run r ON r.id = k.run_id
+      WHERE k.status = 'waiting' AND r.status = 'running'
+        AND r.created_at < clock_timestamp() - ${waitingAfter} * interval '1 millisecond'
+      GROUP BY r.graph ORDER BY r.graph`,
+    id: r => String(r['graph']),
+    message: r => `${String(r['n'])} task(s) of step graph ${String(r['graph'])} waiting more than ${String(waitingAfter)} ms (oldest run ${String(r['oldest_ms'])} ms old)`,
+    subject: r => ({ entity: 'step-graph', entityId: String(r['graph']) }),
+    context: r => ({ graph: String(r['graph']), count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  const stalled = querySource({
+    ...common(options),
+    name: 'step-run-stalled',
+    query: sql`
+      SELECT r.graph, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(r.created_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_step_run r
+      WHERE r.status = 'running'
+        AND r.created_at < clock_timestamp() - ${runningAfter} * interval '1 millisecond'
+        AND NOT EXISTS (SELECT 1 FROM pipework_step_task k WHERE k.run_id = r.id AND k.status = 'waiting')
+      GROUP BY r.graph ORDER BY r.graph`,
+    id: r => String(r['graph']),
+    message: r => `${String(r['n'])} run(s) of step graph ${String(r['graph'])} still running after more than ${String(runningAfter)} ms with no person step waiting (oldest ${String(r['oldest_ms'])} ms)`,
+    subject: r => ({ entity: 'step-graph', entityId: String(r['graph']) }),
+    context: r => ({ graph: String(r['graph']), count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  return composeSources('steps', [failed, unnotified, waiting, stalled])
+}
+
+/**
+ * Raises `pipeline-failed:<pipeline name>` for pipeline runs that failed since the last tick (context `count`,
+ * `lastError`, `lastRunId`, `lastTenantId`), and `pipeline-waiting:<pipeline name>` while runs sit in `waiting` more than
+ * `waitingAfterMs` (default 1 h) after they started (the table keeps no time for when a run began to wait, so the age is
+ * `started_at`), and `pipeline-stalled:<pipeline name>` while runs are `running` more than `runningAfterMs` (default 1 h)
+ * after `started_at` (the process died mid-run). The failed condition clears by the handle's quiet period; the others
+ * resolve when the runs resume or finish. Needs the app's own `pipework_pipeline_runs` table (pipelines have no blueprint key; the app creates it).
+ */
+export function pipelinesSource(options: Basics & { waitingAfterMs?: number; runningAfterMs?: number; settleMs?: number } = {}): AlertSource {
+  const waitingAfter = positive(options.waitingAfterMs ?? HOUR, 'waitingAfterMs', 'alerts.source.pipelines')
+  const runningAfter = positive(options.runningAfterMs ?? HOUR, 'runningAfterMs', 'alerts.source.pipelines')
+  const failed = newRowsSource({
+    ...common(options),
+    name: 'pipeline-failed',
+    table: 'pipework_pipeline_runs',
+    at: 'completed_at',
+    where: sql`t.status = 'failed'`,
+    groupBy: 'pipeline_name',
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    message: g => `${String(g.count)} run(s) of pipeline ${g.id} failed; last error: ${text(errorText(g.last['error']))}`,
+    context: g => ({
+      pipeline: g.id,
+      count: g.count,
+      lastError: errorText(g.last['error']),
+      lastRunId: String(g.last['id']),
+      lastTenantId: String(g.last['tenant_id']),
+    }),
+  })
+  const waiting = querySource({
+    ...common(options),
+    name: 'pipeline-waiting',
+    query: sql`
+      SELECT pipeline_name, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(started_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_pipeline_runs
+      WHERE status = 'waiting' AND started_at < clock_timestamp() - ${waitingAfter} * interval '1 millisecond'
+      GROUP BY pipeline_name ORDER BY pipeline_name`,
+    id: r => String(r['pipeline_name']),
+    message: r => `${String(r['n'])} run(s) of pipeline ${String(r['pipeline_name'])} waiting more than ${String(waitingAfter)} ms (oldest ${String(r['oldest_ms'])} ms)`,
+    subject: r => ({ entity: 'pipeline', entityId: String(r['pipeline_name']) }),
+    context: r => ({ pipeline: String(r['pipeline_name']), count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  const stalled = querySource({
+    ...common(options),
+    name: 'pipeline-stalled',
+    query: sql`
+      SELECT pipeline_name, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(started_at)) * 1000)::bigint AS oldest_ms
+      FROM pipework_pipeline_runs
+      WHERE status = 'running' AND started_at < clock_timestamp() - ${runningAfter} * interval '1 millisecond'
+      GROUP BY pipeline_name ORDER BY pipeline_name`,
+    id: r => String(r['pipeline_name']),
+    message: r => `${String(r['n'])} run(s) of pipeline ${String(r['pipeline_name'])} still running after more than ${String(runningAfter)} ms (oldest ${String(r['oldest_ms'])} ms)`,
+    subject: r => ({ entity: 'pipeline', entityId: String(r['pipeline_name']) }),
+    context: r => ({ pipeline: String(r['pipeline_name']), count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  return composeSources('pipelines', [failed, waiting, stalled])
+}
+
+/**
+ * Raises `trace-expired:<entry kind>` (http, job, cron, test) for traces the expiry sweep force-closed since the last
+ * tick: the sweep stamps `status = 'expired'` and `closed_at`, so `closed_at` is the cursor. Also raises
+ * `trace-expiry-undrained:all` while traces still `open` have an `expires_at` more than `graceMs` (default 26 h: the maintenance job that
+ * runs the sweep is daily, so a trace legitimately stays open up to a day past its deadline) in the past, which means
+ * nothing is running the expiry job. The expired condition clears by the handle's quiet period; the
+ * undrained one resolves once the sweep runs. Needs the blueprint's `trace` block (the `trace` table).
+ */
+export function tracesSource(options: Basics & { graceMs?: number; settleMs?: number } = {}): AlertSource {
+  const grace = positive(options.graceMs ?? TRACE_SWEEP_GRACE, 'graceMs', 'alerts.source.traces')
+  const expired = newRowsSource({
+    ...common(options),
+    name: 'trace-expired',
+    table: 'trace',
+    at: 'closed_at',
+    where: sql`t.status = 'expired'`,
+    groupBy: 'entry_kind',
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    message: g => `${String(g.count)} ${g.id} trace(s) force-closed after expiring; last trace ${String(g.last['id'])}`,
+    context: g => ({
+      entryKind: g.id,
+      count: g.count,
+      lastTraceId: String(g.last['id']),
+      lastTenantId: g.last['tenant_id'] === null || g.last['tenant_id'] === undefined ? null : String(g.last['tenant_id']),
+    }),
+  })
+  const undrained = querySource({
+    ...common(options),
+    name: 'trace-expiry-undrained',
+    query: sql`
+      SELECT count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(expires_at)) * 1000)::bigint AS oldest_ms
+      FROM "trace"
+      WHERE status = 'open' AND expires_at <= clock_timestamp() - ${grace} * interval '1 millisecond'
+      HAVING count(*) > 0`,
+    id: () => 'all',
+    message: r => `${String(r['n'])} open trace(s) expired more than ${String(grace)} ms ago and not closed; is the trace expiry job scheduled?`,
+    context: r => ({ count: Number(r['n']), oldestMs: Number(r['oldest_ms']) }),
+  })
+  return composeSources('traces', [expired, undrained])
+}

package/src/infra/alerts/audit.ts

@@ -0,0 +1,36 @@
+import { sql } from '../../data/db/sql.js'
+import { pipeSystem } from '../../data/db/db.js'
+import type { DB } from '../../data/db/index.js'
+import { boundTransaction } from '../../data/write/audit/bound-transaction.js'
+import { getContext } from '../../request/context/store.js'
+import { resolveActorId } from '../../data/write/audit/actor.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFault, faultMessage } from './faults.js'
+
+/**
+ * Writes an `Alert` audit record under the Flow's open trace step; does nothing when no step is open.
+ * In the Flow's transaction on that database when one is open (a savepoint, so a failed write cannot abort it; it
+ * commits or rolls back with it), else on the system channel.
+ */
+export async function auditAlert(verb: string, key: string, details: Record<string, unknown>, database?: string): Promise<void> {
+  const flow = getContext()
+  const trace = flow?.trace
+  const stepId = trace?.buffer.openStepId ?? null
+  if (flow === undefined || trace === undefined || trace === null || stepId === null) return
+  try {
+    const write = async (db: DB): Promise<void> => {
+      await db.execute(sql`
+        INSERT INTO audit_record
+          (trace_day, trace_id, trace_step_id, tenant_id, actor_id, entity, verb, entity_id, after, occurred_at)
+        VALUES (${trace.openedDay}, ${trace.id}, ${stepId}, ${flow.tenant}, ${resolveActorId(flow)},
+          'Alert', ${verb}, ${key}, ${JSON.stringify(details)}::jsonb, now())
+      `)
+    }
+    const tx = boundTransaction(flow, database)
+    if (tx === null) await write(pipeSystem(database))
+    else await tx.transaction(write)
+  } catch (err) {
+    getBaseLogger().error({ err, traceId: trace.id }, `Failed to write the alert ${verb} audit record`)
+    void recordFault('audit-write', faultMessage(err), { entity: 'Alert', verb })
+  }
+}