pipework 0.252.0 → 0.252.1

package/src/infra/alerts/beat.ts

@@ -0,0 +1,123 @@
+// The heartbeat of each background loop: one row per loop name in pipework_loop_beats, written at the end of every
+// iteration, and the rule that decides when a loop has stopped.
+
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import type { Manifold } from '../../core/pipework.js'
+
+/** The loops pipework itself records. */
+export const LOOP_NAMES = ['alerts.tick', 'jobs.scheduler', 'jobs.reaper', 'jobs.claim', 'support.events'] as const
+export const ALERTS_TICK_LOOP = 'alerts.tick'
+export const DEFAULT_LOOP_EVERY_MS = 60_000
+const MIN_ALLOWANCE_MS = 60_000
+const MAX_ERROR_LENGTH = 500
+
+export interface BeatOutcome {
+  readonly ok: boolean
+  readonly error?: string
+  readonly everyMs: number
+}
+
+/** Records one iteration of a loop. Never throws: a failed beat is logged and the loop carries on. */
+export async function recordBeat(db: DB | (() => DB), name: string, outcome: BeatOutcome): Promise<void> {
+  try {
+    const conn = typeof db === 'function' ? db() : db
+    const ok = outcome.ok
+    const error = ok ? null : (outcome.error ?? 'failed').slice(0, MAX_ERROR_LENGTH)
+    await conn.execute(sql`
+      INSERT INTO pipework_loop_beats (name, last_at, last_ok_at, last_error, expect_every_ms)
+      VALUES (${name}, clock_timestamp(), ${ok ? sql`clock_timestamp()` : sql`NULL`}, ${error}, ${Math.round(outcome.everyMs)})
+      ON CONFLICT (name) DO UPDATE SET
+        last_at = EXCLUDED.last_at,
+        last_ok_at = CASE WHEN ${ok} THEN EXCLUDED.last_ok_at ELSE pipework_loop_beats.last_ok_at END,
+        last_error = EXCLUDED.last_error,
+        expect_every_ms = EXCLUDED.expect_every_ms`)
+  } catch (err) {
+    try {
+      getBaseLogger().warn({ err, loop: name }, 'loop beat could not be recorded')
+    } catch { /* logging must not break the loop either */ }
+  }
+}
+
+/**
+ * `recordBeat` for a loop in a process that is not a pipework surface: writes on the manifold's default database,
+ * outside any Flow, and does nothing where the blueprint has no `alerts`.
+ */
+export function recordBeatFor(instance: Manifold, name: string, outcome: BeatOutcome): Promise<void> {
+  if (!beatsEnabled(instance.config)) return Promise.resolve()
+  return recordBeat(() => instance.pool.getOrCreate(instance.config.database()).drizzle, name, outcome)
+}
+
+/** Whether a blueprint has the `alerts` flag, which brings the beat table. */
+export function beatsEnabled(config: { readonly raw: unknown }): boolean {
+  const raw = config.raw as { readonly alerts?: unknown } | null | undefined
+  return raw?.alerts !== undefined
+}
+
+/** Every beat row with its ages measured on the database's clock; the one query behind the source, the check and the CLI. */
+export const LOOP_BEATS_QUERY = `
+  SELECT name, last_at, last_ok_at, last_error, expect_every_ms,
+         (extract(epoch FROM clock_timestamp() - last_at) * 1000)::bigint AS since_at_ms,
+         (extract(epoch FROM clock_timestamp() - last_ok_at) * 1000)::bigint AS since_ok_ms
+  FROM pipework_loop_beats ORDER BY name`
+
+export interface StaleLoop {
+  readonly name: string
+  readonly message: string
+  readonly lastAt: string
+  readonly lastOkAt: string | null
+  readonly lastError: string | null
+  readonly expectEveryMs: number
+  readonly allowanceMs: number
+  readonly ageMs: number
+}
+
+export function formatAge(ms: number): string {
+  const s = Math.max(0, Math.floor(ms / 1000))
+  if (s >= 3600) return `${String(Math.floor(s / 3600))} h`
+  if (s >= 60) return `${String(Math.floor(s / 60))} min`
+  return `${String(s)} s`
+}
+
+export function allowanceFor(name: string, expectEveryMs: number, maxAge: Readonly<Record<string, number>> | undefined): number {
+  const override = maxAge?.[name]
+  return override ?? Math.max(3 * expectEveryMs, MIN_ALLOWANCE_MS)
+}
+
+const iso = (v: unknown): string => new Date(v as string | Date).toISOString()
+
+/** The loops among `rows` (the result of LOOP_BEATS_QUERY) that are stale under the allowance rule. */
+export function staleLoops(
+  rows: readonly Record<string, unknown>[],
+  options: { maxAge?: Readonly<Record<string, number>>; include?: readonly string[]; exclude?: readonly string[] } = {},
+): StaleLoop[] {
+  const out: StaleLoop[] = []
+  for (const r of rows) {
+    const name = String(r['name'])
+    if (options.include !== undefined && !options.include.includes(name)) continue
+    if (options.exclude?.includes(name) === true) continue
+    const expectEveryMs = Number(r['expect_every_ms'])
+    const allowanceMs = allowanceFor(name, expectEveryMs, options.maxAge)
+    const sinceAt = Number(r['since_at_ms'])
+    const lastOk = r['last_ok_at'] === null || r['last_ok_at'] === undefined ? null : iso(r['last_ok_at'])
+    const sinceOk = lastOk === null ? null : Number(r['since_ok_ms'])
+    if (sinceOk !== null && sinceOk <= allowanceMs) continue
+    const lastError = r['last_error'] === null || r['last_error'] === undefined ? null : String(r['last_error'])
+    // Not run at all within the allowance, or running but failing (a loop that never succeeded is aged from its last run).
+    const notRun = sinceAt > allowanceMs
+    const ageMs = notRun ? sinceAt : (sinceOk ?? sinceAt)
+    const message = notRun
+      ? `${name} has not run in ${formatAge(ageMs)}`
+      : `${name} has not succeeded in ${formatAge(ageMs)}; last error: ${lastError ?? 'none'}`
+    out.push({ name, message, lastAt: iso(r['last_at']), lastOkAt: lastOk, lastError, expectEveryMs, allowanceMs, ageMs })
+  }
+  return out
+}
+
+/** `90s`, `5m` or `2h` as milliseconds; null when it is none of those. */
+export function parseDuration(text: string): number | null {
+  const m = /^(\d+)(s|m|h)$/.exec(text.trim())
+  if (m === null) return null
+  return Number(m[1]) * ({ s: 1000, m: 60_000, h: 3_600_000 }[m[2] as 's' | 'm' | 'h'])
+}

package/src/infra/alerts/client-faults.ts

@@ -0,0 +1,111 @@
+import { Builder } from '../../request/di/builder.js'
+import type { Handler } from '../../request/di/types.js'
+import type { CorsConfig } from '../../request/http/types.js'
+import { schema } from '../../data/schema/namespace.js'
+import { getContext } from '../../request/context/store.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { recordFault, faultMessage } from './faults.js'
+import { __flowStep } from '../../trace/runtime/flow-step.js'
+import type { ClientFaultReport } from '../../client/report-fault.js'
+import { CLIENT_FAULT_MESSAGE_MAX, CLIENT_FAULT_STACK_MAX } from '../../client/report-fault.js'
+import type { Alerts, JsonValue } from './handle.js'
+import { BadRequestError } from './http.js'
+
+export type { ClientFaultReport }
+
+export interface ClientFaultsOptions {
+  /** Route path, e.g. '/client-faults'. No trailing slash. */
+  readonly path: string
+  /** The apps that may report, by name. At least one. A report names one in `app`. */
+  readonly apps: Readonly<Record<string, { readonly owner?: string; readonly runbook?: string }>>
+  /** The alert key for a report. Default `client-fault:<app>:<where>`. */
+  readonly key?: (report: ClientFaultReport) => string
+  /**
+   * The owner and runbook for one report, over the app's: each field returned replaces the app's, an undefined
+   * field or result keeps it. A throw is handled as a throwing `key`: logged and recorded, nothing raised, still 204.
+   */
+  readonly assign?: (report: ClientFaultReport) => { readonly owner?: string; readonly runbook?: string } | undefined
+  /** This route's CORS grant (see `Builder.cors`). */
+  readonly cors?: CorsConfig
+  /** A `rateLimits` class for the route. Default: the class of a route with optional auth. */
+  readonly rateLimit?: string
+}
+
+const c = schema.check
+const reportSchema = (apps: readonly string[]) => c.strictObject({
+  app: c.string().refine(a => apps.includes(a), `must be one of ${apps.join(', ')}`),
+  where: c.string().min(1).max(100).regex(/^[A-Za-z0-9._:/-]+$/),
+  kind: c.enum(['error', 'rejection', 'render', 'request']),
+  severity: c.enum(['warning', 'error']).optional(),
+  code: c.string().max(100).optional(),
+  message: c.string().min(1).max(CLIENT_FAULT_MESSAGE_MAX),
+  stack: c.string().max(CLIENT_FAULT_STACK_MAX).optional(),
+  route: c.string().max(200).startsWith('/').refine(r => !/[?#]|:\/\//.test(r), 'must be a route pattern, not a page URL').optional(),
+  release: c.string().max(100).optional(),
+  traceId: c.uuid().optional(),
+  requestId: c.string().min(1).max(100).regex(/^[A-Za-z0-9._:-]+$/).optional(),
+})
+
+const REF = 'pipework/alerts:client-fault'
+
+/**
+ * One POST route at `options.path` that takes failures reported by browser code (see `reportClientFault` in
+ * `pipework/client`) and answers 204. No session is needed; the tenant and user are used when there is one.
+ * Each report is a trace step `pipework/alerts:client-fault` capturing the report, and raises
+ * `client-fault:<app>:<where>` (or `options.key`) with the app's owner and runbook, or the report's own from `options.assign`. The stack stays in the
+ * step's capture, not the alert. A body that breaks the contract is 400 and raises nothing.
+ */
+export function clientFaults(handle: Alerts, options: ClientFaultsOptions): Handler<Record<string, unknown>, unknown>[] {
+  if (!options.path.startsWith('/') || options.path.length < 2 || options.path.endsWith('/')) {
+    throw new Error(`alerts.clientFaults: path "${options.path}" must start with "/" and have no trailing slash, e.g. "/client-faults".`)
+  }
+  if (Object.keys(options.apps).length === 0) {
+    throw new Error('alerts.clientFaults: apps needs at least one entry, name -> { owner?, runbook? }.')
+  }
+
+  const record = __flowStep({ ref: REF, capture: [null] }, async (report: ClientFaultReport): Promise<void> => {
+    const app = options.apps[report.app]!
+    const flow = getContext()
+    const context: Record<string, JsonValue> = { app: report.app, where: report.where, kind: report.kind }
+    for (const field of ['code', 'route', 'release', 'traceId', 'requestId'] as const) {
+      if (report[field] !== undefined) context[field] = report[field]
+    }
+    try {
+      const key = options.key?.(report) ?? `client-fault:${report.app}:${report.where}`
+      const assigned = options.assign?.(report)
+      const owner = assigned?.owner ?? app.owner
+      const runbook = assigned?.runbook ?? app.runbook
+      await handle.raise(key, {
+        severity: report.severity ?? 'error',
+        message: `${report.app} ${report.where}: ${report.message}`,
+        subject: { ...(flow?.tenant != null ? { tenantId: flow.tenant } : {}), entity: 'client', entityId: report.where },
+        ...(owner !== undefined ? { owner } : {}),
+        ...(runbook !== undefined ? { runbook } : {}),
+        context,
+      })
+    } catch (err) {
+      getBaseLogger().error({ err, app: report.app, where: report.where }, 'Failed to raise a client fault')
+      void recordFault('client-fault-raise', faultMessage(err), { app: report.app })
+    }
+  })
+
+  const reportShape = reportSchema(Object.keys(options.apps))
+  let builder = new Builder().authOptional().input(c.unknown())
+  if (options.cors !== undefined) builder = builder.cors(options.cors)
+  if (options.rateLimit !== undefined) builder = builder.rateLimit(options.rateLimit)
+
+  return [
+    builder.route('POST', options.path).fit(async ({ input }) => {
+      const parsed = reportShape.safeParse(input)
+      if (!parsed.success) {
+        const issue = parsed.error.issues[0]
+        const path = issue?.path.join('.') ?? ''
+        const field = path === '' ? 'body' : path
+        throw new BadRequestError(`Invalid ${field}: ${issue?.message ?? 'does not match the client fault contract'}.`, { field })
+      }
+      const report = parsed.data as ClientFaultReport
+      await record(report)
+      return { status: 204 }
+    }) as unknown as Handler<Record<string, unknown>, unknown>,
+  ]
+}

package/src/infra/alerts/faults.ts

@@ -0,0 +1,156 @@
+// Failures that are swallowed and only logged leave no trace an alert could read. `recordFault` writes one row per
+// such failure to pipework_faults, on the system connection and outside any transaction, so it survives a rollback.
+
+import { sql } from '../../data/db/sql.js'
+import type { DB } from '../../data/db/index.js'
+import { pipeSystem } from '../../data/db/db.js'
+import { getContext } from '../../request/context/store.js'
+import type { Manifold } from '../../core/pipework.js'
+import { getBaseLogger } from '../../core/logging/index.js'
+import { describeError } from '../../core/logging/error-chain.js'
+import { redactText, redactJson } from './redact.js'
+import { beatsEnabled } from './beat.js'
+import type { AlertSeverity, JsonValue } from './handle.js'
+import { newRowsSource, type AlertSource } from './sources.js'
+
+const MAX_MESSAGE_LENGTH = 2000
+
+interface FaultTarget {
+  readonly db: DB | (() => DB)
+  readonly tenant?: string | null
+}
+
+const targets: FaultTarget[] = []
+
+const isManifold = (v: unknown): v is Manifold =>
+  typeof v === 'object' && v !== null && 'pool' in v && 'config' in v
+
+/**
+ * Registers the connection faults raised outside any Flow are written on (a worker's loops, or a custom drain
+ * process); returns the release. Pass the manifold to use its default database, resolved when a fault is written; it
+ * registers nothing where the blueprint has no `alerts`. A `DB` or a function returning one is used as given.
+ */
+export function useFaultTarget(target: Manifold | DB | (() => DB)): () => void {
+  let db: DB | (() => DB)
+  if (isManifold(target)) {
+    if (!beatsEnabled(target.config)) return () => undefined
+    db = () => target.pool.getOrCreate(target.config.database()).drizzle
+  } else {
+    db = target
+  }
+  const entry: FaultTarget = { db }
+  targets.push(entry)
+  return () => {
+    const i = targets.indexOf(entry)
+    if (i >= 0) targets.splice(i, 1)
+  }
+}
+
+function resolveTarget(): { db: DB | (() => DB); tenant: string | null } | null {
+  const flow = getContext()
+  if (flow !== undefined) {
+    if (!beatsEnabled(flow.instance.config)) return null
+    return { db: () => pipeSystem(), tenant: flow.tenant }
+  }
+  const t = targets[targets.length - 1]
+  return t === undefined ? null : { db: t.db, tenant: null }
+}
+
+const warnedDropped = new Set<string>()
+
+function warnDropped(kind: string): void {
+  if (warnedDropped.has(kind)) return
+  warnedDropped.add(kind)
+  try {
+    getBaseLogger().warn({ kind }, `fault "${kind}" dropped: no Flow and no fault target; call alerts.useFaultTarget(manifold) at boot`)
+  } catch { /* logging must not break the caller */ }
+}
+
+/**
+ * Records a failure the code swallowed, for `alerts.source.faults` to raise. Call it as `void recordFault(...)` next
+ * to the log line. Best effort: it never throws, does nothing where the blueprint has no `alerts`, writes outside the
+ * caller's transaction, and records the current tenant. `kind` is a short stable name; `message` is cut at 2000 characters. Email addresses are redacted from both before they are stored.
+ */
+export function recordFault(kind: string, message: string, context?: Readonly<Record<string, JsonValue>>): Promise<void> {
+  return write(resolveTarget, kind, message, context, true)
+}
+
+/** `recordFault` for code that holds the manifold but may run outside any Flow (a trace flush). */
+export function recordFaultFor(instance: Manifold, kind: string, message: string, context?: Readonly<Record<string, JsonValue>>): Promise<void> {
+  const flow = getContext()
+  return write(() => {
+    if (!beatsEnabled(instance.config)) return null
+    return {
+      db: () => instance.pool.getOrCreate(instance.config.database()).drizzle,
+      tenant: flow?.tenant ?? null,
+    }
+  }, kind, message, context, false)
+}
+
+async function write(
+  resolve: () => { db: DB | (() => DB); tenant: string | null } | null,
+  kind: string,
+  message: string,
+  context: Readonly<Record<string, JsonValue>> | undefined,
+  warnWhenUnresolved: boolean,
+): Promise<void> {
+  try {
+    const target = resolve()
+    if (target === null) {
+      if (warnWhenUnresolved && getContext() === undefined) warnDropped(kind)
+      return
+    }
+    const conn = typeof target.db === 'function' ? target.db() : target.db
+    await conn.execute(sql`
+      INSERT INTO pipework_faults (kind, message, context, tenant_id, recorded_at)
+      VALUES (${kind}, ${redactText(message).slice(0, MAX_MESSAGE_LENGTH)},
+        ${context === undefined ? null : JSON.stringify(redactJson(context))}::jsonb, ${target.tenant}, clock_timestamp())`)
+  } catch (err) {
+    try {
+      getBaseLogger().warn({ err, kind }, 'fault could not be recorded')
+    } catch { /* logging must not break the caller */ }
+  }
+}
+
+/** The text of a caught value, for a fault message: its message with its causes' text folded in. */
+export function faultMessage(err: unknown): string {
+  return describeError(err).message
+}
+
+interface Basics { owner?: string; runbook?: string; severity?: AlertSeverity; database?: string }
+
+/**
+ * Raises `fault:<kind>` for failures recorded by `recordFault` since the last tick, one condition per kind: message
+ * `<count> <kind> fault(s); last: <message>`, context `count`, `lastMessage` and `lastContext`. `kinds` limits it to
+ * those kinds (default all). `settleMs` is as in `alerts.source.newRows` (default 30 s). Clears by the handle's quiet
+ * period. Needs `alerts` configured, which brings the `pipework_faults` table.
+ */
+export function faultsSource(options: Basics & { kinds?: readonly string[]; settleMs?: number } = {}): AlertSource {
+  const kinds = options.kinds === undefined ? null : [...new Set(options.kinds)]
+  if (kinds !== null && kinds.length === 0) throw new Error('[pipework] alerts.source.faults kinds must not be empty; omit it to watch every kind.')
+  if (options.settleMs !== undefined && (typeof options.settleMs !== 'number' || !Number.isFinite(options.settleMs) || options.settleMs < 0)) {
+    throw new Error(`[pipework] alerts.source.faults settleMs must be a non-negative number of milliseconds, got ${String(options.settleMs)}.`)
+  }
+  const where = kinds === null ? undefined : sql`t.kind IN (SELECT jsonb_array_elements_text(${JSON.stringify(kinds)}::jsonb))`
+  const text = (v: unknown): string => v === null || v === undefined ? 'none' : String(v)
+  return newRowsSource({
+    name: 'fault',
+    stateKey: kinds === null ? 'fault' : `fault:${[...kinds].sort().join(',')}`,
+    table: 'pipework_faults',
+    at: 'recorded_at',
+    groupBy: 'kind',
+    ...(where !== undefined ? { where } : {}),
+    ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+    ...(options.severity !== undefined ? { severity: options.severity } : {}),
+    ...(options.owner !== undefined ? { owner: options.owner } : {}),
+    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+    ...(options.database !== undefined ? { database: options.database } : {}),
+    message: g => `${String(g.count)} ${g.id} fault(s); last: ${text(g.last['message'])}`,
+    subject: g => ({ entity: 'fault', entityId: g.id }),
+    context: g => ({
+      count: g.count,
+      lastMessage: text(g.last['message']),
+      lastContext: (g.last['context'] ?? null) as JsonValue,
+    }),
+  })
+}