pipework 0.284.0 → 0.286.0

package/src/infra/alerts/statement-sources.ts

@@ -0,0 +1,94 @@
+// An alert source over settled statements: per `pipe.labelled` label, the p90 of its last N durations against a budget.
+
+import type { StatementSettledEvent, StatementSettledFeed } from '../../core/config/statement-observer.js'
+import type { AlertSeverity } from './handle.js'
+import type { AlertCondition, AlertSource } from './sources.js'
+
+export interface StatementTimingOptions {
+  /** Budget in ms per label: `{ 'gate-read': 1000 }`. A label not named here is never recorded. At least one. */
+  readonly budgets: Readonly<Record<string, number>>
+  /** N: statements per label the p90 is taken over. The source stays quiet for a label until N have settled. */
+  readonly window: number
+  /** Default `warning`, as the backlog source. */
+  readonly severity?: AlertSeverity
+  readonly owner?: string
+  readonly runbook?: string
+}
+
+/** What `alerts.source.statementTiming` returns: an `AlertSource` that is also passed as `observability.statementTiming`. */
+export type StatementTimingSource = AlertSource & StatementSettledFeed
+
+interface Ring { readonly samples: number[]; next: number }
+
+/** Nearest rank: the smallest sample that at least 90% of the samples are at or under. */
+function p90(samples: readonly number[]): number {
+  const sorted = [...samples].sort((a, b) => a - b)
+  return sorted[Math.ceil(sorted.length * 0.9) - 1]!
+}
+
+/**
+ * Raises `statement-slow:<label>` while the p90 duration of a label's last `window` settled statements is over
+ * that label's budget; resolves when it is back under. Statements are fed in-process by
+ * `observability.statementTiming` (pass the returned source there as well as to `alerts.create({ sources })`;
+ * the app's own `onStatementSettled` still fires). Only the labels in `budgets` are recorded, each in a ring of
+ * `window` numbers, so memory is bounded by `window` times the labels named; unlabelled statements and other
+ * labels are ignored. Failed statements count, with the time they took. The window is this process's: a tick sees
+ * only statements that settled in its own process, and a label that stops being called keeps its last window.
+ * Context carries `label`, `p90Ms`, `window` and `budgetMs`.
+ */
+export function statementTimingSource(options: StatementTimingOptions): StatementTimingSource {
+  const who = 'alerts.source.statementTiming'
+  if (!Number.isInteger(options.window) || options.window < 1) {
+    throw new Error(`[pipework] ${who} window must be a positive integer number of statements, got ${String(options.window)}.`)
+  }
+  const entries = Object.entries(options.budgets)
+  if (entries.length === 0) throw new Error(`[pipework] ${who} needs budgets: at least one label and its budget in ms.`)
+  const budgets = new Map<string, number>()
+  for (const [label, budget] of entries) {
+    if (label === '') throw new Error(`[pipework] ${who} budgets: a label must not be empty.`)
+    if (typeof budget !== 'number' || !Number.isFinite(budget) || budget <= 0) {
+      throw new Error(`[pipework] ${who} budget for "${label}" must be a positive number of milliseconds, got ${String(budget)}.`)
+    }
+    budgets.set(label, budget)
+  }
+  const size = options.window
+  const severity = options.severity ?? 'warning'
+  const rings = new Map<string, Ring>()
+  let attached = false
+  const prefix = 'statement-slow:'
+
+  return {
+    name: 'statement-slow',
+    owns: key => key.startsWith(prefix),
+    observe(event: StatementSettledEvent): void {
+      if (event.label === null || !budgets.has(event.label)) return
+      let ring = rings.get(event.label)
+      if (ring === undefined) { ring = { samples: [], next: 0 }; rings.set(event.label, ring) }
+      if (ring.samples.length < size) ring.samples.push(event.durationMs)
+      else { ring.samples[ring.next] = event.durationMs; ring.next = (ring.next + 1) % size }
+    },
+    attach(): void { attached = true },
+    evaluate(): Promise<AlertCondition[]> {
+      if (!attached) {
+        return Promise.reject(new Error(`[pipework] ${who} was never given to the blueprint: set observability.statementTiming to this source, or it sees no statements.`))
+      }
+      const out: AlertCondition[] = []
+      for (const [label, budget] of budgets) {
+        const ring = rings.get(label)
+        if (ring === undefined || ring.samples.length < size) continue
+        const value = p90(ring.samples)
+        if (value <= budget) continue
+        out.push({
+          key: `${prefix}${label}`,
+          severity,
+          message: `p90 of the last ${String(size)} "${label}" statements is ${String(value)} ms, over its ${String(budget)} ms budget`,
+          subject: { entity: 'statement-label', entityId: label },
+          ...(options.owner !== undefined ? { owner: options.owner } : {}),
+          ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+          context: { label, p90Ms: value, window: size, budgetMs: budget },
+        })
+      }
+      return Promise.resolve(out)
+    },
+  }
+}