pipework 0.173.0 → 0.174.0

package/dist/infra/alerts/namespace.js.map

@@ -1 +1 @@
-{"version":3,"file":"namespace.js","sourceRoot":"","sources":["../../../src/infra/alerts/namespace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAEpK,oHAAoH;AACpH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,gCAAgC;IAChC,MAAM,EAAE,YAAY;IACpB,0HAA0H;IAC1H,IAAI,EAAE,UAAU;IAChB,wLAAwL;IACxL,YAAY;IACZ,oHAAoH;IACpH,aAAa,EAAE,iBAAiB;IAChC,2DAA2D;IAC3D,OAAO,EAAE,WAAW;IACpB,+DAA+D;IAC/D,MAAM,EAAE,UAAU;IAClB,kJAAkJ;IAClJ,SAAS;IACT,MAAM,EAAE;QACN,4EAA4E;QAC5E,UAAU,EAAE,gBAAgB;QAC5B,iFAAiF;QACjF,WAAW,EAAE,iBAAiB;QAC9B,mGAAmG;QACnG,OAAO,EAAE,aAAa;QACtB,yGAAyG;QACzG,cAAc,EAAE,oBAAoB;QACpC,yDAAyD;QACzD,MAAM,EAAE,YAAY;QACpB,6IAA6I;QAC7I,SAAS,EAAE,eAAe;QAC1B,gNAAgN;QAChN,WAAW,EAAE,iBAAiB;KAC/B;CACO,CAAA"}
+{"version":3,"file":"namespace.js","sourceRoot":"","sources":["../../../src/infra/alerts/namespace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,aAAa,EAAE,YAAY,EAAE,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAChM,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AAEjG,oHAAoH;AACpH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,gCAAgC;IAChC,MAAM,EAAE,YAAY;IACpB,0HAA0H;IAC1H,IAAI,EAAE,UAAU;IAChB,wLAAwL;IACxL,YAAY;IACZ,oHAAoH;IACpH,aAAa,EAAE,iBAAiB;IAChC,2DAA2D;IAC3D,OAAO,EAAE,WAAW;IACpB,+DAA+D;IAC/D,MAAM,EAAE,UAAU;IAClB,kJAAkJ;IAClJ,SAAS;IACT,MAAM,EAAE;QACN,4EAA4E;QAC5E,UAAU,EAAE,gBAAgB;QAC5B,iFAAiF;QACjF,WAAW,EAAE,iBAAiB;QAC9B,mGAAmG;QACnG,OAAO,EAAE,aAAa;QACtB,yGAAyG;QACzG,cAAc,EAAE,oBAAoB;QACpC,yDAAyD;QACzD,MAAM,EAAE,YAAY;QACpB,6IAA6I;QAC7I,SAAS,EAAE,eAAe;QAC1B,2HAA2H;QAC3H,KAAK,EAAE,WAAW;QAClB,iIAAiI;QACjI,OAAO,EAAE,aAAa;QACtB,0IAA0I;QAC1I,cAAc,EAAE,oBAAoB;QACpC,yJAAyJ;QACzJ,aAAa,EAAE,mBAAmB;QAClC,gIAAgI;QAChI,OAAO,EAAE,aAAa;QACtB,gNAAgN;QAChN,WAAW,EAAE,iBAAiB;KAC/B;CACO,CAAA"}

package/dist/infra/alerts/sources.d.ts

@@ -1,4 +1,5 @@
 import type { HealthCheck } from '../../core/lifecycle/types.js';
+import type { SQL } from '../../data/db/index.js';
 import type { Manifold } from '../../core/pipework.js';
 import type { AlertSeverity, JsonValue } from './handle.js';
 /** A condition a source finds true right now. */
@@ -29,12 +30,74 @@ interface TableOption {
     readonly table?: string;
     readonly database?: string;
 }
+type Row = Record<string, unknown>;
+interface SourceBasics {
+    readonly severity?: AlertSeverity;
+    readonly owner?: string;
+    readonly runbook?: string;
+    readonly database?: string;
+}
+/** One source from several: it evaluates each in turn and owns what any of them owns. */
+export declare function composeSources(name: string, parts: readonly AlertSource[]): AlertSource;
+export interface QuerySourceOptions extends SourceBasics {
+    /** Key prefix: conditions are keyed `${name}:${id}`. Non-empty, no ':'. */
+    readonly name: string;
+    /** Returns one row per subject that is in trouble right now. */
+    readonly query: SQL;
+    readonly id: (row: Row) => string;
+    readonly message: (row: Row) => string;
+    readonly context?: (row: Row) => Record<string, JsonValue>;
+    readonly subject?: (row: Row) => AlertCondition['subject'];
+}
+/**
+ * For a condition that is true right now: runs `query` (as the system, any tenant) on every tick and raises
+ * `${name}:${id}` for each row. An open alert under `${name}:` that a tick no longer returns is resolved.
+ */
+export declare function querySource(options: QuerySourceOptions): AlertSource;
+export interface NewRowsGroup {
+    readonly id: string;
+    readonly count: number;
+    readonly last: Row;
+    readonly sum: number;
+}
+export interface NewRowsSourceOptions extends SourceBasics {
+    /** Key prefix: conditions are keyed `${name}:${id}`; also the cursor's state key unless `stateKey` is given. */
+    readonly name: string;
+    /** The table holding the recorded failures (an identifier, not schema-qualified). */
+    readonly table: string;
+    /** The timestamp column a row is stamped with when the failure is recorded. */
+    readonly at: string;
+    /** Extra filter; the table is aliased `t`. */
+    readonly where?: SQL;
+    /** A column of the table, or an SQL expression over alias `t`; one condition per distinct value. */
+    readonly groupBy: string | SQL;
+    readonly message: (group: NewRowsGroup) => string;
+    readonly context?: (group: NewRowsGroup) => Record<string, JsonValue>;
+    readonly subject?: (group: NewRowsGroup) => AlertCondition['subject'];
+    /** A numeric column to total per group, handed to the callbacks as `sum` (0 without it). */
+    readonly sum?: string;
+    /** Rows are read once they are older than this many ms (default 30_000); see `newRowsSource`. */
+    readonly settleMs?: number;
+    /** The cursor's key in `pipework_alert_source_state`; default `name`. */
+    readonly stateKey?: string;
+}
+/**
+ * For a condition that is "new since the last tick": raises `${name}:${id}` for each group of rows in `table`
+ * stamped in `at` after the cursor. The cursor lives in `pipework_alert_source_state`, shared by every process;
+ * the first evaluation only stores it, so history is not replayed. Each evaluation reads rows with
+ * `cursor < at <= clock_timestamp() - settleMs` and moves the cursor to that upper bound, so a row stamped with
+ * its transaction's start time that commits within `settleMs` is still read. `settleMs` must exceed the longest
+ * transaction that writes the table. It has no `owns`: the condition clears by the handle's quiet period
+ * (`resolveAfterQuietMs` on `alerts.create`).
+ */
+export declare function newRowsSource(options: NewRowsSourceOptions): AlertSource;
 /**
  * Raises `job-dead-letter:<jobType>` for jobs that exhausted their attempts since the last
  * evaluation (any process: the position is kept in `pipework_alert_source_state`; the first
  * evaluation starts from now, so history is not replayed). The condition clears only by the
  * handle's quiet period, so set `resolveAfterQuietMs` on `alerts.create`. `types` limits it to those job
  * types and `excludeTypes` leaves them out (not both, neither empty); each filter keeps its own position.
+ * Jobs are read `settleMs` (default 30_000) after they dead-letter, so one that commits late is not missed.
  * Conditions carry `owner`, `runbook` and a context of `count` and `lastError`.
  */
 export declare function deadLetterSource(options?: TableOption & {
@@ -43,6 +106,7 @@ export declare function deadLetterSource(options?: TableOption & {
     excludeTypes?: readonly string[];
     owner?: string;
     runbook?: string;
+    settleMs?: number;
 }): AlertSource;
 /** Raises `job-long-running:<jobType>` while a job of that type has been claimed for longer than `olderThanMs`; resolves when none is. */
 export declare function longRunningSource(options: TableOption & {

package/dist/infra/alerts/sources.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../../../src/infra/alerts/sources.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAI3D,iDAAiD;AACjD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAA;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC5E,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAA;CACvD;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;IAC3B,QAAQ,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;CACtC;AAED,UAAU,WAAW;IAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE;AAQ7E;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,WAAW,GAAG;IAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAqDrM;AAED,0IAA0I;AAC1I,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,WAAW,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAA;CAAE,GAAG,WAAW,CAqBvH;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAsC9K;AAED,oIAAoI;AACpI,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,EAAE,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,aAAa,CAAA;CAAO,GAAG,WAAW,CAqBpH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAgDlL;AAUD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAmDpM;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAmBtN;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;CAAO,GAAG,WAAW,CAoBvK"}
+{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../../../src/infra/alerts/sources.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAChE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAI3D,iDAAiD;AACjD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAA;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC5E,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAA;CACvD;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;IAC3B,QAAQ,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;CACtC;AAED,UAAU,WAAW;IAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE;AAE7E,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAElC,UAAU,YAAY;IAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE;AAe5I,yFAAyF;AACzF,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG,WAAW,CAWvF;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,gEAAgE;IAChE,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAA;IACnB,QAAQ,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,MAAM,CAAA;IACjC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,MAAM,CAAA;IACtC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAC1D,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,cAAc,CAAC,SAAS,CAAC,CAAA;CAC3D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,WAAW,CAuBpE;AAED,MAAM,WAAW,YAAY;IAAG,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE;AAEvH,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,gHAAgH;IAChH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,qFAAqF;IACrF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,8CAA8C;IAC9C,QAAQ,CAAC,KAAK,CAAC,EAAE,GAAG,CAAA;IACpB,oGAAoG;IACpG,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAA;IAC9B,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,MAAM,CAAA;IACjD,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IACrE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,cAAc,CAAC,SAAS,CAAC,CAAA;IACrE,4FAA4F;IAC5F,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;IACrB,iGAAiG;IACjG,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,WAAW,CA2DxE;AAQD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,WAAW,GAAG;IAAE,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAwCxN;AAED,0IAA0I;AAC1I,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,WAAW,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAA;CAAE,GAAG,WAAW,CAgBvH;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CA6B9K;AAED,oIAAoI;AACpI,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,EAAE,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,aAAa,CAAA;CAAO,GAAG,WAAW,CAqBpH;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAqClL;AAUD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAmDpM;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,WAAW,CAmBtN;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,CAAA;CAAO,GAAG,WAAW,CAoBvK"}

package/dist/infra/alerts/sources.js

@@ -4,10 +4,133 @@ import { pipeSystem } from '../../data/db/db.js';
 import { assertValidIdentifier, quoteIdentifier } from '../../data/db/identifiers.js';
 import { ALERTS_TICK_LOOP, LOOP_BEATS_QUERY, staleLoops } from './beat.js';
 import { getContext } from '../../request/context/store.js';
+function assertSourceName(name, who) {
+    if (typeof name !== 'string' || name === '' || name.includes(':')) {
+        throw new Error(`[pipework] ${who}: name must be a non-empty string without ':', got ${JSON.stringify(name)}.`);
+    }
+}
+function ownerFields(options) {
+    return {
+        ...(options.owner !== undefined ? { owner: options.owner } : {}),
+        ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
+    };
+}
+/** One source from several: it evaluates each in turn and owns what any of them owns. */
+export function composeSources(name, parts) {
+    const owning = parts.filter(p => p.owns !== undefined);
+    return {
+        name,
+        ...(owning.length > 0 ? { owns: (key) => owning.some(p => p.owns(key)) } : {}),
+        async evaluate() {
+            const out = [];
+            for (const part of parts)
+                out.push(...await part.evaluate());
+            return out;
+        },
+    };
+}
+/**
+ * For a condition that is true right now: runs `query` (as the system, any tenant) on every tick and raises
+ * `${name}:${id}` for each row. An open alert under `${name}:` that a tick no longer returns is resolved.
+ */
+export function querySource(options) {
+    assertSourceName(options.name, 'alerts.source.query');
+    const prefix = `${options.name}:`;
+    const severity = options.severity ?? 'error';
+    return {
+        name: options.name,
+        owns: key => key.startsWith(prefix),
+        async evaluate() {
+            const rows = await pipeSystem(options.database).execute(options.query);
+            return rows.map((r) => {
+                const subject = options.subject?.(r);
+                const context = options.context?.(r);
+                return {
+                    key: `${prefix}${options.id(r)}`,
+                    severity,
+                    message: options.message(r),
+                    ...(subject !== undefined ? { subject } : {}),
+                    ...ownerFields(options),
+                    ...(context !== undefined ? { context } : {}),
+                };
+            });
+        },
+    };
+}
+/**
+ * For a condition that is "new since the last tick": raises `${name}:${id}` for each group of rows in `table`
+ * stamped in `at` after the cursor. The cursor lives in `pipework_alert_source_state`, shared by every process;
+ * the first evaluation only stores it, so history is not replayed. Each evaluation reads rows with
+ * `cursor < at <= clock_timestamp() - settleMs` and moves the cursor to that upper bound, so a row stamped with
+ * its transaction's start time that commits within `settleMs` is still read. `settleMs` must exceed the longest
+ * transaction that writes the table. It has no `owns`: the condition clears by the handle's quiet period
+ * (`resolveAfterQuietMs` on `alerts.create`).
+ */
+export function newRowsSource(options) {
+    assertSourceName(options.name, 'alerts.source.newRows');
+    const settleMs = options.settleMs ?? 30_000;
+    if (typeof settleMs !== 'number' || !Number.isFinite(settleMs) || settleMs < 0) {
+        throw new Error(`[pipework] alerts.source.newRows settleMs must be a non-negative number of milliseconds, got ${String(settleMs)}.`);
+    }
+    const table = quoteIdentifier(options.table, 'newRows table');
+    const at = `t.${quoteIdentifier(options.at, 'newRows at column')}`;
+    const group = typeof options.groupBy === 'string' ? sql.raw(`t.${quoteIdentifier(options.groupBy, 'newRows groupBy column')}`) : options.groupBy;
+    const total = options.sum === undefined ? sql `0` : sql.raw(`sum(t.${quoteIdentifier(options.sum, 'newRows sum column')})`);
+    const where = options.where === undefined ? sql `` : sql `AND (${options.where})`;
+    const stateKey = options.stateKey ?? options.name;
+    const severity = options.severity ?? 'error';
+    const prefix = `${options.name}:`;
+    return {
+        name: options.name,
+        async evaluate() {
+            const db = pipeSystem(options.database);
+            const first = await db.execute(sql `
+        INSERT INTO pipework_alert_source_state (name, cursor_at)
+        VALUES (${stateKey}, clock_timestamp() - ${settleMs} * interval '1 millisecond')
+        ON CONFLICT (name) DO NOTHING RETURNING name`);
+            if (first.length > 0)
+                return [];
+            const rows = await db.execute(sql `
+        WITH cur AS (
+          SELECT cursor_at, clock_timestamp() - ${settleMs} * interval '1 millisecond' AS upto
+          FROM pipework_alert_source_state WHERE name = ${stateKey} FOR UPDATE),
+        totals AS (
+          SELECT (${group})::text AS gid, count(*)::int AS n, (${total})::float8 AS total
+          FROM ${sql.raw(table)} t, cur
+          WHERE ${sql.raw(at)} > cur.cursor_at AND ${sql.raw(at)} <= cur.upto ${where}
+          GROUP BY 1),
+        newest AS (
+          SELECT DISTINCT ON (1) (${group})::text AS gid, to_jsonb(t) AS last
+          FROM ${sql.raw(table)} t, cur
+          WHERE ${sql.raw(at)} > cur.cursor_at AND ${sql.raw(at)} <= cur.upto ${where}
+          ORDER BY 1, ${sql.raw(at)} DESC),
+        found AS (
+          SELECT totals.gid AS id, n, total, last FROM totals JOIN newest ON newest.gid IS NOT DISTINCT FROM totals.gid),
+        moved AS (
+          UPDATE pipework_alert_source_state s SET cursor_at = GREATEST(s.cursor_at, cur.upto)
+          FROM cur WHERE s.name = ${stateKey} RETURNING 1)
+        SELECT id, n, total, last, (SELECT count(*) FROM moved) AS moved FROM found ORDER BY id`);
+            return rows.map((r) => {
+                const last = (typeof r['last'] === 'string' ? JSON.parse(r['last']) : r['last']);
+                const g = { id: r['id'] === null ? '(none)' : String(r['id']), count: Number(r['n']), last, sum: Number(r['total']) };
+                const subject = options.subject?.(g);
+                const context = options.context?.(g);
+                return {
+                    key: `${prefix}${g.id}`,
+                    severity,
+                    message: options.message(g),
+                    ...(subject !== undefined ? { subject } : {}),
+                    ...ownerFields(options),
+                    ...(context !== undefined ? { context } : {}),
+                };
+            });
+        },
+    };
+}
 function jobsTable(table) {
     const name = table ?? 'pipework_jobs';
     assertValidIdentifier(name, 'queue table');
-    return quoteIdentifier(name, 'queue table');
+    return name;
 }
 /**
  * Raises `job-dead-letter:<jobType>` for jobs that exhausted their attempts since the last
@@ -15,6 +138,7 @@ function jobsTable(table) {
  * evaluation starts from now, so history is not replayed). The condition clears only by the
  * handle's quiet period, so set `resolveAfterQuietMs` on `alerts.create`. `types` limits it to those job
  * types and `excludeTypes` leaves them out (not both, neither empty); each filter keeps its own position.
+ * Jobs are read `settleMs` (default 30_000) after they dead-letter, so one that commits late is not missed.
  * Conditions carry `owner`, `runbook` and a context of `count` and `lastError`.
  */
 export function deadLetterSource(options = {}) {
@@ -27,7 +151,6 @@ export function deadLetterSource(options = {}) {
         }
     }
     const jobs = jobsTable(options.table);
-    const severity = options.severity ?? 'error';
     const name = 'job-dead-letter';
     const mode = options.types !== undefined ? 'only' : options.excludeTypes !== undefined ? 'except' : null;
     const listed = [...new Set(options.types ?? options.excludeTypes ?? [])].sort();
@@ -35,65 +158,47 @@ export function deadLetterSource(options = {}) {
     const stateKey = mode === null ? name
         : `${name}:${mode}:${joined.length > 200 ? createHash('sha256').update(joined).digest('hex') : joined}`;
     const listJson = JSON.stringify(listed);
-    const typeFilter = mode === 'only' ? sql `AND j.job_type IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
-        : mode === 'except' ? sql `AND j.job_type NOT IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
+    const typeFilter = mode === 'only' ? sql `AND t.job_type IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
+        : mode === 'except' ? sql `AND t.job_type NOT IN (SELECT jsonb_array_elements_text(${listJson}::jsonb))`
             : sql ``;
-    return {
-        name,
-        async evaluate() {
-            const db = pipeSystem(options.database);
-            const first = await db.execute(sql `
-        INSERT INTO pipework_alert_source_state (name, cursor_at) VALUES (${stateKey}, clock_timestamp())
-        ON CONFLICT (name) DO NOTHING RETURNING name`);
-            if (first.length > 0)
-                return [];
-            const rows = await db.execute(sql `
-        WITH cur AS (SELECT cursor_at FROM pipework_alert_source_state WHERE name = ${stateKey} FOR UPDATE),
-        found AS (
-          SELECT j.job_type, count(*)::int AS n, max(j.completed_at) AS hi,
-                 (array_agg(j.error_payload->>'message' ORDER BY j.completed_at DESC))[1] AS last_error
-          FROM ${sql.raw(jobs)} j, cur
-          WHERE j.status = 'dead_letter' AND j.completed_at > cur.cursor_at ${typeFilter}
-          GROUP BY j.job_type),
-        moved AS (
-          UPDATE pipework_alert_source_state s SET cursor_at = GREATEST(s.cursor_at, (SELECT max(hi) FROM found))
-          WHERE s.name = ${stateKey} AND EXISTS (SELECT 1 FROM found) RETURNING 1)
-        SELECT job_type, n, last_error, (SELECT count(*) FROM moved) AS moved FROM found ORDER BY job_type`);
-            return rows.map(r => ({
-                key: `job-dead-letter:${String(r['job_type'])}`,
-                severity,
-                message: `${String(r['n'])} ${String(r['job_type'])} job(s) dead-lettered`
-                    + (r['last_error'] !== null && r['last_error'] !== undefined ? `; last error: ${String(r['last_error'])}` : ''),
-                subject: { entity: 'job', entityId: String(r['job_type']) },
-                ...(options.owner !== undefined ? { owner: options.owner } : {}),
-                ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
-                context: { count: Number(r['n']), lastError: r['last_error'] === null || r['last_error'] === undefined ? null : String(r['last_error']) },
-            }));
-        },
+    const lastError = (g) => {
+        const payload = g.last['error_payload'];
+        const message = typeof payload === 'object' && payload !== null ? payload['message'] : undefined;
+        return message === null || message === undefined ? null : String(message);
     };
+    return newRowsSource({
+        name,
+        stateKey,
+        table: jobs,
+        at: 'completed_at',
+        where: sql `t.status = 'dead_letter' ${typeFilter}`,
+        groupBy: 'job_type',
+        ...(options.settleMs !== undefined ? { settleMs: options.settleMs } : {}),
+        ...(options.severity !== undefined ? { severity: options.severity } : {}),
+        ...(options.database !== undefined ? { database: options.database } : {}),
+        ...ownerFields(options),
+        message: g => `${String(g.count)} ${g.id} job(s) dead-lettered` + (lastError(g) !== null ? `; last error: ${lastError(g)}` : ''),
+        subject: g => ({ entity: 'job', entityId: g.id }),
+        context: g => ({ count: g.count, lastError: lastError(g) }),
+    });
 }
 /** Raises `job-long-running:<jobType>` while a job of that type has been claimed for longer than `olderThanMs`; resolves when none is. */
 export function longRunningSource(options) {
-    const jobs = jobsTable(options.table);
-    const severity = options.severity ?? 'warning';
-    return {
+    const jobs = quoteIdentifier(jobsTable(options.table), 'queue table');
+    return querySource({
         name: 'job-long-running',
-        owns: key => key.startsWith('job-long-running:'),
-        async evaluate() {
-            const rows = await pipeSystem(options.database).execute(sql `
-        SELECT job_type, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(claimed_at)) * 1000)::bigint AS oldest_ms
-        FROM ${sql.raw(jobs)}
-        WHERE status IN ('claimed', 'running')
-          AND claimed_at < clock_timestamp() - ${options.olderThanMs} * interval '1 millisecond'
-        GROUP BY job_type ORDER BY job_type`);
-            return rows.map(r => ({
-                key: `job-long-running:${String(r['job_type'])}`,
-                severity,
-                message: `${String(r['n'])} ${String(r['job_type'])} job(s) running for more than ${String(options.olderThanMs)} ms (oldest ${String(r['oldest_ms'])} ms)`,
-                subject: { entity: 'job', entityId: String(r['job_type']) },
-            }));
-        },
-    };
+        severity: options.severity ?? 'warning',
+        ...(options.database !== undefined ? { database: options.database } : {}),
+        query: sql `
+      SELECT job_type, count(*)::int AS n, (extract(epoch FROM clock_timestamp() - min(claimed_at)) * 1000)::bigint AS oldest_ms
+      FROM ${sql.raw(jobs)}
+      WHERE status IN ('claimed', 'running')
+        AND claimed_at < clock_timestamp() - ${options.olderThanMs} * interval '1 millisecond'
+      GROUP BY job_type ORDER BY job_type`,
+        id: r => String(r['job_type']),
+        message: r => `${String(r['n'])} ${String(r['job_type'])} job(s) running for more than ${String(options.olderThanMs)} ms (oldest ${String(r['oldest_ms'])} ms)`,
+        subject: r => ({ entity: 'job', entityId: String(r['job_type']) }),
+    });
 }
 /**
  * Raises `queue-backlog:<jobType>` while more than `maxPending` jobs of a type are ready to claim, or the
@@ -104,40 +209,31 @@ export function backlogSource(options) {
     if (options.maxPending === undefined && options.maxOldestAgeMs === undefined) {
         throw new Error('[pipework] alerts.source.backlog needs maxPending, maxOldestAgeMs or both.');
     }
-    const jobs = jobsTable(options.table);
-    const severity = options.severity ?? 'warning';
+    const jobs = quoteIdentifier(jobsTable(options.table), 'queue table');
     const maxPending = options.maxPending ?? null;
     const maxAge = options.maxOldestAgeMs ?? null;
-    return {
+    return querySource({
         name: 'queue-backlog',
-        owns: key => key.startsWith('queue-backlog:'),
-        async evaluate() {
-            const rows = await pipeSystem(options.database).execute(sql `
-        SELECT job_type, n, running, oldest_ms FROM (
-          SELECT job_type,
-                 (count(*) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp())))::int AS n,
-                 (count(*) FILTER (WHERE status IN ('claimed', 'running')))::int AS running,
-                 (extract(epoch FROM clock_timestamp() - min(queued_at) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp()))) * 1000)::bigint AS oldest_ms
-          FROM ${sql.raw(jobs)}
-          WHERE status IN ('pending', 'claimed', 'running')
-          GROUP BY job_type) p
-        WHERE (${maxPending}::int IS NOT NULL AND n > ${maxPending}::int)
-           OR (${maxAge}::bigint IS NOT NULL AND oldest_ms > ${maxAge}::bigint)
-        ORDER BY job_type`);
-            return rows.map(r => {
-                const oldest = Number(r['oldest_ms']);
-                return {
-                    key: `queue-backlog:${String(r['job_type'])}`,
-                    severity,
-                    message: `${String(r['n'])} ${String(r['job_type'])} job(s) waiting (${String(r['running'])} running), the oldest for ${String(oldest)} ms`,
-                    subject: { entity: 'job', entityId: String(r['job_type']) },
-                    ...(options.owner !== undefined ? { owner: options.owner } : {}),
-                    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
-                    context: { ready: Number(r['n']), running: Number(r['running']), oldestReadyWaitMs: oldest },
-                };
-            });
-        },
-    };
+        severity: options.severity ?? 'warning',
+        ...(options.database !== undefined ? { database: options.database } : {}),
+        ...ownerFields(options),
+        query: sql `
+      SELECT job_type, n, running, oldest_ms FROM (
+        SELECT job_type,
+               (count(*) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp())))::int AS n,
+               (count(*) FILTER (WHERE status IN ('claimed', 'running')))::int AS running,
+               (extract(epoch FROM clock_timestamp() - min(queued_at) FILTER (WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= clock_timestamp()))) * 1000)::bigint AS oldest_ms
+        FROM ${sql.raw(jobs)}
+        WHERE status IN ('pending', 'claimed', 'running')
+        GROUP BY job_type) p
+      WHERE (${maxPending}::int IS NOT NULL AND n > ${maxPending}::int)
+         OR (${maxAge}::bigint IS NOT NULL AND oldest_ms > ${maxAge}::bigint)
+      ORDER BY job_type`,
+        id: r => String(r['job_type']),
+        message: r => `${String(r['n'])} ${String(r['job_type'])} job(s) waiting (${String(r['running'])} running), the oldest for ${String(Number(r['oldest_ms']))} ms`,
+        subject: r => ({ entity: 'job', entityId: String(r['job_type']) }),
+        context: r => ({ ready: Number(r['n']), running: Number(r['running']), oldestReadyWaitMs: Number(r['oldest_ms']) }),
+    });
 }
 /** Raises `health:<name>` while a check reports unhealthy or throws; resolves when it is healthy again. Degraded is not a fault. */
 export function healthSource(checks, options = {}) {
@@ -177,47 +273,36 @@ export function unitFaultSource(options) {
     if (options.units !== undefined && options.units.length === 0) {
         throw new Error('[pipework] alerts.source.unitFault units must not be empty; omit it to watch every unit.');
     }
-    const severity = options.severity ?? 'error';
     const units = options.units === undefined ? null : JSON.stringify(options.units);
-    return {
+    const text = (v) => v === null || v === undefined ? null : String(v);
+    return querySource({
         name: 'unit-fault',
-        owns: key => key.startsWith('unit-fault:'),
-        async evaluate() {
-            const rows = await pipeSystem(options.database).execute(sql `
-        SELECT unit, count(*)::int AS n,
-               (array_agg(result ORDER BY at DESC))[1] AS last_result,
-               (array_agg(exit_code ORDER BY at DESC))[1] AS last_exit_code,
-               (array_agg(exit_status ORDER BY at DESC))[1] AS last_exit_status,
-               min(at) AS first_at, max(at) AS last_at,
-               ceil(extract(epoch FROM clock_timestamp() - min(at)) / 60)::int AS minutes
-        FROM pipework_unit_faults
-        WHERE at > clock_timestamp() - ${options.window} * interval '1 millisecond'
-          AND (${units}::jsonb IS NULL OR unit IN (SELECT jsonb_array_elements_text(${units}::jsonb)))
-        GROUP BY unit ORDER BY unit`);
-            const text = (v) => v === null || v === undefined ? null : String(v);
-            return rows.map(r => {
-                const unit = String(r['unit']);
-                const lastResult = String(r['last_result']);
-                const count = Number(r['n']);
-                return {
-                    key: `unit-fault:${unit}`,
-                    severity,
-                    message: `${unit}: ${lastResult} x${String(count)} in ${String(Math.max(1, Number(r['minutes'])))} min`,
-                    subject: { entity: 'unit', entityId: unit },
-                    ...(options.owner !== undefined ? { owner: options.owner } : {}),
-                    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
-                    context: {
-                        count,
-                        lastResult,
-                        lastExitCode: text(r['last_exit_code']),
-                        lastExitStatus: text(r['last_exit_status']),
-                        firstAt: new Date(r['first_at']).toISOString(),
-                        lastAt: new Date(r['last_at']).toISOString(),
-                    },
-                };
-            });
-        },
-    };
+        severity: options.severity ?? 'error',
+        ...(options.database !== undefined ? { database: options.database } : {}),
+        ...ownerFields(options),
+        query: sql `
+      SELECT unit, count(*)::int AS n,
+             (array_agg(result ORDER BY at DESC))[1] AS last_result,
+             (array_agg(exit_code ORDER BY at DESC))[1] AS last_exit_code,
+             (array_agg(exit_status ORDER BY at DESC))[1] AS last_exit_status,
+             min(at) AS first_at, max(at) AS last_at,
+             ceil(extract(epoch FROM clock_timestamp() - min(at)) / 60)::int AS minutes
+      FROM pipework_unit_faults
+      WHERE at > clock_timestamp() - ${options.window} * interval '1 millisecond'
+        AND (${units}::jsonb IS NULL OR unit IN (SELECT jsonb_array_elements_text(${units}::jsonb)))
+      GROUP BY unit ORDER BY unit`,
+        id: r => String(r['unit']),
+        message: r => `${String(r['unit'])}: ${String(r['last_result'])} x${String(Number(r['n']))} in ${String(Math.max(1, Number(r['minutes'])))} min`,
+        subject: r => ({ entity: 'unit', entityId: String(r['unit']) }),
+        context: r => ({
+            count: Number(r['n']),
+            lastResult: String(r['last_result']),
+            lastExitCode: text(r['last_exit_code']),
+            lastExitStatus: text(r['last_exit_status']),
+            firstAt: new Date(r['first_at']).toISOString(),
+            lastAt: new Date(r['last_at']).toISOString(),
+        }),
+    });
 }
 const pad2 = (n) => String(n).padStart(2, '0');
 function untilText(reset, now) {
@@ -240,47 +325,47 @@ export function quotaExhaustedSource(options = {}) {
     if (Array.isArray(keys) && keys.length === 0) {
         throw new Error('[pipework] alerts.source.quotaExhausted keys must not be empty; omit it to watch every key.');
     }
-    const severity = options.severity ?? 'error';
     const keyFilter = Array.isArray(keys)
         ? sql `AND key IN (SELECT jsonb_array_elements_text(${JSON.stringify([...new Set(keys)])}::jsonb))`
         : sql ``;
-    return {
+    const base = querySource({
         name: 'quota-exhausted',
-        owns: key => key.startsWith('quota-exhausted:'),
+        severity: options.severity ?? 'error',
+        ...(options.database !== undefined ? { database: options.database } : {}),
+        ...ownerFields(options),
+        query: sql `
+      SELECT key, remaining, "limit", reset_at,
+             (remaining > ${floor} AND refused_at IS NOT NULL AND refused_at >= updated_at) AS refused_only,
+             (refused_at IS NOT NULL AND refused_at >= updated_at) AS refused
+      FROM outbound_quota
+      WHERE reset_at > now() ${keyFilter}
+        AND (remaining <= ${floor} OR (refused_at IS NOT NULL AND refused_at >= updated_at))
+      ORDER BY key`,
+        id: r => String(r['key']),
+        message: r => {
+            const remaining = Number(r['remaining']);
+            const limit = r['limit'] === null ? null : Number(r['limit']);
+            const head = limit === null ? `${String(r['key'])} ${String(remaining)} remaining` : `${String(r['key'])} ${String(remaining)}/${String(limit)}`;
+            return `${head} until ${untilText(new Date(r['reset_at']), new Date())}${r['refused_only'] === true ? '; a call was refused' : ''}`;
+        },
+        subject: r => ({ entity: 'quota', entityId: String(r['key']) }),
+        context: r => ({
+            key: String(r['key']),
+            remaining: Number(r['remaining']),
+            limit: r['limit'] === null ? null : Number(r['limit']),
+            resetAt: new Date(r['reset_at']).toISOString(),
+            refused: r['refused'] === true,
+        }),
+    });
+    if (!(keys instanceof RegExp))
+        return base;
+    return {
+        ...base,
         async evaluate() {
-            const rows = await pipeSystem(options.database).execute(sql `
-        SELECT key, remaining, "limit", reset_at,
-               (remaining > ${floor} AND refused_at IS NOT NULL AND refused_at >= updated_at) AS refused_only,
-               (refused_at IS NOT NULL AND refused_at >= updated_at) AS refused
-        FROM outbound_quota
-        WHERE reset_at > now() ${keyFilter}
-          AND (remaining <= ${floor} OR (refused_at IS NOT NULL AND refused_at >= updated_at))
-        ORDER BY key`);
-            const now = new Date();
-            const out = [];
-            for (const r of rows) {
-                const key = String(r['key']);
-                if (keys instanceof RegExp) {
-                    keys.lastIndex = 0;
-                    if (!keys.test(key))
-                        continue;
-                }
-                const remaining = Number(r['remaining']);
-                const limit = r['limit'] === null ? null : Number(r['limit']);
-                const resetAt = new Date(r['reset_at']);
-                const refusedOnly = r['refused_only'] === true;
-                const head = limit === null ? `${key} ${String(remaining)} remaining` : `${key} ${String(remaining)}/${String(limit)}`;
-                out.push({
-                    key: `quota-exhausted:${key}`,
-                    severity,
-                    message: `${head} until ${untilText(resetAt, now)}${refusedOnly ? '; a call was refused' : ''}`,
-                    subject: { entity: 'quota', entityId: key },
-                    ...(options.owner !== undefined ? { owner: options.owner } : {}),
-                    ...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
-                    context: { key, remaining, limit, resetAt: resetAt.toISOString(), refused: r['refused'] === true },
-                });
-            }
-            return out;
+            return (await base.evaluate()).filter(c => {
+                keys.lastIndex = 0;
+                return keys.test(c.key.slice('quota-exhausted:'.length));
+            });
         },
     };
 }