pipework 0.285.0 → 0.287.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +1 -1
- package/REFERENCE.md +9 -1
- package/dist/async/jobs/claims.d.ts +5 -1
- package/dist/async/jobs/claims.d.ts.map +1 -1
- package/dist/async/jobs/claims.js.map +1 -1
- package/dist/async/jobs/queue.d.ts.map +1 -1
- package/dist/async/jobs/queue.js +3 -3
- package/dist/async/jobs/queue.js.map +1 -1
- package/dist/core/config/load.d.ts +1 -1
- package/dist/core/config/load.d.ts.map +1 -1
- package/dist/core/config/load.js +6 -1
- package/dist/core/config/load.js.map +1 -1
- package/dist/core/config/namespace.d.ts +1 -0
- package/dist/core/config/namespace.d.ts.map +1 -1
- package/dist/core/config/schema.d.ts +3 -1
- package/dist/core/config/schema.d.ts.map +1 -1
- package/dist/core/config/schema.js +3 -0
- package/dist/core/config/schema.js.map +1 -1
- package/dist/core/config/statement-observer.d.ts +12 -0
- package/dist/core/config/statement-observer.d.ts.map +1 -1
- package/dist/core/config/statement-observer.js +26 -1
- package/dist/core/config/statement-observer.js.map +1 -1
- package/dist/data/db/expressions.d.ts +10 -0
- package/dist/data/db/expressions.d.ts.map +1 -1
- package/dist/data/db/expressions.js +14 -0
- package/dist/data/db/expressions.js.map +1 -1
- package/dist/data/db/filter.d.ts +4 -2
- package/dist/data/db/filter.d.ts.map +1 -1
- package/dist/data/db/filter.js +4 -2
- package/dist/data/db/filter.js.map +1 -1
- package/dist/data/db/namespace.d.ts +3 -1
- package/dist/data/db/namespace.d.ts.map +1 -1
- package/dist/data/db/namespace.js +1 -1
- package/dist/data/db/namespace.js.map +1 -1
- package/dist/data/db/query-label.d.ts +15 -1
- package/dist/data/db/query-label.d.ts.map +1 -1
- package/dist/data/db/query-label.js +26 -1
- package/dist/data/db/query-label.js.map +1 -1
- package/dist/data/db/query-watch.js +2 -2
- package/dist/data/db/query-watch.js.map +1 -1
- package/dist/data/jsonb/namespace.d.ts +3 -1
- package/dist/data/jsonb/namespace.d.ts.map +1 -1
- package/dist/data/jsonb/namespace.js +3 -1
- package/dist/data/jsonb/namespace.js.map +1 -1
- package/dist/data/jsonb/query.d.ts +8 -0
- package/dist/data/jsonb/query.d.ts.map +1 -1
- package/dist/data/jsonb/query.js +19 -3
- package/dist/data/jsonb/query.js.map +1 -1
- package/dist/data/query/postgres-js/session.d.ts.map +1 -1
- package/dist/data/query/postgres-js/session.js +2 -1
- package/dist/data/query/postgres-js/session.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/infra/alerts/index.d.ts +1 -0
- package/dist/infra/alerts/index.d.ts.map +1 -1
- package/dist/infra/alerts/namespace.d.ts +3 -0
- package/dist/infra/alerts/namespace.d.ts.map +1 -1
- package/dist/infra/alerts/namespace.js +3 -0
- package/dist/infra/alerts/namespace.js.map +1 -1
- package/dist/infra/alerts/statement-sources.d.ts +27 -0
- package/dist/infra/alerts/statement-sources.d.ts.map +1 -0
- package/dist/infra/alerts/statement-sources.js +84 -0
- package/dist/infra/alerts/statement-sources.js.map +1 -0
- package/package.json +1 -1
- package/src/async/jobs/claims.ts +3 -1
- package/src/async/jobs/queue.ts +4 -3
- package/src/core/config/load.ts +7 -2
- package/src/core/config/schema.ts +4 -1
- package/src/core/config/statement-observer.ts +32 -0
- package/src/data/db/expressions.ts +15 -0
- package/src/data/db/filter.ts +4 -2
- package/src/data/db/namespace.ts +1 -1
- package/src/data/db/query-label.ts +28 -1
- package/src/data/db/query-watch.ts +2 -2
- package/src/data/jsonb/namespace.ts +3 -1
- package/src/data/jsonb/query.ts +22 -3
- package/src/data/query/postgres-js/session.ts +2 -1
- package/src/index.ts +1 -1
- package/src/infra/alerts/index.ts +1 -0
- package/src/infra/alerts/namespace.ts +3 -0
- package/src/infra/alerts/statement-sources.ts +94 -0
package/src/core/config/load.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { resolveEnvironment, resolveEnvironmentStrict, resolveDbUrl, resolveTest
|
|
|
4
4
|
import { ConfigError } from './errors.js'
|
|
5
5
|
import { resolveEnvVars } from './env-resolve.js'
|
|
6
6
|
import { consumeConfigBaseDir } from './base-dir.js'
|
|
7
|
-
import type
|
|
7
|
+
import { composeSettledObservers, type StatementObserver, type StatementSettledObserver } from './statement-observer.js'
|
|
8
8
|
|
|
9
9
|
export interface ResolvedObservability {
|
|
10
10
|
readonly slowQueryMs: number
|
|
@@ -247,6 +247,8 @@ function buildResolved(config: Blueprint, environment: Environment, baseDir: str
|
|
|
247
247
|
|
|
248
248
|
const env = Object.freeze(resolveEnvVars((config.env ?? {}) as import('./env-types.js').EnvDefs, environment))
|
|
249
249
|
|
|
250
|
+
config.observability?.statementTiming?.attach()
|
|
251
|
+
|
|
250
252
|
return {
|
|
251
253
|
raw: config,
|
|
252
254
|
environment,
|
|
@@ -261,7 +263,10 @@ function buildResolved(config: Blueprint, environment: Environment, baseDir: str
|
|
|
261
263
|
logAllQueries: config.observability?.logAllQueries ?? false,
|
|
262
264
|
poolMetricsInterval: config.observability?.poolMetricsInterval ?? 0,
|
|
263
265
|
onStatement: config.observability?.onStatement,
|
|
264
|
-
onStatementSettled:
|
|
266
|
+
onStatementSettled: composeSettledObservers([
|
|
267
|
+
...(config.observability?.onStatementSettled !== undefined ? [config.observability.onStatementSettled] : []),
|
|
268
|
+
...(config.observability?.statementTiming !== undefined ? [config.observability.statementTiming.observe] : []),
|
|
269
|
+
]),
|
|
265
270
|
},
|
|
266
271
|
trace: {
|
|
267
272
|
hotRetentionDays: config.trace?.hotRetentionDays ?? 7,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod/v4'
|
|
2
|
-
import type { StatementObserver, StatementSettledObserver } from './statement-observer.js'
|
|
2
|
+
import type { StatementObserver, StatementSettledObserver, StatementSettledFeed } from './statement-observer.js'
|
|
3
3
|
|
|
4
4
|
const SETTING_NAME = /^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?$/
|
|
5
5
|
|
|
@@ -188,6 +188,9 @@ const observabilitySchema = z.object({
|
|
|
188
188
|
onStatementSettled: z.custom<StatementSettledObserver>(v => typeof v === 'function', { message: 'Expected a function' })
|
|
189
189
|
.optional()
|
|
190
190
|
.describe('Per-statement completion hook — fires once per statement when it settles, with label, statementId, statementPrefix, durationMs, rows, connectionId and (on failure) a sanitized error. Covers every statement through pipework\'s query path (pipe(), pipe.* writes, connection.drizzle, transactions, savepoints, the test harness), not the driver\'s own BEGIN/COMMIT, the raw client door or the migration runner\'s connection'),
|
|
191
|
+
statementTiming: z.custom<StatementSettledFeed>(v => typeof v === 'object' && v !== null && typeof (v as { observe?: unknown }).observe === 'function' && typeof (v as { attach?: unknown }).attach === 'function', { message: 'Expected the value returned by alerts.source.statementTiming' })
|
|
192
|
+
.optional()
|
|
193
|
+
.describe('Feeds alerts.source.statementTiming from every settled statement; fires alongside onStatementSettled, which keeps working'),
|
|
191
194
|
})
|
|
192
195
|
|
|
193
196
|
const envVarDefSchema = z.discriminatedUnion('type', [
|
|
@@ -131,3 +131,35 @@ export interface StatementSettledEvent {
|
|
|
131
131
|
* A returned promise is not awaited.
|
|
132
132
|
*/
|
|
133
133
|
export type StatementSettledObserver = (event: StatementSettledEvent) => void | Promise<void>
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Something fed by every settled statement in addition to `observability.onStatementSettled`, passed as
|
|
137
|
+
* `observability.statementTiming` (`alerts.source.statementTiming` returns one). Both it and the app's own hook
|
|
138
|
+
* fire for each statement; neither replaces the other. `attach` is called once when the blueprint resolves, so the
|
|
139
|
+
* feed can tell that it was wired in.
|
|
140
|
+
*/
|
|
141
|
+
export interface StatementSettledFeed {
|
|
142
|
+
readonly observe: StatementSettledObserver
|
|
143
|
+
attach(): void
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The hooks that fire for each settled statement, in order, one throwing never keeping the next from running. */
|
|
147
|
+
export function composeSettledObservers(observers: readonly StatementSettledObserver[]): StatementSettledObserver | undefined {
|
|
148
|
+
if (observers.length === 0) return undefined
|
|
149
|
+
if (observers.length === 1) return observers[0]
|
|
150
|
+
return (event) => {
|
|
151
|
+
const pending: Promise<void>[] = []
|
|
152
|
+
const thrown: unknown[] = []
|
|
153
|
+
for (const observer of observers) {
|
|
154
|
+
try {
|
|
155
|
+
const out = observer(event)
|
|
156
|
+
if (out !== undefined) pending.push(out)
|
|
157
|
+
} catch (err) {
|
|
158
|
+
thrown.push(err)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (thrown.length > 0) throw thrown[0]
|
|
162
|
+
if (pending.length > 0) return Promise.all(pending).then(() => undefined)
|
|
163
|
+
return undefined
|
|
164
|
+
}
|
|
165
|
+
}
|
|
@@ -341,6 +341,21 @@ export function arrayAppend<C extends Column>(col: C, value: ElementOf<C>): SQL<
|
|
|
341
341
|
return sql<C['_']['data']>`array_append(${col}, ${element(col, value)}::${el})`
|
|
342
342
|
}
|
|
343
343
|
|
|
344
|
+
/**
|
|
345
|
+
* An array built from expressions: `ARRAY[e1, e2, ...]::<element>[]`, for the second argument of
|
|
346
|
+
* `arrayOverlaps` / `arrayContains` / `arrayContained`:
|
|
347
|
+
* `arrayOverlaps(t.keys, arrayOf(t.keys, concat(o.key, ':x'), concat(o.key, ':*')))`.
|
|
348
|
+
* `like` is an array column that supplies the element type (the one the result is compared with, or any
|
|
349
|
+
* column of the same type); it is read for its type only and does not appear in the SQL.
|
|
350
|
+
* A column or SQL expression element is used as is; a JS value is bound and converted by the element
|
|
351
|
+
* column's driver mapping. With no elements the result is the empty array `ARRAY[]::<element>[]`.
|
|
352
|
+
*/
|
|
353
|
+
export function arrayOf<C extends Column>(like: C, ...elements: Array<ElementOf<C> | Column | SQL>): SQL<C['_']['data']> {
|
|
354
|
+
const el = sql.raw(elementType(like, 'arrayOf'))
|
|
355
|
+
const parts = elements.map(e => (is(e, DrizzleColumn) || is(e, DrizzleSQL) ? e : element(like, e)))
|
|
356
|
+
return sql<C['_']['data']>`ARRAY[${sql.join(parts.map(p => sql`${p}`), sql`, `)}]::${el}[]`
|
|
357
|
+
}
|
|
358
|
+
|
|
344
359
|
/** PostgreSQL `array_remove(col, value)` — drops every element equal to `value`. `value` is bound and cast to the element type. */
|
|
345
360
|
export function arrayRemove<C extends Column>(col: C, value: ElementOf<C>): SQL<C['_']['data']> {
|
|
346
361
|
const el = sql.raw(elementType(col, 'arrayRemove'))
|
package/src/data/db/filter.ts
CHANGED
|
@@ -5,9 +5,9 @@ import {
|
|
|
5
5
|
} from '../query/index.js'
|
|
6
6
|
import { countAgg, countDistinctAgg, sumAgg, sumDistinctAgg, avgAgg, avgDistinctAgg, maxAgg, minAgg, boolAnd, boolOr, arrayAgg, stringAgg, jsonbAgg, jsonbObjectAgg, jsonbBuildObject } from './aggregates.js'
|
|
7
7
|
import { except, exceptAll, intersect, intersectAll, union, unionAll } from '../query/pg-core/index.js'
|
|
8
|
-
import { increment, decrement, add, subtract, coalesce, now, scalar, cast, collate, interval, clockTimestamp, strpos, textContains, startsWith, left, right, length, concat, arrayReplace, arrayHas, splitPart, lower, upper, trim, replace, substring, reverse, lpad, rpad, regexpReplace, regexpMatch, stringToArray, cardinality, arrayAt, arrayPosition, arraySlice, arrayToString, arrayAppend, arrayRemove, dateTrunc, rowNumber, rank, denseRank, over, identifier, withRecursive, caseWhen, walCurrentLsn, walReplayLsn, replicaFresh } from './expressions.js'
|
|
8
|
+
import { increment, decrement, add, subtract, coalesce, now, scalar, cast, collate, interval, clockTimestamp, strpos, textContains, startsWith, left, right, length, concat, arrayReplace, arrayHas, splitPart, lower, upper, trim, replace, substring, reverse, lpad, rpad, regexpReplace, regexpMatch, stringToArray, cardinality, arrayAt, arrayPosition, arraySlice, arrayToString, arrayAppend, arrayOf, arrayRemove, dateTrunc, rowNumber, rank, denseRank, over, identifier, withRecursive, caseWhen, walCurrentLsn, walReplayLsn, replicaFresh } from './expressions.js'
|
|
9
9
|
import { ago, castUuid, castJsonb, castTimestamp, castBoolean, castNumeric, castInteger, castText, nextSequence } from './composed.js'
|
|
10
|
-
import { json as jsonbExtract, pathText as jsonbExtractText, contains as jsonbContains, merge as jsonbMerge, remove as jsonbRemove } from '../jsonb/query.js'
|
|
10
|
+
import { json as jsonbExtract, pathText as jsonbExtractText, contains as jsonbContains, merge as jsonbMerge, concat as jsonbConcat, remove as jsonbRemove } from '../jsonb/query.js'
|
|
11
11
|
import { arrayAny, arrayAll, jsonbElements } from './array-elements.js'
|
|
12
12
|
import { inList, notInList } from './in-list.js'
|
|
13
13
|
|
|
@@ -56,6 +56,7 @@ export const filter = {
|
|
|
56
56
|
arraySlice,
|
|
57
57
|
arrayToString,
|
|
58
58
|
arrayAppend,
|
|
59
|
+
arrayOf,
|
|
59
60
|
arrayRemove,
|
|
60
61
|
dateTrunc,
|
|
61
62
|
rowNumber,
|
|
@@ -81,6 +82,7 @@ export const filter = {
|
|
|
81
82
|
jsonbExtractText,
|
|
82
83
|
jsonbContains,
|
|
83
84
|
jsonbMerge,
|
|
85
|
+
jsonbConcat,
|
|
84
86
|
jsonbRemove,
|
|
85
87
|
arrayAny,
|
|
86
88
|
arrayAll,
|
package/src/data/db/namespace.ts
CHANGED
|
@@ -38,7 +38,7 @@ export const pipe = Object.assign(getDb, {
|
|
|
38
38
|
/** Query operators and aggregates — eq, gt, lt, and, or, count, sum, etc. */
|
|
39
39
|
filter,
|
|
40
40
|
|
|
41
|
-
/** Names every statement `fn` starts, in `query:slow`, `query:large` and `query:in-flight` log lines (as `label`, beside the `sql` prefix): `pipe.labelled('servable-footprints', async () => await ...)`. A query builder is lazy: `fn` must await it, not return it, or it runs outside the scope. */
|
|
41
|
+
/** Names every statement `fn` starts, in `query:slow`, `query:large` and `query:in-flight` log lines (as `label`, beside the `sql` prefix) and, as a leading block comment on the SQL sent, in `pg_stat_activity.query`, server logs and `pg_stat_statements` text (the comment does not change the statement's queryid). In the comment the label is made safe: comment delimiters are split with a space, control characters become spaces, and it is cut to 80 characters; the log lines carry it as given. `pipe.labelled('servable-footprints', async () => await ...)`. A query builder is lazy: `fn` must await it, not return it, or it runs outside the scope. */
|
|
42
42
|
labelled,
|
|
43
43
|
|
|
44
44
|
/** Raw statement tag — for a statement pipework does not model. Interpolated values are parameters; `sql.raw` is not. Runs on the caller's connection, so inside a request it is the request transaction. Writes still go through pipe.update / insert / delete, which is what audit hooks. */
|
|
@@ -2,7 +2,34 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
|
|
2
2
|
|
|
3
3
|
const scope = new AsyncLocalStorage<string>()
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
const COMMENT_LABEL_CHARS = 80
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Makes a label safe to sit inside a `/* ... *\/` comment. Postgres block comments nest, so both `*\/`
|
|
9
|
+
* and `/*` are broken apart by a space (repeated, so `*\/*` cannot recombine), control characters
|
|
10
|
+
* (newlines, NUL) become spaces, and the text is cut to 80 characters so it leaves room in
|
|
11
|
+
* `pg_stat_activity.query` (1024 bytes by default). Returns undefined when nothing printable is left.
|
|
12
|
+
*/
|
|
13
|
+
export function commentSafeLabel(label: string): string | undefined {
|
|
14
|
+
let safe = label.replace(/[\u0000-\u001f\u007f\u2028\u2029]+/g, ' ').trim().slice(0, COMMENT_LABEL_CHARS)
|
|
15
|
+
while (/\*\/|\/\*/.test(safe)) safe = safe.replace(/\*\/|\/\*/g, m => `${m[0]!} ${m[1]!}`)
|
|
16
|
+
safe = safe.trim()
|
|
17
|
+
return safe === '' ? undefined : safe
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* `sqlText` with the enclosing `labelled` label as a leading `/* label *\/` comment, so the label shows in
|
|
22
|
+
* `pg_stat_activity.query`, server logs and `pg_stat_statements` text. Unchanged outside a scope.
|
|
23
|
+
* A comment is chosen over `application_name` because it costs no round trip, needs no transaction
|
|
24
|
+
* and does not change `pg_stat_statements`' queryid (which ignores comments), so labels never split a statement's stats.
|
|
25
|
+
*/
|
|
26
|
+
export function withLabelComment(sqlText: string): string {
|
|
27
|
+
const label = scope.getStore()
|
|
28
|
+
const safe = label === undefined ? undefined : commentSafeLabel(label)
|
|
29
|
+
return safe === undefined ? sqlText : `/* ${safe} */ ${sqlText}`
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Runs `fn` with `label` naming every statement it starts, in slow-query and in-flight-query log lines and as a leading SQL comment (see `withLabelComment`). */
|
|
6
33
|
export function labelled<T>(label: string, fn: () => T): T {
|
|
7
34
|
return scope.run(label, fn)
|
|
8
35
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import postgres from 'postgres'
|
|
2
2
|
import { createHash } from 'node:crypto'
|
|
3
3
|
import { getLogger } from '../../core/logging/logger.js'
|
|
4
|
-
import { currentQueryLabel } from './query-label.js'
|
|
4
|
+
import { currentQueryLabel, withLabelComment } from './query-label.js'
|
|
5
5
|
import { DrizzleQueryError } from '../query/errors.js'
|
|
6
6
|
import { parameterBytes, setOperationArms } from './query-size.js'
|
|
7
7
|
import type { StatementWatch } from '../query/postgres-js/session.js'
|
|
@@ -129,7 +129,7 @@ export function createStatementWatch(options: StatementWatchOptions): StatementW
|
|
|
129
129
|
WHERE datname = current_database() AND pid <> pg_backend_pid() AND state <> 'idle'
|
|
130
130
|
AND (query = $1 OR (length(query) >= ${String(PG_TRUNCATED_QUERY_LENGTH)} AND left($1, length(query)) = query))
|
|
131
131
|
ORDER BY abs(extract(epoch FROM query_start) - $2::float8) LIMIT 1`,
|
|
132
|
-
[sqlText, startedMs / 1000],
|
|
132
|
+
[withLabelComment(sqlText), startedMs / 1000],
|
|
133
133
|
).then(rows => {
|
|
134
134
|
const row = rows[0]
|
|
135
135
|
return row === undefined
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { contains, merge, remove, containedBy, hasKey, text, json, path, pathText } from './query.js'
|
|
1
|
+
import { contains, merge, concat, remove, containedBy, hasKey, text, json, path, pathText } from './query.js'
|
|
2
2
|
import { boolean, number, integer } from './composed.js'
|
|
3
3
|
|
|
4
4
|
/** JSONB query operators — typed helpers for PostgreSQL JSONB containment, key checks, and field extraction. */
|
|
@@ -7,6 +7,8 @@ export const jsonb = {
|
|
|
7
7
|
contains,
|
|
8
8
|
/** Merges an object into a JSONB column in `.set()`: `coalesce(col, '{}') || value`. Top-level keys overwrite; others are kept. */
|
|
9
9
|
merge,
|
|
10
|
+
/** Appends to a JSONB column in `.set()`: `col || $1::jsonb`, value bound as jsonb. An array appends its elements; a null column stays null. */
|
|
11
|
+
concat,
|
|
10
12
|
/** Removes top-level keys from a JSONB column in `.set()`: `col - 'key'`. Composes as the column of `jsonb.merge`. */
|
|
11
13
|
remove,
|
|
12
14
|
/** The <@ contained-by operator — tests whether a JSONB column is contained by the given value. */
|
package/src/data/jsonb/query.ts
CHANGED
|
@@ -43,6 +43,25 @@ export function pathText(column: Column | SQL, ...keys: string[]): SQL<string> {
|
|
|
43
43
|
return text(intermediate, finalKey)
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Appends to a JSONB column for `.set()`, typically in `onConflictDoUpdate`: `col || $1::jsonb`, with `value`
|
|
48
|
+
* serialized and bound as jsonb. Appending an array to an array column adds its elements (`[1] || [2]` is `[1, 2]`);
|
|
49
|
+
* an object merges its top-level keys. The append happens in the statement, so concurrent upserts each
|
|
50
|
+
* add their element and none is lost. A null column stays null (SQL `||`): seed the column on insert, or
|
|
51
|
+
* wrap it in `coalesce(col, '[]'::jsonb)`.
|
|
52
|
+
*/
|
|
53
|
+
export function concat(column: Column | SQL, value: unknown): SQL {
|
|
54
|
+
const json: unknown = JSON.stringify(value)
|
|
55
|
+
if (typeof json !== 'string') {
|
|
56
|
+
throw new Error(
|
|
57
|
+
'[pipework] jsonb.concat() needs a JSON-serializable value, got ' + typeof value + '.\n\n' +
|
|
58
|
+
' undefined, functions and symbols have no JSON form.\n' +
|
|
59
|
+
" Fix: pass a value JSON can hold: jsonb.concat(col, [{ id: 1 }]).\n",
|
|
60
|
+
)
|
|
61
|
+
}
|
|
62
|
+
return sql`(${column} || ${json}::jsonb)`
|
|
63
|
+
}
|
|
64
|
+
|
|
46
65
|
/**
|
|
47
66
|
* Merges `value` into a JSONB column for `.set()`: `coalesce(col, '{}'::jsonb) || value::jsonb`. Top-level keys
|
|
48
67
|
* in `value` overwrite; others are kept; a null column becomes `value`. The merge happens in the statement,
|
|
@@ -57,7 +76,7 @@ export function merge(column: Column | SQL, value: Record<string, unknown>): SQL
|
|
|
57
76
|
" Fix: pass an object: jsonb.merge(col, { key: 'value' }).\n",
|
|
58
77
|
)
|
|
59
78
|
}
|
|
60
|
-
return sql`coalesce(${column}, '{}'::jsonb) || ${JSON.stringify(value)}::jsonb`
|
|
79
|
+
return sql`(coalesce(${column}, '{}'::jsonb) || ${JSON.stringify(value)}::jsonb)`
|
|
61
80
|
}
|
|
62
81
|
|
|
63
82
|
/**
|
|
@@ -81,6 +100,6 @@ export function remove(column: Column | SQL, ...keys: string[]): SQL {
|
|
|
81
100
|
)
|
|
82
101
|
}
|
|
83
102
|
}
|
|
84
|
-
if (keys.length === 1) return sql
|
|
85
|
-
return sql
|
|
103
|
+
if (keys.length === 1) return sql`(${column} - ${keys[0]!}::text)`
|
|
104
|
+
return sql`(${column} - array[${sql.join(keys.map(k => sql`${k}`), sql`, `)}]::text[])`
|
|
86
105
|
}
|
|
@@ -13,6 +13,7 @@ import type { RelationalSchemaConfig, TablesRelationalConfig } from '../relation
|
|
|
13
13
|
import { fillPlaceholders, type Query } from '../sql/sql.js';
|
|
14
14
|
import { tracer } from '../tracing.js';
|
|
15
15
|
import { type Assume, mapResultRow } from '../utils.js';
|
|
16
|
+
import { withLabelComment } from '../../db/query-label.js';
|
|
16
17
|
import { propagateObserved, throughErrorBoundary } from './driver-error-boundary.js';
|
|
17
18
|
|
|
18
19
|
/** Wraps one statement's execution; installed by the pool to time it and report it while it runs. */
|
|
@@ -26,7 +27,7 @@ function noteSeen(): void {}
|
|
|
26
27
|
|
|
27
28
|
/** Starts the statement and tells the watch which pending query it is, so it can learn the connection it runs on. */
|
|
28
29
|
function startUnsafe(client: Sql, query: string, params: unknown[], seen: (pending: object) => void) {
|
|
29
|
-
const pending = client.unsafe(query, params as any[]);
|
|
30
|
+
const pending = client.unsafe(withLabelComment(query), params as any[]);
|
|
30
31
|
seen(pending);
|
|
31
32
|
return pending;
|
|
32
33
|
}
|
package/src/index.ts
CHANGED
|
@@ -21,7 +21,7 @@ export type { TimeoutOptions } from './data/db/timeouts.js'
|
|
|
21
21
|
// sanctioned door to a raw connection; keeps the WRAPPED_PACKAGES postgres ban airtight.
|
|
22
22
|
export { tap } from './data/db/pool.js'
|
|
23
23
|
export type { ManagedConnection, ConnectOptions } from './data/db/pool.js'
|
|
24
|
-
export type { StatementObserver, StatementEvent, StatementSettledObserver, StatementSettledEvent, StatementSettledError } from './core/config/statement-observer.js'
|
|
24
|
+
export type { StatementObserver, StatementEvent, StatementSettledObserver, StatementSettledEvent, StatementSettledError, StatementSettledFeed } from './core/config/statement-observer.js'
|
|
25
25
|
|
|
26
26
|
export type { DefinedTable, ExternalTable, DomainDefinition, DefinitionMethods, FieldDescriptor, FieldModifiers, FieldMeta, DefineOptions,
|
|
27
27
|
FactoryBuilder, FactoryOptions, TraitDefinition, FieldColumn, ProjectedColumns,
|
|
@@ -10,3 +10,4 @@ export type { AlertsHttpOptions } from './http.js'
|
|
|
10
10
|
export { clientFaults } from './client-faults.js'
|
|
11
11
|
export type { ClientFaultsOptions, ClientFaultReport } from './client-faults.js'
|
|
12
12
|
export type { RequestFaultsOptions } from './request-faults.js'
|
|
13
|
+
export type { StatementTimingOptions, StatementTimingSource } from './statement-sources.js'
|
|
@@ -8,6 +8,7 @@ import { recordFault, recordFaultFor, useFaultTarget, faultsSource } from './fau
|
|
|
8
8
|
import { recordBeatFor } from './beat.js'
|
|
9
9
|
import { deliveryFailedSource, webhookFailedSource, creditsSource } from './subsystem-sources.js'
|
|
10
10
|
import { emailSource, storageSource, accessLinksSource, lockoutsSource } from './infra-sources.js'
|
|
11
|
+
import { statementTimingSource } from './statement-sources.js'
|
|
11
12
|
import { schedulesSource, stepsSource, pipelinesSource, tracesSource } from './async-sources.js'
|
|
12
13
|
|
|
13
14
|
/** Alerts namespace — raise and resolve keyed conditions, grouped, delivered to sinks through an outbox, traced. */
|
|
@@ -75,6 +76,8 @@ export const alerts = {
|
|
|
75
76
|
traces: tracesSource,
|
|
76
77
|
/** Raises `fault:<kind>` per kind of swallowed failure recorded by `recordFault` since the last tick (the pipework subsystems record audit-write, quota-record, credits-callback, trace-steps-lost, queue-* and scheduler faults). `kinds` limits it. */
|
|
77
78
|
faults: faultsSource,
|
|
79
|
+
/** Raises `statement-slow:<label>` while the p90 of a `pipe.labelled` label's last `window` settled statements is over that label's budget; give the returned source to `observability.statementTiming` too. */
|
|
80
|
+
statementTiming: statementTimingSource,
|
|
78
81
|
/** Raises `loop-stalled:<name>` while a background loop (scheduler, reaper, claim loop, support drain) has not run or succeeded within its allowance; the tick itself is covered by `pipework alerts stale`. */
|
|
79
82
|
loopStalled: loopStalledSource,
|
|
80
83
|
},
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// An alert source over settled statements: per `pipe.labelled` label, the p90 of its last N durations against a budget.
|
|
2
|
+
|
|
3
|
+
import type { StatementSettledEvent, StatementSettledFeed } from '../../core/config/statement-observer.js'
|
|
4
|
+
import type { AlertSeverity } from './handle.js'
|
|
5
|
+
import type { AlertCondition, AlertSource } from './sources.js'
|
|
6
|
+
|
|
7
|
+
export interface StatementTimingOptions {
|
|
8
|
+
/** Budget in ms per label: `{ 'gate-read': 1000 }`. A label not named here is never recorded. At least one. */
|
|
9
|
+
readonly budgets: Readonly<Record<string, number>>
|
|
10
|
+
/** N: statements per label the p90 is taken over. The source stays quiet for a label until N have settled. */
|
|
11
|
+
readonly window: number
|
|
12
|
+
/** Default `warning`, as the backlog source. */
|
|
13
|
+
readonly severity?: AlertSeverity
|
|
14
|
+
readonly owner?: string
|
|
15
|
+
readonly runbook?: string
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** What `alerts.source.statementTiming` returns: an `AlertSource` that is also passed as `observability.statementTiming`. */
|
|
19
|
+
export type StatementTimingSource = AlertSource & StatementSettledFeed
|
|
20
|
+
|
|
21
|
+
interface Ring { readonly samples: number[]; next: number }
|
|
22
|
+
|
|
23
|
+
/** Nearest rank: the smallest sample that at least 90% of the samples are at or under. */
|
|
24
|
+
function p90(samples: readonly number[]): number {
|
|
25
|
+
const sorted = [...samples].sort((a, b) => a - b)
|
|
26
|
+
return sorted[Math.ceil(sorted.length * 0.9) - 1]!
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Raises `statement-slow:<label>` while the p90 duration of a label's last `window` settled statements is over
|
|
31
|
+
* that label's budget; resolves when it is back under. Statements are fed in-process by
|
|
32
|
+
* `observability.statementTiming` (pass the returned source there as well as to `alerts.create({ sources })`;
|
|
33
|
+
* the app's own `onStatementSettled` still fires). Only the labels in `budgets` are recorded, each in a ring of
|
|
34
|
+
* `window` numbers, so memory is bounded by `window` times the labels named; unlabelled statements and other
|
|
35
|
+
* labels are ignored. Failed statements count, with the time they took. The window is this process's: a tick sees
|
|
36
|
+
* only statements that settled in its own process, and a label that stops being called keeps its last window.
|
|
37
|
+
* Context carries `label`, `p90Ms`, `window` and `budgetMs`.
|
|
38
|
+
*/
|
|
39
|
+
export function statementTimingSource(options: StatementTimingOptions): StatementTimingSource {
|
|
40
|
+
const who = 'alerts.source.statementTiming'
|
|
41
|
+
if (!Number.isInteger(options.window) || options.window < 1) {
|
|
42
|
+
throw new Error(`[pipework] ${who} window must be a positive integer number of statements, got ${String(options.window)}.`)
|
|
43
|
+
}
|
|
44
|
+
const entries = Object.entries(options.budgets)
|
|
45
|
+
if (entries.length === 0) throw new Error(`[pipework] ${who} needs budgets: at least one label and its budget in ms.`)
|
|
46
|
+
const budgets = new Map<string, number>()
|
|
47
|
+
for (const [label, budget] of entries) {
|
|
48
|
+
if (label === '') throw new Error(`[pipework] ${who} budgets: a label must not be empty.`)
|
|
49
|
+
if (typeof budget !== 'number' || !Number.isFinite(budget) || budget <= 0) {
|
|
50
|
+
throw new Error(`[pipework] ${who} budget for "${label}" must be a positive number of milliseconds, got ${String(budget)}.`)
|
|
51
|
+
}
|
|
52
|
+
budgets.set(label, budget)
|
|
53
|
+
}
|
|
54
|
+
const size = options.window
|
|
55
|
+
const severity = options.severity ?? 'warning'
|
|
56
|
+
const rings = new Map<string, Ring>()
|
|
57
|
+
let attached = false
|
|
58
|
+
const prefix = 'statement-slow:'
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
name: 'statement-slow',
|
|
62
|
+
owns: key => key.startsWith(prefix),
|
|
63
|
+
observe(event: StatementSettledEvent): void {
|
|
64
|
+
if (event.label === null || !budgets.has(event.label)) return
|
|
65
|
+
let ring = rings.get(event.label)
|
|
66
|
+
if (ring === undefined) { ring = { samples: [], next: 0 }; rings.set(event.label, ring) }
|
|
67
|
+
if (ring.samples.length < size) ring.samples.push(event.durationMs)
|
|
68
|
+
else { ring.samples[ring.next] = event.durationMs; ring.next = (ring.next + 1) % size }
|
|
69
|
+
},
|
|
70
|
+
attach(): void { attached = true },
|
|
71
|
+
evaluate(): Promise<AlertCondition[]> {
|
|
72
|
+
if (!attached) {
|
|
73
|
+
return Promise.reject(new Error(`[pipework] ${who} was never given to the blueprint: set observability.statementTiming to this source, or it sees no statements.`))
|
|
74
|
+
}
|
|
75
|
+
const out: AlertCondition[] = []
|
|
76
|
+
for (const [label, budget] of budgets) {
|
|
77
|
+
const ring = rings.get(label)
|
|
78
|
+
if (ring === undefined || ring.samples.length < size) continue
|
|
79
|
+
const value = p90(ring.samples)
|
|
80
|
+
if (value <= budget) continue
|
|
81
|
+
out.push({
|
|
82
|
+
key: `${prefix}${label}`,
|
|
83
|
+
severity,
|
|
84
|
+
message: `p90 of the last ${String(size)} "${label}" statements is ${String(value)} ms, over its ${String(budget)} ms budget`,
|
|
85
|
+
subject: { entity: 'statement-label', entityId: label },
|
|
86
|
+
...(options.owner !== undefined ? { owner: options.owner } : {}),
|
|
87
|
+
...(options.runbook !== undefined ? { runbook: options.runbook } : {}),
|
|
88
|
+
context: { label, p90Ms: value, window: size, budgetMs: budget },
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
return Promise.resolve(out)
|
|
92
|
+
},
|
|
93
|
+
}
|
|
94
|
+
}
|