pipework 0.290.0 → 0.292.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/REFERENCE.md +81 -1
- package/dist/auth/tenant/scoped-db.d.ts +9 -0
- package/dist/auth/tenant/scoped-db.d.ts.map +1 -1
- package/dist/auth/tenant/scoped-db.js +14 -0
- package/dist/auth/tenant/scoped-db.js.map +1 -1
- package/dist/data/db/function-definition.d.ts +46 -0
- package/dist/data/db/function-definition.d.ts.map +1 -0
- package/dist/data/db/function-definition.js +84 -0
- package/dist/data/db/function-definition.js.map +1 -0
- package/dist/data/db/function-source.d.ts +33 -2
- package/dist/data/db/function-source.d.ts.map +1 -1
- package/dist/data/db/function-source.js +47 -19
- package/dist/data/db/function-source.js.map +1 -1
- package/dist/data/db/namespace.d.ts +5 -3
- package/dist/data/db/namespace.d.ts.map +1 -1
- package/dist/data/db/namespace.js +5 -3
- package/dist/data/db/namespace.js.map +1 -1
- package/dist/data/db/rows.d.ts +1 -0
- package/dist/data/db/rows.d.ts.map +1 -1
- package/dist/data/db/rows.js +1 -1
- package/dist/data/db/rows.js.map +1 -1
- package/dist/data/migrate/diff.d.ts +7 -0
- package/dist/data/migrate/diff.d.ts.map +1 -1
- package/dist/data/migrate/diff.js +37 -0
- package/dist/data/migrate/diff.js.map +1 -1
- package/dist/data/migrate/history.d.ts +2 -0
- package/dist/data/migrate/history.d.ts.map +1 -1
- package/dist/data/migrate/history.js +8 -0
- package/dist/data/migrate/history.js.map +1 -1
- package/dist/data/migrate/snapshot.d.ts +3 -0
- package/dist/data/migrate/snapshot.d.ts.map +1 -1
- package/dist/data/migrate/snapshot.js +13 -0
- package/dist/data/migrate/snapshot.js.map +1 -1
- package/dist/data/migrate/sql-emit.d.ts.map +1 -1
- package/dist/data/migrate/sql-emit.js +30 -2
- package/dist/data/migrate/sql-emit.js.map +1 -1
- package/dist/data/write/audit/verb-insert.d.ts.map +1 -1
- package/dist/data/write/audit/verb-insert.js +4 -1
- package/dist/data/write/audit/verb-insert.js.map +1 -1
- package/dist/data/write/audit/verb-update.d.ts.map +1 -1
- package/dist/data/write/audit/verb-update.js +14 -3
- package/dist/data/write/audit/verb-update.js.map +1 -1
- package/dist/data/write/batch-insert.d.ts +35 -0
- package/dist/data/write/batch-insert.d.ts.map +1 -0
- package/dist/data/write/batch-insert.js +171 -0
- package/dist/data/write/batch-insert.js.map +1 -0
- package/dist/data/write/execute.d.ts +1 -1
- package/dist/data/write/execute.d.ts.map +1 -1
- package/dist/data/write/execute.js +89 -2
- package/dist/data/write/execute.js.map +1 -1
- package/dist/data/write/index.d.ts +2 -0
- package/dist/data/write/index.d.ts.map +1 -1
- package/dist/data/write/index.js +1 -0
- package/dist/data/write/index.js.map +1 -1
- package/dist/data/write/insert.d.ts +8 -0
- package/dist/data/write/insert.d.ts.map +1 -1
- package/dist/data/write/insert.js +14 -0
- package/dist/data/write/insert.js.map +1 -1
- package/dist/data/write/state.d.ts +10 -0
- package/dist/data/write/state.d.ts.map +1 -1
- package/dist/data/write/update-each.d.ts +28 -0
- package/dist/data/write/update-each.d.ts.map +1 -0
- package/dist/data/write/update-each.js +104 -0
- package/dist/data/write/update-each.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/test/function-reads.d.ts +97 -0
- package/dist/test/function-reads.d.ts.map +1 -0
- package/dist/test/function-reads.js +202 -0
- package/dist/test/function-reads.js.map +1 -0
- package/dist/test/index.d.ts +1 -0
- package/dist/test/index.d.ts.map +1 -1
- package/dist/test/index.js +1 -0
- package/dist/test/index.js.map +1 -1
- package/package.json +2 -2
- package/src/auth/tenant/scoped-db.ts +13 -0
- package/src/data/db/function-definition.ts +172 -0
- package/src/data/db/function-source.ts +86 -14
- package/src/data/db/namespace.ts +5 -3
- package/src/data/db/rows.ts +1 -1
- package/src/data/migrate/diff.ts +45 -0
- package/src/data/migrate/history.ts +9 -0
- package/src/data/migrate/snapshot.ts +17 -0
- package/src/data/migrate/sql-emit.ts +33 -3
- package/src/data/write/audit/verb-insert.ts +4 -1
- package/src/data/write/audit/verb-update.ts +13 -3
- package/src/data/write/batch-insert.ts +179 -0
- package/src/data/write/execute.ts +77 -2
- package/src/data/write/index.ts +2 -0
- package/src/data/write/insert.ts +15 -0
- package/src/data/write/state.ts +6 -0
- package/src/data/write/update-each.ts +120 -0
- package/src/index.ts +2 -1
- package/src/test/function-reads.ts +290 -0
- package/src/test/index.ts +1 -0
package/src/data/write/state.ts
CHANGED
|
@@ -35,6 +35,12 @@ export interface WriteState {
|
|
|
35
35
|
readonly onConflictColumns?: readonly string[]
|
|
36
36
|
/** `ON CONFLICT DO NOTHING` for `insert`; `target` is field keys, absent for the bare clause. */
|
|
37
37
|
readonly onConflict?: { readonly action: 'nothing'; readonly target?: readonly string[] }
|
|
38
|
+
/** `insert` only: send every row as one statement (`INSERT ... SELECT FROM unnest(...)`), whatever the row count. */
|
|
39
|
+
readonly batch?: boolean
|
|
40
|
+
/** `insert` batch only: the columns already encoded for this statement (set by the batch planner). */
|
|
41
|
+
readonly encoded?: unknown
|
|
42
|
+
/** `update` only: one update for many keys, `from` a keys rowset; the result is the row count per input key. */
|
|
43
|
+
readonly each?: { readonly source: unknown; readonly ordinal: SQL; readonly count: number }
|
|
38
44
|
/** What `RETURNING` should produce. `null` (default) emits no RETURNING. */
|
|
39
45
|
readonly returning?: ReturningShape
|
|
40
46
|
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// `pipe.updateEach(table, { keys, data, where, set })`: one UPDATE for many keys, a count per key.
|
|
2
|
+
//
|
|
3
|
+
// UPDATE "t" SET ... FROM (select * from unnest($1::uuid[], $2::int[], $3::int[]) as "rows"("id", "ordinal", "_i")) "rows"
|
|
4
|
+
// WHERE <where> RETURNING "rows"."_i"
|
|
5
|
+
//
|
|
6
|
+
// The keys travel as `pipe.rows` columns (one parameter per column), each with its input index `_i`. The
|
|
7
|
+
// rows the update returns are counted per `_i`, which is what tells a key that matched nothing (it never
|
|
8
|
+
// appears in the result) from one that matched once. A key that did not match exactly once rolls the update
|
|
9
|
+
// back (a savepoint inside the caller's transaction) and throws naming the key.
|
|
10
|
+
|
|
11
|
+
import type { SQL } from '../db/types.js'
|
|
12
|
+
import { pipe } from '../db/db.js'
|
|
13
|
+
import { rows } from '../db/rows.js'
|
|
14
|
+
import type { RowsColumns, RowsData, RowsSource } from '../db/rows.js'
|
|
15
|
+
import type { PgTable } from '../query/pg-core/index.js'
|
|
16
|
+
import { assertNotExternal } from '../domain/external-registry.js'
|
|
17
|
+
import type { DefinedTable } from '../domain/types.js'
|
|
18
|
+
import type { UpdateSetValues } from './values.js'
|
|
19
|
+
import { executeWrite } from './execute.js'
|
|
20
|
+
import { batchLimits } from './batch-insert.js'
|
|
21
|
+
|
|
22
|
+
const ORDINAL = '_i'
|
|
23
|
+
|
|
24
|
+
export interface UpdateEachOptions<T extends PgTable, TKeys extends RowsColumns> {
|
|
25
|
+
/** The key rowset's columns, each with its SQL type: `{ id: 'uuid', ordinal: 'int' }`. Read in `where` and `set` as `k.id`. */
|
|
26
|
+
readonly keys: TKeys
|
|
27
|
+
/** One array per column, all the same length; element i of each is input key i. */
|
|
28
|
+
readonly data: RowsData<TKeys>
|
|
29
|
+
/** Which columns identify an input key: they must be distinct across inputs, and name the key in an error. Default: all of `keys`. */
|
|
30
|
+
readonly key?: ReadonlyArray<keyof TKeys & string>
|
|
31
|
+
/** The match for one key, as for `pipe.update().where()`: `and(eq(t.id, k.id), isNull(t.closedAt))`. */
|
|
32
|
+
readonly where: (k: RowsSource<'rows', TKeys>) => SQL | undefined
|
|
33
|
+
/** What to set; a key column is a valid value: `{ closedAt: k.ordinal }`. */
|
|
34
|
+
readonly set: (k: RowsSource<'rows', TKeys>) => UpdateSetValues<T>
|
|
35
|
+
/** `'one'` (default): every key must update exactly one row or the update is refused. `'any'`: no refusal, read the counts. */
|
|
36
|
+
readonly expect?: 'one' | 'any'
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function describeKey(columns: readonly string[], data: Record<string, readonly unknown[]>, i: number): string {
|
|
40
|
+
const part = (c: string): string => {
|
|
41
|
+
const v = data[c]![i]
|
|
42
|
+
return `${c}=${v instanceof Date ? v.toISOString() : JSON.stringify(v, (_k, x: unknown) => (typeof x === 'bigint' ? x.toString() : x))}`
|
|
43
|
+
}
|
|
44
|
+
return `{ ${columns.map(part).join(', ')} }`
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Updates the rows each input key matches in one statement and returns how many rows each key updated, in
|
|
49
|
+
* input order. By default a key that updated zero rows or more than one makes the call throw, naming the key,
|
|
50
|
+
* with the whole update undone; so does a key repeated in `data`. Runs in the caller's transaction and is audited
|
|
51
|
+
* as `pipe.update()` is (one audit row per row changed). An empty `data` returns `[]` without a statement.
|
|
52
|
+
*/
|
|
53
|
+
export async function updateEach<T extends DefinedTable<any> & PgTable, const TKeys extends RowsColumns>( // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
54
|
+
table: T,
|
|
55
|
+
options: UpdateEachOptions<T, TKeys>,
|
|
56
|
+
): Promise<number[]> {
|
|
57
|
+
assertNotExternal(table, 'update')
|
|
58
|
+
const names = Object.keys(options.keys)
|
|
59
|
+
if (names.includes(ORDINAL)) throw new Error(`[pipework] pipe.updateEach: "${ORDINAL}" is reserved for the input index; rename that key column.`)
|
|
60
|
+
const data = options.data as Record<string, readonly unknown[]>
|
|
61
|
+
const count = names.length === 0 ? 0 : (data[names[0]!]?.length ?? 0)
|
|
62
|
+
const identity: readonly string[] = options.key ?? names
|
|
63
|
+
for (const c of identity) {
|
|
64
|
+
if (!names.includes(c)) throw new Error(`[pipework] pipe.updateEach: key column "${c}" is not one of the key rowset's columns (${names.join(', ')}).`)
|
|
65
|
+
}
|
|
66
|
+
const seen = new Map<string, number>()
|
|
67
|
+
for (let i = 0; i < count; i++) {
|
|
68
|
+
const id = describeKey(identity, data, i)
|
|
69
|
+
const first = seen.get(id)
|
|
70
|
+
if (first !== undefined) {
|
|
71
|
+
throw new Error(`[pipework] pipe.updateEach: key ${id} appears at inputs ${String(first)} and ${String(i)}. Pass each key once; a repeated key would update one row twice.`)
|
|
72
|
+
}
|
|
73
|
+
seen.set(id, i)
|
|
74
|
+
}
|
|
75
|
+
if (count === 0) return []
|
|
76
|
+
|
|
77
|
+
// Over `batchLimits` the keys go as consecutive slices, in order, in the one transaction below: a refusal
|
|
78
|
+
// anywhere undoes every slice.
|
|
79
|
+
const slices: [number, number][] = []
|
|
80
|
+
let from = 0
|
|
81
|
+
let bytes = 0
|
|
82
|
+
for (let i = 0; i < count; i++) {
|
|
83
|
+
let rowBytes = 0
|
|
84
|
+
for (const c of names) { const v = data[c]![i]; rowBytes += typeof v === 'string' ? Buffer.byteLength(v) : 8 }
|
|
85
|
+
if (i > from && (i - from >= batchLimits.rows || bytes + rowBytes > batchLimits.bytes)) { slices.push([from, i]); from = i; bytes = 0 }
|
|
86
|
+
bytes += rowBytes
|
|
87
|
+
}
|
|
88
|
+
slices.push([from, count])
|
|
89
|
+
|
|
90
|
+
const dbName = table.$options.database
|
|
91
|
+
const runSlice = async (a: number, b: number): Promise<number[]> => {
|
|
92
|
+
const part = Object.fromEntries(names.map(c => [c, data[c]!.slice(a, b)]))
|
|
93
|
+
const source = rows({ ...options.keys, [ORDINAL]: 'int' }, { ...part, [ORDINAL]: Array.from({ length: b - a }, (_, i) => i) } as never)
|
|
94
|
+
const k = source as unknown as RowsSource<'rows', TKeys>
|
|
95
|
+
const where = options.where(k)
|
|
96
|
+
if (where === undefined) throw new Error('[pipework] pipe.updateEach: where() returned no condition. Match each key, e.g. eq(t.id, k.id): without one every key would update every row.')
|
|
97
|
+
return await executeWrite({
|
|
98
|
+
verb: 'update' as const,
|
|
99
|
+
table,
|
|
100
|
+
set: options.set(k) as Record<string, unknown>,
|
|
101
|
+
where,
|
|
102
|
+
each: { source, ordinal: (source as unknown as Record<string, SQL>)[ORDINAL]!, count: b - a },
|
|
103
|
+
}) as number[]
|
|
104
|
+
}
|
|
105
|
+
const run = async (): Promise<number[]> => {
|
|
106
|
+
const counts: number[] = []
|
|
107
|
+
for (const [a, b] of slices) for (const n of await runSlice(a, b)) counts.push(n)
|
|
108
|
+
if ((options.expect ?? 'one') === 'one') {
|
|
109
|
+
const bad = counts.findIndex(n => n !== 1)
|
|
110
|
+
if (bad !== -1) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
`[pipework] pipe.updateEach: key ${describeKey(identity, data, bad)} (input ${String(bad)}) matched ${String(counts[bad])} rows of "${table.$name}", expected exactly one. ` +
|
|
113
|
+
'Nothing was updated. A key matching none is absent or no longer eligible (check the where); a key matching several needs a narrower where. Pass expect: \'any\' to read the counts instead.',
|
|
114
|
+
)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return counts
|
|
118
|
+
}
|
|
119
|
+
return dbName === undefined ? pipe.transaction(run) : pipe.transaction(dbName, run)
|
|
120
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -11,7 +11,8 @@ export type { Aggregate, EmptyableAggregate, AggregateOptions, JsonOf } from './
|
|
|
11
11
|
export type { ValueColumnDef } from './data/db/values.js'
|
|
12
12
|
export type { RowsColumns, RowsData, RowsSource, RowsColumnValue } from './data/db/rows.js'
|
|
13
13
|
export type { ArraySource, ElementOf, JsonbElements } from './data/db/array-elements.js'
|
|
14
|
-
export type { FunctionSpec, FunctionSource, FunctionSourceFields } from './data/db/function-source.js'
|
|
14
|
+
export type { FunctionSpec, ScalarFunctionSpec, FunctionSource, FunctionSourceFields, DefinedCall, FunctionRef } from './data/db/function-source.js'
|
|
15
|
+
export type { FunctionDefinition, FunctionVolatility } from './data/db/function-definition.js'
|
|
15
16
|
export type { SerializableOptions } from './data/db/serializable.js'
|
|
16
17
|
export { StatementTimeoutError, LockTimeoutError } from './data/db/timeouts.js'
|
|
17
18
|
export type { TimeoutOptions } from './data/db/timeouts.js'
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { sql } from '../data/db/sql.js'
|
|
2
|
+
import { quoteIdentifier } from '../data/db/identifiers.js'
|
|
3
|
+
import type { PostgresJsDatabase } from '../data/query/postgres-js/index.js'
|
|
4
|
+
import type { SQL } from '../data/db/types.js'
|
|
5
|
+
import type { DefinedCall, FunctionRef } from '../data/db/function-source.js'
|
|
6
|
+
|
|
7
|
+
/** The state of the planner's statistics a measurement was taken under. */
|
|
8
|
+
export type StatisticsState = 'fresh' | 'stale' | 'missing'
|
|
9
|
+
|
|
10
|
+
export interface FunctionReadMeasurement {
|
|
11
|
+
/** The `history` the seed was called with. */
|
|
12
|
+
readonly history: number
|
|
13
|
+
readonly statistics: StatisticsState
|
|
14
|
+
/** Rows scanned by the plan's relation scans: (rows returned + rows removed by filter or recheck) x loops, summed. */
|
|
15
|
+
readonly touched: number
|
|
16
|
+
/** The scans, largest first, for the message that names where the rows went. */
|
|
17
|
+
readonly scans: readonly { readonly node: string; readonly relation: string; readonly touched: number }[]
|
|
18
|
+
/** The plan has a Function Scan on the function, or its call text: the call was not inlined. */
|
|
19
|
+
readonly calledAsFunction: boolean
|
|
20
|
+
/** The plan has a CTE Scan. */
|
|
21
|
+
readonly cteScan: boolean
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface FunctionReadReport {
|
|
25
|
+
readonly function: string
|
|
26
|
+
/** `not-checked` means the read touched no rows on the fixture, so nothing was proved either way. */
|
|
27
|
+
readonly status: 'passed' | 'failed' | 'not-checked'
|
|
28
|
+
readonly problems: readonly string[]
|
|
29
|
+
/** States that could not be measured (and why), so a pass never hides them: with any, `status` is `not-checked` unless something failed. */
|
|
30
|
+
readonly notChecked: readonly string[]
|
|
31
|
+
readonly measurements: readonly FunctionReadMeasurement[]
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface FunctionReadOptions {
|
|
35
|
+
/** The function under test: what `pipe.fn(...)` returned. */
|
|
36
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any typed call
|
|
37
|
+
readonly fn: DefinedCall<(...values: any[]) => unknown>
|
|
38
|
+
/** Representative arguments: keys that exist in the fixture at both sizes. */
|
|
39
|
+
readonly args: readonly unknown[]
|
|
40
|
+
/** The tables the function reads, `schema.table` (or `table` for public): their statistics are held and cleared by the check. */
|
|
41
|
+
readonly tables: readonly string[]
|
|
42
|
+
/**
|
|
43
|
+
* Leaves the database holding exactly `history` rows of history for the same keys the call looks up: it may
|
|
44
|
+
* truncate and refill. Called with the small size, then with the large one (ten times the small by default).
|
|
45
|
+
*/
|
|
46
|
+
readonly seed: (db: PostgresJsDatabase, history: number) => Promise<void>
|
|
47
|
+
/** The two history sizes. Default `[200, 2000]`. */
|
|
48
|
+
readonly history?: readonly [number, number]
|
|
49
|
+
/** How far a count may exceed its baseline: `baseline * (1 + relative) + absolute`. Default 10% and 5 rows. */
|
|
50
|
+
readonly tolerance?: { readonly relative?: number; readonly absolute?: number }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface PlanNode {
|
|
54
|
+
'Node Type'?: string
|
|
55
|
+
'Relation Name'?: string
|
|
56
|
+
'Schema'?: string
|
|
57
|
+
'Function Name'?: string
|
|
58
|
+
'Actual Rows'?: number
|
|
59
|
+
'Actual Loops'?: number
|
|
60
|
+
'Rows Removed by Filter'?: number
|
|
61
|
+
'Rows Removed by Index Recheck'?: number
|
|
62
|
+
Plans?: PlanNode[]
|
|
63
|
+
[key: string]: unknown
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function walk(node: PlanNode, visit: (n: PlanNode) => void): void {
|
|
67
|
+
visit(node)
|
|
68
|
+
for (const child of node.Plans ?? []) walk(child, visit)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function qualified(ref: FunctionRef): string {
|
|
72
|
+
return ref.schema === 'public' ? ref.name : `${ref.schema}.${ref.name}`
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Rows touched and the shape facts of one EXPLAIN (ANALYZE, VERBOSE, FORMAT JSON) plan. */
|
|
76
|
+
export function measurePlan(root: PlanNode, ref: FunctionRef, history: number, statistics: StatisticsState): FunctionReadMeasurement {
|
|
77
|
+
const scans: { node: string; relation: string; touched: number }[] = []
|
|
78
|
+
let calledAsFunction = false
|
|
79
|
+
let cteScan = false
|
|
80
|
+
const callText = new RegExp(`(?:^|[^\\w])${ref.name}\\s*\\(`)
|
|
81
|
+
walk(root, node => {
|
|
82
|
+
if (node['Relation Name'] !== undefined) {
|
|
83
|
+
const loops = node['Actual Loops'] ?? 1
|
|
84
|
+
const rows = (node['Actual Rows'] ?? 0) + (node['Rows Removed by Filter'] ?? 0) + (node['Rows Removed by Index Recheck'] ?? 0)
|
|
85
|
+
scans.push({
|
|
86
|
+
node: node['Node Type'] ?? 'Scan',
|
|
87
|
+
relation: `${node['Schema'] ?? 'public'}.${node['Relation Name']}`,
|
|
88
|
+
touched: rows * loops,
|
|
89
|
+
})
|
|
90
|
+
}
|
|
91
|
+
if (node['Node Type'] === 'CTE Scan') cteScan = true
|
|
92
|
+
if (node['Function Name'] === ref.name) calledAsFunction = true
|
|
93
|
+
for (const [key, value] of Object.entries(node)) {
|
|
94
|
+
if (key === 'Plans') continue
|
|
95
|
+
if ((key === 'Output' || key === 'Function Call' || key.endsWith('Filter') || key === 'Join Filter' || key === 'Index Cond') && callText.test(JSON.stringify(value))) calledAsFunction = true
|
|
96
|
+
}
|
|
97
|
+
})
|
|
98
|
+
scans.sort((a, b) => b.touched - a.touched)
|
|
99
|
+
return { history, statistics, touched: scans.reduce((n, s) => n + s.touched, 0), scans, calledAsFunction, cteScan }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
interface Catalog {
|
|
103
|
+
readonly language: string
|
|
104
|
+
readonly volatility: string
|
|
105
|
+
readonly config: readonly string[] | null
|
|
106
|
+
readonly securityDefiner: boolean
|
|
107
|
+
readonly strict: boolean
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Why Postgres would not inline the function, from its catalog row: the settings a plan cannot show. */
|
|
111
|
+
export function catalogProblems(name: string, c: Catalog | undefined): string[] {
|
|
112
|
+
if (c === undefined) return [`the function ${name} does not exist in the database. Run the migration that creates it before checking it.`]
|
|
113
|
+
const problems: string[] = []
|
|
114
|
+
if (c.config !== null && c.config.length > 0) {
|
|
115
|
+
problems.push(
|
|
116
|
+
`${name} has a SET clause (${c.config.join(', ')}). Postgres never inlines a function with a SET clause, so each call is a black box the planner cannot push keys into. ` +
|
|
117
|
+
'Remove it: a session setting such as jit belongs on the role (`ALTER ROLE app SET jit = off`).',
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
if (c.language !== 'sql') {
|
|
121
|
+
problems.push(`${name} is LANGUAGE ${c.language}. Postgres inlines only LANGUAGE sql functions. Write the read as one SELECT; a computation that is not one SELECT is a derivation, not a read.`)
|
|
122
|
+
}
|
|
123
|
+
if (c.volatility === 'v') problems.push(`${name} is VOLATILE. Postgres does not inline a volatile function. Declare it STABLE or IMMUTABLE.`)
|
|
124
|
+
if (c.securityDefiner) problems.push(`${name} is SECURITY DEFINER. Postgres does not inline a security definer function.`)
|
|
125
|
+
if (c.strict) problems.push(`${name} is STRICT. Postgres does not inline a strict set-returning function.`)
|
|
126
|
+
return problems
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async function readCatalog(db: PostgresJsDatabase, ref: FunctionRef): Promise<Catalog | undefined> {
|
|
130
|
+
const signature = `${quoteIdentifier(ref.schema, 'schema')}.${quoteIdentifier(ref.name, 'function')}(${ref.args.join(', ')})`
|
|
131
|
+
const rows = await db.execute<{ lang: string; vol: string; config: string[] | null; secdef: boolean; strict: boolean }>(sql`
|
|
132
|
+
select l.lanname as lang, p.provolatile::text as vol, p.proconfig as config, p.prosecdef as secdef, p.proisstrict as strict
|
|
133
|
+
from pg_proc p join pg_language l on l.oid = p.prolang
|
|
134
|
+
where p.oid = to_regprocedure(${signature})`)
|
|
135
|
+
const r = rows[0]
|
|
136
|
+
return r === undefined ? undefined : { language: r.lang, volatility: r.vol, config: r.config, securityDefiner: r.secdef, strict: r.strict }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function explain(db: PostgresJsDatabase, opts: FunctionReadOptions, history: number, statistics: StatisticsState): Promise<FunctionReadMeasurement> {
|
|
140
|
+
const source = opts.fn(...opts.args)
|
|
141
|
+
return await db.transaction(async tx => {
|
|
142
|
+
// Parallel workers would split the loop counts the measurement multiplies by.
|
|
143
|
+
await tx.execute(sql`set local max_parallel_workers_per_gather = 0`)
|
|
144
|
+
// The call is the typed source (or expression) pipe.fn returns, so the plan is the one the caller's own query gets.
|
|
145
|
+
const query = opts.fn.ref.setReturning
|
|
146
|
+
? tx.select().from(source as unknown as Parameters<ReturnType<typeof tx.select>['from']>[0])
|
|
147
|
+
: sql`select ${source as unknown as SQL} as value`
|
|
148
|
+
const rows = await tx.execute<{ 'QUERY PLAN': unknown }>(sql`explain (analyze, verbose, format json) ${query as unknown as SQL}`)
|
|
149
|
+
const raw = rows[0]?.['QUERY PLAN']
|
|
150
|
+
const parsed = (typeof raw === 'string' ? JSON.parse(raw) : raw) as { Plan: PlanNode }[]
|
|
151
|
+
return measurePlan(parsed[0]!.Plan, opts.fn.ref, history, statistics)
|
|
152
|
+
})
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async function setAutovacuum(db: PostgresJsDatabase, tables: readonly string[], on: boolean): Promise<void> {
|
|
156
|
+
for (const t of tables) {
|
|
157
|
+
try {
|
|
158
|
+
await db.execute(sql.raw(`alter table ${qualifiedTable(t)} ${on ? 'reset (autovacuum_enabled)' : 'set (autovacuum_enabled = false)'}`))
|
|
159
|
+
} catch (err) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
`[pipework] checkFunctionRead cannot change autovacuum on ${t}: ${err instanceof Error ? err.message : String(err)}\n\n` +
|
|
162
|
+
' Fix: run the check as the role that owns the tables (or a superuser). Autovacuum must be off on them for the run, or a background ANALYZE could turn the stale and missing measurements into fresh ones.\n',
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function qualifiedTable(table: string): string {
|
|
169
|
+
const parts = table.split('.')
|
|
170
|
+
if (parts.length > 2) throw new Error(`[pipework] checkFunctionRead: "${table}" is not a table name; use "schema.table" or "table".`)
|
|
171
|
+
return parts.map(p => quoteIdentifier(p, 'table')).join('.')
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const where = (m: FunctionReadMeasurement): string => m.scans.length === 0 ? '' : ` Largest scan: ${m.scans[0]!.node} on ${m.scans[0]!.relation} (${m.scans[0]!.touched} rows).`
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Plans one call of a pipework-defined read at two history sizes and under three states of the planner's
|
|
178
|
+
* statistics, and reports whether it is a key lookup: inlined into the calling query, touching a number of rows
|
|
179
|
+
* that does not grow with history, and no more of them under stale or missing statistics than after ANALYZE.
|
|
180
|
+
* Rows touched is counted from `EXPLAIN (ANALYZE)`: each relation scan's rows returned plus rows removed by
|
|
181
|
+
* filter or recheck, times its loops, summed. A read that touches no rows on the fixture is `not-checked`, never passed.
|
|
182
|
+
*
|
|
183
|
+
* The statistics states: after ANALYZE at the small size the large history is seeded without analyzing (stale), then
|
|
184
|
+
* analyzed (fresh), then the tables' pg_statistic rows are deleted (missing). Deleting them needs a superuser, as the
|
|
185
|
+
* test cluster's role is; autovacuum is switched off on the tables for the run and restored.
|
|
186
|
+
*/
|
|
187
|
+
export async function checkFunctionRead(db: PostgresJsDatabase, opts: FunctionReadOptions): Promise<FunctionReadReport> {
|
|
188
|
+
const ref = opts.fn.ref
|
|
189
|
+
const name = qualified(ref)
|
|
190
|
+
const [small, large] = opts.history ?? [200, 2000]
|
|
191
|
+
const relative = opts.tolerance?.relative ?? 0.1
|
|
192
|
+
const absolute = opts.tolerance?.absolute ?? 5
|
|
193
|
+
const analyze = async (): Promise<void> => { for (const t of opts.tables) await db.execute(sql.raw(`analyze ${qualifiedTable(t)}`)) }
|
|
194
|
+
|
|
195
|
+
const problems = catalogProblems(name, await readCatalog(db, ref))
|
|
196
|
+
const measurements: FunctionReadMeasurement[] = []
|
|
197
|
+
const role = (await db.execute<{ role: string; super: boolean }>(sql`select current_user::text as role, rolsuper as super from pg_roles where rolname = current_user`))[0]
|
|
198
|
+
const notChecked: string[] = []
|
|
199
|
+
if (role?.super !== true) {
|
|
200
|
+
notChecked.push(
|
|
201
|
+
`Missing statistics were not checked: role "${role?.role ?? 'unknown'}" is not a superuser, so it cannot delete the tables' pg_statistic rows. ` +
|
|
202
|
+
'Run the check as a superuser (the disposable test cluster\'s role; PIPEWORK_TEST_SUPERUSER_URL).',
|
|
203
|
+
)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
await setAutovacuum(db, opts.tables, false)
|
|
207
|
+
try {
|
|
208
|
+
await opts.seed(db, small)
|
|
209
|
+
await analyze()
|
|
210
|
+
const freshSmall = await explain(db, opts, small, 'fresh')
|
|
211
|
+
await opts.seed(db, large)
|
|
212
|
+
const stale = await explain(db, opts, large, 'stale')
|
|
213
|
+
await analyze()
|
|
214
|
+
const freshLarge = await explain(db, opts, large, 'fresh')
|
|
215
|
+
measurements.push(freshSmall, stale, freshLarge)
|
|
216
|
+
if (notChecked.length === 0) {
|
|
217
|
+
for (const t of opts.tables) await db.execute(sql`delete from pg_statistic where starelid = ${qualifiedTable(t)}::regclass`)
|
|
218
|
+
measurements.push(await explain(db, opts, large, 'missing'))
|
|
219
|
+
}
|
|
220
|
+
} finally {
|
|
221
|
+
await setAutovacuum(db, opts.tables, true)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
return judgeFunctionRead(name, problems, measurements, { relative, absolute }, notChecked)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* The verdict on four measurements (fresh at the small history, then stale, fresh and missing at the large one)
|
|
229
|
+
* and the problems the catalog already named. Pure, so each verdict can be tested without a database.
|
|
230
|
+
*/
|
|
231
|
+
export function judgeFunctionRead(
|
|
232
|
+
name: string,
|
|
233
|
+
catalog: readonly string[],
|
|
234
|
+
measurements: readonly FunctionReadMeasurement[],
|
|
235
|
+
tolerance: { readonly relative: number; readonly absolute: number },
|
|
236
|
+
notChecked: readonly string[] = [],
|
|
237
|
+
): FunctionReadReport {
|
|
238
|
+
const [freshSmall, stale, freshLarge, missing] = measurements as [FunctionReadMeasurement, FunctionReadMeasurement, FunctionReadMeasurement, FunctionReadMeasurement | undefined]
|
|
239
|
+
const allowed = (baseline: number): number => baseline * (1 + tolerance.relative) + tolerance.absolute
|
|
240
|
+
const problems = [...catalog]
|
|
241
|
+
const small = freshSmall.history
|
|
242
|
+
const large = freshLarge.history
|
|
243
|
+
const done = (status: FunctionReadReport['status']): FunctionReadReport => ({
|
|
244
|
+
function: name,
|
|
245
|
+
status: status === 'passed' && notChecked.length > 0 ? 'not-checked' : status,
|
|
246
|
+
problems: [...problems, ...notChecked],
|
|
247
|
+
notChecked,
|
|
248
|
+
measurements,
|
|
249
|
+
})
|
|
250
|
+
|
|
251
|
+
if (measurements.some(m => m.calledAsFunction)) {
|
|
252
|
+
if (problems.length === 0) {
|
|
253
|
+
problems.push(
|
|
254
|
+
`${name} was not inlined: the plan calls it as a function (a Function Scan), so the planner could not push the call's keys into its body. ` +
|
|
255
|
+
'Postgres inlines a function only when it is LANGUAGE sql, one SELECT, STABLE or IMMUTABLE, without SET, SECURITY DEFINER or STRICT, and (for a set) the body is a plain query.',
|
|
256
|
+
)
|
|
257
|
+
}
|
|
258
|
+
return done('failed')
|
|
259
|
+
}
|
|
260
|
+
if (freshLarge.touched === 0) {
|
|
261
|
+
if (problems.length > 0) return done('failed')
|
|
262
|
+
problems.push(`${name} was not exercised: the call touched no rows on the fixture at ${large} rows of history. Seed rows for the keys in \`args\` so the read has something to look up.`)
|
|
263
|
+
return done('not-checked')
|
|
264
|
+
}
|
|
265
|
+
if (freshLarge.touched > allowed(freshSmall.touched)) {
|
|
266
|
+
problems.push(
|
|
267
|
+
`${name} touched ${freshSmall.touched} rows at ${small} rows of history and ${freshLarge.touched} at ${large}: its rows touched grow with history, so it scans history instead of looking keys up.${where(freshLarge)}` +
|
|
268
|
+
(freshLarge.cteScan ? ' The plan has a CTE Scan: a MATERIALIZED CTE (or one used twice) is computed in full before the call\'s keys apply; use it once without MATERIALIZED, or write it as a subquery.' : '') +
|
|
269
|
+
' Give the read an index on its key columns and filter on them in the body.',
|
|
270
|
+
)
|
|
271
|
+
}
|
|
272
|
+
for (const m of [stale, missing]) {
|
|
273
|
+
if (m !== undefined && m.touched > allowed(freshLarge.touched)) {
|
|
274
|
+
problems.push(
|
|
275
|
+
`${name} touched ${m.touched} rows with ${m.statistics} statistics against ${freshLarge.touched} after ANALYZE (at ${large} rows of history): its plan depends on the statistics being current.${where(m)} ` +
|
|
276
|
+
'Make the plan the only reasonable one: an index on the key columns, and a filter the index serves.',
|
|
277
|
+
)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return done(problems.length === 0 ? 'passed' : 'failed')
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** `checkFunctionRead`, throwing a `[pipework]` error unless the read passed; `not-checked` throws too. */
|
|
284
|
+
export async function expectFunctionRead(db: PostgresJsDatabase, opts: FunctionReadOptions): Promise<FunctionReadReport> {
|
|
285
|
+
const report = await checkFunctionRead(db, opts)
|
|
286
|
+
if (report.status !== 'passed') {
|
|
287
|
+
throw new Error(`[pipework] function read ${report.function} ${report.status === 'failed' ? 'is not a key lookup' : 'was not checked'}:\n\n${report.problems.map(p => ` - ${p}`).join('\n')}\n`)
|
|
288
|
+
}
|
|
289
|
+
return report
|
|
290
|
+
}
|
package/src/test/index.ts
CHANGED
|
@@ -17,3 +17,4 @@ export { pipework as vitestPlugin, type PipeworkVitestOptions } from './vitest-p
|
|
|
17
17
|
export { resolveVitestConfig, resolveVitestModules } from './vitest-workspace.js'
|
|
18
18
|
export { tap, type ManagedConnection, type ConnectOptions } from '../data/db/pool.js'
|
|
19
19
|
export { crawlFixtureSite, type CrawlFixtureOptions, type CrawlFixtureRoute, type CrawlFixtureRequest, type CrawlFixtureSite } from './crawl-fixture.js'
|
|
20
|
+
export { checkFunctionRead, expectFunctionRead, judgeFunctionRead, measurePlan, catalogProblems, type FunctionReadOptions, type FunctionReadReport, type FunctionReadMeasurement, type StatisticsState } from './function-reads.js'
|