vouchington-tooling 0.38.0 → 0.38.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +1 -1
- package/dist/sql-ast/index.d.mts +0 -4
- package/dist/sql-ast/index.mjs +0 -2
- package/package.json +1 -1
- package/skills/bounded-iteration/SKILL.md +28 -5
- package/skills/bounded-iteration/references/sweep-patterns.md +375 -5
- package/skills/manifest.json +4 -4
- package/skills/postgres-node-performance-tuning/references/performance-patterns.md +7 -4
- package/skills/postgres-partitioning-uuid-v7/SKILL.md +9 -0
- package/skills/postgres-partitioning-uuid-v7/references/partition-lifecycle.md +36 -0
- package/dist/sql-ast/constraint-do-block.d.mts +0 -11
- package/dist/sql-ast/constraint-do-block.mjs +0 -84
- package/dist/sql-ast/constraint-shared.d.mts +0 -19
- package/dist/sql-ast/constraint-shared.mjs +0 -20
- package/dist/sql-ast/constraint.d.mts +0 -2
- package/dist/sql-ast/constraint.mjs +0 -63
- package/dist/sql-ast/implicit-indexes.d.mts +0 -3
- package/dist/sql-ast/implicit-indexes.mjs +0 -66
- package/dist/sql-ast/index-metadata.d.mts +0 -19
- package/dist/sql-ast/index-metadata.mjs +0 -100
package/dist/index.d.mts
CHANGED
|
@@ -12,8 +12,8 @@ export { buildSessionFrictionReport, classifyFrictionObservation, FRICTION_LOG_M
|
|
|
12
12
|
export type { FrictionEvent, FrictionEventKind, FrictionLogOptions, FrictionLogReadResult, FrictionObservation, JournalEntry, JournalLoader, JournalLoadResult, PermissionRequestObservation, SessionFrictionReport, SessionFrictionReportOptions, ToolResultObservation, } from './session-friction/index.mts';
|
|
13
13
|
export { EphemeralListenerAttemptsExhaustedError, isRunnerReservedPort, listenOnRunnerUnreservedEphemeralPort, loadRunnerPortPolicy, runnerPortPolicy, validateRunnerPortPolicy, } from './runner-port-policy/index.mts';
|
|
14
14
|
export type { EphemeralListenerOptions, RunnerPortPolicy } from './runner-port-policy/index.mts';
|
|
15
|
-
export {
|
|
16
|
-
export type {
|
|
15
|
+
export { extractCreateTableMetadata, extractDefaultFunction, extractFuncCallArgColumnNames, initSqlAst, lineOfUtf8ByteOffset, MissingSqlAstParserError, parseSql, } from './sql-ast/index.mts';
|
|
16
|
+
export type { SqlCreateTableColumn, SqlCreateTableMetadata } from './sql-ast/index.mts';
|
|
17
17
|
export { dollarQuoteEnd, lineOf, maskSqlQuotedText, readDollarQuoteDelimiter, readStringLiteral, splitSqlStatements, sqlFragments, stripSqlComments, } from './sql-scanner/index.mts';
|
|
18
18
|
export { auditCiJobRuntime, parseWorkflowNameMatch } from './gha-runtime-audit/index.mts';
|
|
19
19
|
export type { GhApiExecutor, RuntimeAuditOptions, RuntimeAuditResult, RuntimeAuditWorkflowFilter, RuntimeJobResult, RuntimeSample, } from './gha-runtime-audit/index.mts';
|
package/dist/index.mjs
CHANGED
|
@@ -6,7 +6,7 @@ export { runRetrospectiveFacts } from "./retrospective-facts/index.mjs";
|
|
|
6
6
|
export { appendJournal, assertSessionId, cleanupSnapshotPartitions, partitionSnapshot, probeBlackboard, readJournal, resolveBlackboardConnection, } from "./agent-blackboard/index.mjs";
|
|
7
7
|
export { buildSessionFrictionReport, classifyFrictionObservation, FRICTION_LOG_MAX_EVENTS, isConformingCiFailureBlock, normalizeCommandPrefix, readFrictionLog, recordFriction, } from "./session-friction/index.mjs";
|
|
8
8
|
export { EphemeralListenerAttemptsExhaustedError, isRunnerReservedPort, listenOnRunnerUnreservedEphemeralPort, loadRunnerPortPolicy, runnerPortPolicy, validateRunnerPortPolicy, } from "./runner-port-policy/index.mjs";
|
|
9
|
-
export {
|
|
9
|
+
export { extractCreateTableMetadata, extractDefaultFunction, extractFuncCallArgColumnNames, initSqlAst, lineOfUtf8ByteOffset, MissingSqlAstParserError, parseSql, } from "./sql-ast/index.mjs";
|
|
10
10
|
export { dollarQuoteEnd, lineOf, maskSqlQuotedText, readDollarQuoteDelimiter, readStringLiteral, splitSqlStatements, sqlFragments, stripSqlComments, } from "./sql-scanner/index.mjs";
|
|
11
11
|
export { auditCiJobRuntime, parseWorkflowNameMatch } from "./gha-runtime-audit/index.mjs";
|
|
12
12
|
export { checkGhaWorkspacePolicy } from "./gha-workspace-policy/index.mjs";
|
package/dist/sql-ast/index.d.mts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
export { extractMigrationConstraintMetadata } from './constraint.mts';
|
|
2
|
-
export type { ForeignKey, SqlMigrationConstraintMetadata } from './constraint-shared.mts';
|
|
3
1
|
export { extractCreateTableMetadata } from './create-table.mts';
|
|
4
2
|
export type { SqlCreateTableColumn, SqlCreateTableColumnConstraint, SqlCreateTableMetadata, } from './create-table.mts';
|
|
5
3
|
export { extractDefaultFunction, extractFuncCallArgColumnNames } from './default-function.mts';
|
|
6
|
-
export { extractCreateIndexMetadata } from './index-metadata.mts';
|
|
7
4
|
export { extractIndexShapes } from './index-shapes.mts';
|
|
8
|
-
export type { SqlCreateIndexMetadata, SqlIndexParam } from './index-metadata.mts';
|
|
9
5
|
export { lineOfUtf8ByteOffset } from './line-of-offset.mts';
|
|
10
6
|
export { initSqlAst, MissingSqlAstParserError, parseSql } from './parser.mts';
|
|
11
7
|
export { extractViewDeclarations } from './view-declarations.mts';
|
package/dist/sql-ast/index.mjs
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
export { extractMigrationConstraintMetadata } from "./constraint.mjs";
|
|
2
1
|
export { extractCreateTableMetadata } from "./create-table.mjs";
|
|
3
2
|
export { extractDefaultFunction, extractFuncCallArgColumnNames } from "./default-function.mjs";
|
|
4
|
-
export { extractCreateIndexMetadata } from "./index-metadata.mjs";
|
|
5
3
|
export { extractIndexShapes } from "./index-shapes.mjs";
|
|
6
4
|
export { lineOfUtf8ByteOffset } from "./line-of-offset.mjs";
|
|
7
5
|
export { initSqlAst, MissingSqlAstParserError, parseSql } from "./parser.mjs";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vouchington-tooling",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.1",
|
|
4
4
|
"description": "Vouchington CLI and extractable tooling libraries.",
|
|
5
5
|
"homepage": "https://github.com/vouchington/vouchington-tooling/tree/main/packages/vouchington-tooling#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -15,6 +15,10 @@ A run's cost is the rows it reads and the work it starts, not the memory it hold
|
|
|
15
15
|
serves: a work-item or dirty-marker row, a parent id, a due timestamp with a partial index, or
|
|
16
16
|
a time or UUIDv7 window past a high-water mark. Column-to-column comparisons, `OR` of unrelated
|
|
17
17
|
conditions, and `EXISTS` as the only condition are not filters. (`postgres-required-predicates`)
|
|
18
|
+
A per-row threshold becomes one bucket per distinct value, comparing the stored fact with a
|
|
19
|
+
bound, never with another column; `timestamptz + interval` is not immutable, so it cannot be
|
|
20
|
+
indexed. See [per-row interval](references/sweep-patterns.md#per-row-interval-one-bucket-per-threshold)
|
|
21
|
+
and [age milestones](references/sweep-patterns.md#age-milestones-from-the-uuidv7-key).
|
|
18
22
|
2. **Cap every run.** Every loop has a per-run cap (batches, rows, or a deadline), reports whether
|
|
19
23
|
work remains, and resumes from a persisted position. Never loop until empty, and never default
|
|
20
24
|
a cap to infinity.
|
|
@@ -36,12 +40,31 @@ A run's cost is the rows it reads and the work it starts, not the memory it hold
|
|
|
36
40
|
8. **Tunables are runtime configuration.** Batch sizes, page sizes, caps, and windows come from
|
|
37
41
|
the consumer's runtime configuration with a validated default and a hard maximum, and are bound
|
|
38
42
|
as SQL parameters. `LIMIT 1` and an external API's page maximum are contracts, not tunables.
|
|
39
|
-
9. **
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
9. **One dispatcher per signal.** Split an `OR` of unrelated reasons into one dispatcher per
|
|
44
|
+
reason, each with its own index-served predicate, position, and schedule, all enqueuing the same
|
|
45
|
+
idempotent job. Writes that cause a signal enqueue directly; the sweep is the backstop. See
|
|
46
|
+
[one dispatcher per signal](references/sweep-patterns.md#one-dispatcher-per-signal).
|
|
47
|
+
10. **Deduplicate jobs by entity and triggering event.** A job id keyed only by the entity drops a
|
|
48
|
+
new trigger while the last job runs or is retained. Key it by entity and event; dedup lasts
|
|
49
|
+
only while the record is retained, and a failed retained job still claims its id. See
|
|
50
|
+
[queue job ids](references/sweep-patterns.md#queue-job-ids-for-deduplication).
|
|
51
|
+
11. **Repair through an existing reconciler.** Repeat a fire-and-forget side effect (cache fill,
|
|
52
|
+
filter add, enqueue) idempotently from the reconciler that already sweeps recently changed
|
|
53
|
+
rows. Do not add a sweeper. See
|
|
54
|
+
[repair](references/sweep-patterns.md#repair-through-an-existing-reconciler).
|
|
55
|
+
12. **Membership filters are never rebuilt on a schedule.** Add on write, rebuild in full only on
|
|
56
|
+
an operator action or a detected failure, and stop trusting the filter after a failed add.
|
|
57
|
+
Deleted keys and growth only cost false positives. See
|
|
58
|
+
[membership filters](references/sweep-patterns.md#membership-filters).
|
|
59
|
+
13. **Global rankings stay one periodic aggregate.** A result over every row needs every score;
|
|
60
|
+
incremental per-row inputs move the cost to every write without freshening the ranking. Give
|
|
61
|
+
the pass an EXPLAIN budget. See [global ranking](references/sweep-patterns.md#global-ranking).
|
|
62
|
+
14. **External listings carry the provider's continuation token.** Cap pages per run and enqueue a
|
|
63
|
+
continuation with the token; restarting at page one lets failing front items starve the rest.
|
|
64
|
+
See [external listing](references/sweep-patterns.md#external-listing-with-a-continuation-token).
|
|
65
|
+
15. **Verify with a plan.** EXPLAIN at representative cardinality shows no Seq Scan on an
|
|
43
66
|
unbounded table, and an index that supplies the `ORDER BY`, so the scan stops at the
|
|
44
|
-
`LIMIT` instead of reading and sorting every match.
|
|
67
|
+
`LIMIT` instead of reading and sorting every match. Compare rows read with rows returned.
|
|
45
68
|
|
|
46
69
|
For work-item, lease, and cursor tables, read [postgres-schema-design](../postgres-schema-design/SKILL.md).
|
|
47
70
|
For query shape and streaming, read [performance patterns](../postgres-node-performance-tuning/references/performance-patterns.md).
|
|
@@ -293,12 +293,376 @@ const fresh = candidates.filter((ref) => !known.has(ref))
|
|
|
293
293
|
When the goal is only to insert the new rows, `INSERT ... ON CONFLICT DO NOTHING RETURNING` skips
|
|
294
294
|
the lookup.
|
|
295
295
|
|
|
296
|
-
##
|
|
296
|
+
## Per-row interval: one bucket per threshold
|
|
297
|
+
|
|
298
|
+
When the rule lives on the row, such as a per-site refresh interval, compare the stored fact with
|
|
299
|
+
a bound per distinct threshold value, never with another column. `swept_at` stores when the last
|
|
300
|
+
_completed_ sweep started; set it from the sweep's own start time when the sweep reports no more
|
|
301
|
+
work. Each interval value is a bucket the index can seek into. A recursive CTE walks the leading
|
|
302
|
+
column of the index one distinct value at a time (a loose index scan, or skip scan), so finding
|
|
303
|
+
the buckets reads one index entry per bucket, not every enabled site. Each bucket is capped on its
|
|
304
|
+
own, so one busy bucket cannot starve the others. Each branch inside a bucket has its own cap too,
|
|
305
|
+
so a steady stream of never-swept sites cannot starve overdue rows. A run returns at most
|
|
306
|
+
2 times `$1` rows per bucket:
|
|
297
307
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
308
|
+
```sql
|
|
309
|
+
CREATE INDEX idx_sites__refresh_due
|
|
310
|
+
ON sites (refresh_interval_days, swept_at NULLS FIRST, id)
|
|
311
|
+
WHERE is_enabled;
|
|
312
|
+
|
|
313
|
+
WITH RECURSIVE bucket AS (
|
|
314
|
+
(SELECT refresh_interval_days FROM sites
|
|
315
|
+
WHERE is_enabled
|
|
316
|
+
ORDER BY refresh_interval_days LIMIT 1)
|
|
317
|
+
UNION ALL
|
|
318
|
+
SELECT next_bucket.refresh_interval_days
|
|
319
|
+
FROM bucket
|
|
320
|
+
CROSS JOIN LATERAL (
|
|
321
|
+
SELECT refresh_interval_days FROM sites
|
|
322
|
+
WHERE is_enabled AND refresh_interval_days > bucket.refresh_interval_days
|
|
323
|
+
ORDER BY refresh_interval_days LIMIT 1
|
|
324
|
+
) next_bucket
|
|
325
|
+
)
|
|
326
|
+
SELECT due.id, bucket.refresh_interval_days
|
|
327
|
+
FROM bucket
|
|
328
|
+
CROSS JOIN LATERAL (
|
|
329
|
+
(SELECT id FROM sites
|
|
330
|
+
WHERE is_enabled AND refresh_interval_days = bucket.refresh_interval_days
|
|
331
|
+
AND swept_at IS NULL
|
|
332
|
+
ORDER BY id LIMIT $1)
|
|
333
|
+
UNION ALL
|
|
334
|
+
(SELECT id FROM sites
|
|
335
|
+
WHERE is_enabled AND refresh_interval_days = bucket.refresh_interval_days
|
|
336
|
+
AND swept_at < now() - bucket.refresh_interval_days * interval '1 day'
|
|
337
|
+
ORDER BY swept_at, id LIMIT $1)
|
|
338
|
+
) due;
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
The loose scan beats a small thresholds table here because the intervals are chosen per row and
|
|
342
|
+
there is no second source of truth to drift. If the intervals are a fixed, reviewed set, a
|
|
343
|
+
thresholds table is just as bounded. Either way the bucket count must itself be bounded, for
|
|
344
|
+
example by a `CHECK` that restricts `refresh_interval_days` to the allowed values. Declare the
|
|
345
|
+
column `NOT NULL` with a default as well: a NULL interval is never enumerated by the loose scan
|
|
346
|
+
(`>` skips NULL) and never compares as due, so that site would never be swept.
|
|
347
|
+
|
|
348
|
+
Comparing `swept_at < now() - refresh_interval_days * interval '1 day'` alone cannot use an index:
|
|
349
|
+
the right side depends on the row's own column, and `timestamptz + interval` is not immutable, so
|
|
350
|
+
the sum cannot be indexed or stored as a generated column. Inside a bucket the interval is a
|
|
351
|
+
constant, so the bound is a plain range on `swept_at`.
|
|
352
|
+
|
|
353
|
+
Each condition is its own branch. `swept_at IS NULL OR swept_at < $bound` is not one index range:
|
|
354
|
+
the planner makes `refresh_interval_days = k` the index condition and applies the `OR` as a filter.
|
|
355
|
+
When fewer rows are due than the cap, that scan reads every row in the bucket. Verify by rows, not
|
|
356
|
+
scan type: the plan's actual rows read stay close to the rows returned, and `Rows Removed by
|
|
357
|
+
Filter` is near zero.
|
|
358
|
+
|
|
359
|
+
## Age milestones from the UUIDv7 key
|
|
360
|
+
|
|
361
|
+
An account crosses age `M` when `now - M` passes its creation time, which the key encodes. So
|
|
362
|
+
"crossed `M` since the last run" is a primary-key range with no stored column. `$1` is the previous
|
|
363
|
+
run's time minus an overlap (the window start), `$2` the window end, fixed when the window opens,
|
|
364
|
+
`$3` the milestone interval, `$4` the last `id` already read (the nil UUID at the start of a
|
|
365
|
+
window), and `$5` the cap. This holds only when ids are minted at creation and never for imports
|
|
366
|
+
or backfills; where that is not guaranteed, drive milestones from an indexed immutable `created_at`
|
|
367
|
+
range instead (see the [parent lower bound](../../postgres-partitioning-uuid-v7/references/partition-lifecycle.md#parent-lower-bound)
|
|
368
|
+
conditions on historical ids):
|
|
369
|
+
|
|
370
|
+
```sql
|
|
371
|
+
CREATE INDEX idx_accounts__open ON accounts (id) WHERE closed_at IS NULL;
|
|
372
|
+
|
|
373
|
+
SELECT id FROM accounts
|
|
374
|
+
WHERE id >= min_uuidv7($1::timestamptz - $3::interval)
|
|
375
|
+
AND id < min_uuidv7($2::timestamptz - $3::interval)
|
|
376
|
+
AND id > $4
|
|
377
|
+
AND closed_at IS NULL
|
|
378
|
+
ORDER BY id
|
|
379
|
+
LIMIT $5;
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
Without the partial index, `closed_at IS NULL` is a filter on the primary-key range, and a window
|
|
383
|
+
where most accounts are closed reads every one of them. Verify by rows: the plan's rows read stay
|
|
384
|
+
close to the rows returned.
|
|
385
|
+
|
|
386
|
+
A window can hold more accounts than the cap, so persist the last `id` returned after each batch
|
|
387
|
+
and page inside the fixed window with `id > $4`. Advance the window only after a short batch: the
|
|
388
|
+
next window starts an overlap before this one's end (`$2`), with the nil UUID as `$4`. Moving the
|
|
389
|
+
start to the window's end after a capped batch would skip the accounts not yet read. This is the
|
|
390
|
+
`age_milestone` signal's `last_id` in the dispatcher cursor table below.
|
|
391
|
+
|
|
392
|
+
## One dispatcher per signal
|
|
393
|
+
|
|
394
|
+
The anti-pattern is one keyset walk over every account with an `OR` of unrelated reasons. No index
|
|
395
|
+
serves the `OR`, so every account is read to find the few that need work:
|
|
396
|
+
|
|
397
|
+
```sql
|
|
398
|
+
SELECT a.id
|
|
399
|
+
FROM accounts a
|
|
400
|
+
WHERE a.id > $1
|
|
401
|
+
AND (a.id < min_uuidv7(now() - $2::interval)
|
|
402
|
+
OR EXISTS (SELECT 1 FROM subscriptions s
|
|
403
|
+
WHERE s.account_id = a.id AND s.expires_at < now())
|
|
404
|
+
OR EXISTS (SELECT 1 FROM plan_changes c
|
|
405
|
+
WHERE c.account_id = a.id AND c.id > $3))
|
|
406
|
+
ORDER BY a.id
|
|
407
|
+
LIMIT $4;
|
|
408
|
+
```
|
|
409
|
+
|
|
410
|
+
Replace it with one dispatcher per signal, each with its own index-served predicate, position, and
|
|
411
|
+
schedule. A signal backed by several timestamp columns reads one indexed range per column, joined
|
|
412
|
+
with `UNION ALL`, not an `OR` across columns. A window can hold more rows than the cap, so each
|
|
413
|
+
dispatcher pages inside it with a composite keyset position, as in the high-water-mark window.
|
|
414
|
+
`$1` is the window end, `$2` and `$3` the position (`last_at`, `last_id`), and `$4` the cap:
|
|
415
|
+
|
|
416
|
+
```sql
|
|
417
|
+
-- age milestones: the accounts range above, with `id > $3` as the position and
|
|
418
|
+
-- `id < min_uuidv7($1 - milestone)` as the window end
|
|
419
|
+
|
|
420
|
+
-- subscription expirations: one indexed range per timestamp column, merged by (at, id)
|
|
421
|
+
-- (catches only expiries that pass in real time; see the note below)
|
|
422
|
+
SELECT at, id, account_id FROM (
|
|
423
|
+
(SELECT expires_at AS at, id, account_id FROM subscriptions
|
|
424
|
+
WHERE (expires_at, id) > ($2, $3) AND expires_at < $1
|
|
425
|
+
ORDER BY expires_at, id LIMIT $4)
|
|
426
|
+
UNION ALL
|
|
427
|
+
(SELECT grace_ends_at AS at, id, account_id FROM subscriptions
|
|
428
|
+
WHERE (grace_ends_at, id) > ($2, $3) AND grace_ends_at < $1
|
|
429
|
+
ORDER BY grace_ends_at, id LIMIT $4)
|
|
430
|
+
) either_column
|
|
431
|
+
ORDER BY at, id
|
|
432
|
+
LIMIT $4;
|
|
433
|
+
|
|
434
|
+
-- plan changes: a UUIDv7 window, with `last_id` (seeded at the overlap boundary) as the position
|
|
435
|
+
SELECT id, account_id FROM plan_changes
|
|
436
|
+
WHERE id > $3 AND id < min_uuidv7($1)
|
|
437
|
+
ORDER BY id
|
|
438
|
+
LIMIT $4;
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
Each dispatcher keeps its window end and position in one cursor table keyed by an enum:
|
|
442
|
+
|
|
443
|
+
```sql
|
|
444
|
+
CREATE TYPE account_recalc_signals AS ENUM ('age_milestone', 'subscription_expiry', 'plan_change');
|
|
445
|
+
|
|
446
|
+
CREATE TABLE account_recalc_cursors (
|
|
447
|
+
signal account_recalc_signals PRIMARY KEY,
|
|
448
|
+
changes_through_at timestamptz NOT NULL,
|
|
449
|
+
last_at timestamptz NOT NULL,
|
|
450
|
+
last_id uuid NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'
|
|
451
|
+
);
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
`changes_through_at` is the window end, fixed when the window opens. Advance `(last_at, last_id)`
|
|
455
|
+
after each batch with a compare-and-set. When a batch is short, the window is drained: the next
|
|
456
|
+
window opens with a new `changes_through_at`, and its position restarts an overlap before the
|
|
457
|
+
previous end. A UUIDv7-keyed signal (`plan_change`) seeds `last_id` with
|
|
458
|
+
`min_uuidv7(previous end - overlap)`, never the nil UUID, which would rescan from the start of
|
|
459
|
+
the table; a timestamp-keyed signal sets `last_at` to the same boundary with the nil `last_id`.
|
|
460
|
+
|
|
461
|
+
`expires_at` and `grace_ends_at` are mutable business timestamps, the same trap as using
|
|
462
|
+
`occurred_at` as a high-water mark in the
|
|
463
|
+
[high-water-mark window](#high-water-mark-window-with-an-overlap). A row updated or imported with
|
|
464
|
+
an expiry older than the cursor position is never selected, so the dispatcher catches only
|
|
465
|
+
expiries that pass in real time. Any write that sets an expiry at or before now, or before the
|
|
466
|
+
cursor, enqueues the job directly in the write path or records an immutable change event
|
|
467
|
+
(`plan_changes` is one) that a UUIDv7-keyed dispatcher reads.
|
|
468
|
+
|
|
469
|
+
All of them enqueue the same `recalculateAccount` job with the per-event ids below, so a repeat
|
|
470
|
+
read is harmless. Writes that cause a signal, such as recording a plan change, enqueue the job in
|
|
471
|
+
the write path. The dispatchers are the backstop for a missed enqueue.
|
|
472
|
+
|
|
473
|
+
## Queue job ids for deduplication
|
|
474
|
+
|
|
475
|
+
A queue that deduplicates by job id ignores an add whose id already exists, including a job that is
|
|
476
|
+
running or retained after completion. An id keyed only by the entity drops a new trigger:
|
|
477
|
+
|
|
478
|
+
```ts
|
|
479
|
+
// Account A's job is running when A's subscription expires: this second add is ignored.
|
|
480
|
+
await queue.add('recalculateAccount', { accountId }, { jobId: `recalc__${accountId}` })
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
Key the id by the entity and the event instead. Re-reading the same event in an overlap gives the
|
|
484
|
+
same id, and a new event gets a new id:
|
|
485
|
+
|
|
486
|
+
```ts
|
|
487
|
+
await queue.add(
|
|
488
|
+
'recalculateAccount',
|
|
489
|
+
{ accountId, changeId },
|
|
490
|
+
{ jobId: `recalc__${accountId}__planChanged__${changeId}` },
|
|
491
|
+
)
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
While the job record is retained, each trigger is enqueued at most once; that does not mean it
|
|
495
|
+
runs. Deduplication lasts only as long as the record: retention must cover the longest overlap or
|
|
496
|
+
replay interval, or a re-read event enqueues again after the record is removed. A requirement that
|
|
497
|
+
a trigger take effect exactly once needs a durable event ledger, not job ids. If a job fails for
|
|
498
|
+
good and its record is retained, the same id is still claimed, so retrying it needs an explicit
|
|
499
|
+
reactivate-or-remove step before the next add.
|
|
500
|
+
|
|
501
|
+
## Repair through an existing reconciler
|
|
502
|
+
|
|
503
|
+
When a side effect after commit is fire-and-forget (a cache fill, a filter add, an enqueue), repair
|
|
504
|
+
it in the reconciler that already reads a `changed_through_at` window of recently changed rows. Do
|
|
505
|
+
not create a new sweeper. `$1` and `$2` are the position (`changed_at`, `id`), seeded an overlap
|
|
506
|
+
before the previous window's end, `$3` the window end fixed when the window opens, and `$4` the cap:
|
|
507
|
+
|
|
508
|
+
```sql
|
|
509
|
+
SELECT id, handle, version
|
|
510
|
+
FROM accounts
|
|
511
|
+
WHERE (changed_at, id) > ($1, $2) AND changed_at <= $3
|
|
512
|
+
ORDER BY changed_at, id
|
|
513
|
+
LIMIT $4;
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
Advance the position after each batch with a compare-and-set, and open the next window only after a
|
|
517
|
+
short batch, as in the dispatcher cursor table.
|
|
518
|
+
|
|
519
|
+
```ts
|
|
520
|
+
for (const account of await db.query(recentlyChangedAccountsSql, [
|
|
521
|
+
...position,
|
|
522
|
+
to,
|
|
523
|
+
config.batchSize,
|
|
524
|
+
])) {
|
|
525
|
+
// Monotonic: the write compares `version`, so a delayed repair never overwrites a newer value.
|
|
526
|
+
await accountCache.fillIfNotOlder(account.id, account, account.version)
|
|
527
|
+
await addHandle(account.handle) // idempotent; the invalidating helper from membership filters
|
|
528
|
+
}
|
|
529
|
+
```
|
|
530
|
+
|
|
531
|
+
Idempotence alone is not enough, because the overlap reads rows twice and a delayed repair can land
|
|
532
|
+
after a newer write. Make each repair monotonic: compare an integer `version` that every update
|
|
533
|
+
increments (`version = version + 1`) on write, or invalidate the cache entry and let the next read
|
|
534
|
+
load the current row. Do not compare `changed_at = now()`: it is the transaction's start time and
|
|
535
|
+
is not monotonic across concurrent transactions.
|
|
536
|
+
|
|
537
|
+
## Membership filters
|
|
538
|
+
|
|
539
|
+
A membership filter needs every key, but never on a schedule. The write path adds immediately after
|
|
540
|
+
commit. A failed add stops trusting the filter, by clearing its ready marker so reads fall back to
|
|
541
|
+
the source of truth, and enqueues a rebuild. The job id includes the filter generation: a fixed id
|
|
542
|
+
would stay claimed once its record is retained, so a later failure could never enqueue. Each
|
|
543
|
+
invalidation bumps the generation, so concurrent enqueues for one failure share an id and a new
|
|
544
|
+
failure gets a new one. The reconciler repairs missed adds from its window, and a read that finds
|
|
545
|
+
the filter missing enqueues the same rebuild:
|
|
546
|
+
|
|
547
|
+
```ts
|
|
548
|
+
async function enqueueRebuild(generation: number, afterId = 'start'): Promise<void> {
|
|
549
|
+
try {
|
|
550
|
+
const jobId = `rebuildFilter__handles__${generation}__${afterId}`
|
|
551
|
+
await queue.add('rebuildFilter', { generation, afterId }, { jobId })
|
|
552
|
+
} catch (error) {
|
|
553
|
+
logger.warn({ error }, 'could not enqueue the filter rebuild') // best effort
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
export async function addHandle(handle: string): Promise<void> {
|
|
558
|
+
try {
|
|
559
|
+
await handleFilter.add(handle)
|
|
560
|
+
} catch {
|
|
561
|
+
await enqueueRebuild(await handleFilter.invalidate()) // clears ready, bumps the generation
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
export async function hasHandle(handle: string): Promise<boolean> {
|
|
566
|
+
if (!(await handleFilter.isReady())) {
|
|
567
|
+
await enqueueRebuild(await handleFilter.generation())
|
|
568
|
+
return handleExistsInDatabase(handle) // the read falls back even if the enqueue failed
|
|
569
|
+
}
|
|
570
|
+
// Hint only: a negative means "definitely absent" for keys committed before the last
|
|
571
|
+
// successful add or rebuild. A positive may be a false positive, so verify it at the source.
|
|
572
|
+
try {
|
|
573
|
+
if (!(await handleFilter.has(handle))) return false
|
|
574
|
+
} catch {
|
|
575
|
+
await enqueueRebuild(await handleFilter.invalidate()) // a failing filter is not trusted
|
|
576
|
+
}
|
|
577
|
+
return handleExistsInDatabase(handle)
|
|
578
|
+
}
|
|
579
|
+
```
|
|
580
|
+
|
|
581
|
+
The add happens after commit, so while it is in flight a committed handle can be missing from a
|
|
582
|
+
ready filter. The filter is therefore only a hint for callers that tolerate a just-committed miss,
|
|
583
|
+
such as an availability pre-check. Correctness-critical paths, such as a uniqueness check or a
|
|
584
|
+
read-your-writes lookup, use the database or its unique constraint and never trust a negative.
|
|
585
|
+
|
|
586
|
+
The rebuild is an explicit, capped, resumable backfill with a keyset cursor, as in the
|
|
587
|
+
high-water-mark window. Each capped page enqueues its continuation with the cursor in the job id
|
|
588
|
+
(`rebuildFilter__handles__${generation}__${afterId}`), because a generation-only id would block the
|
|
589
|
+
next page of the same generation. Each page checks that its generation is still current before
|
|
590
|
+
scanning and again before enqueueing a continuation, and a stale page exits without doing
|
|
591
|
+
anything. The rebuild captures its generation when it starts and sets the
|
|
592
|
+
ready marker last, by a compare-and-set that succeeds only while that generation is still current
|
|
593
|
+
(`markReady(generation)`), so a stale rebuild never overwrites a newer invalidation. Deleted keys and growth past
|
|
594
|
+
capacity only cost false positives, so they are an operator's rebuild, not a schedule.
|
|
595
|
+
|
|
596
|
+
## Global ranking
|
|
597
|
+
|
|
598
|
+
A result that depends on every row, such as a percentile tier, needs every row's score whatever the
|
|
599
|
+
design. Keep one periodic aggregate. `$1` and `$2` are the tier cutoffs:
|
|
600
|
+
|
|
601
|
+
```sql
|
|
602
|
+
WITH per_plan AS (
|
|
603
|
+
SELECT s.channel_id, s.plan_id, count(*) AS subscribers
|
|
604
|
+
FROM subscriptions s
|
|
605
|
+
WHERE s.canceled_at IS NULL
|
|
606
|
+
GROUP BY s.channel_id, s.plan_id
|
|
607
|
+
), weighted AS (
|
|
608
|
+
SELECT p.channel_id, sum(p.subscribers * plan.subscriber_weight) AS score
|
|
609
|
+
FROM per_plan p
|
|
610
|
+
JOIN plans plan ON plan.id = p.plan_id
|
|
611
|
+
GROUP BY p.channel_id
|
|
612
|
+
), scores AS (
|
|
613
|
+
-- Start from channels so one with no active subscribers scores 0 instead of dropping out.
|
|
614
|
+
SELECT c.id AS channel_id, coalesce(w.score, 0) AS score
|
|
615
|
+
FROM channels c
|
|
616
|
+
LEFT JOIN weighted w ON w.channel_id = c.id
|
|
617
|
+
), ranked AS (
|
|
618
|
+
SELECT channel_id, PERCENT_RANK() OVER (ORDER BY score) AS percentile, score
|
|
619
|
+
FROM scores
|
|
620
|
+
)
|
|
621
|
+
SELECT channel_id,
|
|
622
|
+
CASE WHEN score = 0 THEN 'low'
|
|
623
|
+
WHEN percentile >= $1 THEN 'top'
|
|
624
|
+
WHEN percentile >= $2 THEN 'mid'
|
|
625
|
+
ELSE 'low' END AS tier
|
|
626
|
+
FROM ranked;
|
|
627
|
+
```
|
|
628
|
+
|
|
629
|
+
`PERCENT_RANK()` gives tied scores the lowest rank, so a large tie at score 0 stays at percentile 0
|
|
630
|
+
instead of reading as a high cumulative fraction, and score 0 is forced into the low tier. Write
|
|
631
|
+
back only the changed tiers, in capped keyset batches over the result. Keeping a subscriber
|
|
632
|
+
count current on every subscribe and unsubscribe is more total work: it makes one hot `channels`
|
|
633
|
+
row take every write, and a plan change fans out to every channel the account follows. It does not
|
|
634
|
+
make tiers fresher either, because the percentile still needs every channel's score. Give the one
|
|
635
|
+
aggregate pass an EXPLAIN budget at representative size.
|
|
636
|
+
|
|
637
|
+
## External listing with a continuation token
|
|
638
|
+
|
|
639
|
+
An object-storage or API listing loop caps the pages per run. When more remain, it enqueues a
|
|
640
|
+
continuation job carrying the provider's continuation token. Restarting from page one each run is
|
|
641
|
+
not a bound: items that keep failing at the front starve the rest.
|
|
642
|
+
|
|
643
|
+
```ts
|
|
644
|
+
export async function listingJob(
|
|
645
|
+
{ token, traversalId = randomUUID() }: { token?: string; traversalId?: string },
|
|
646
|
+
config: ListingConfig,
|
|
647
|
+
): Promise<void> {
|
|
648
|
+
let next = token // traversalId is minted when a traversal starts at page one
|
|
649
|
+
for (let page = 0; page < config.maxPagesPerRun; page += 1) {
|
|
650
|
+
const result = await provider.list({ continuationToken: next, maxKeys: config.pageSize })
|
|
651
|
+
await enqueueItems(result.items)
|
|
652
|
+
if (result.nextToken !== undefined && result.nextToken === next) {
|
|
653
|
+
throw new Error('Provider repeated its continuation token') // never loop or re-enqueue it
|
|
654
|
+
}
|
|
655
|
+
next = result.nextToken
|
|
656
|
+
if (!next) return
|
|
657
|
+
}
|
|
658
|
+
// The traversal id keeps a provider that reuses a token across traversals from colliding.
|
|
659
|
+
await queue.add(
|
|
660
|
+
'listing',
|
|
661
|
+
{ token: next, traversalId },
|
|
662
|
+
{ jobId: `listing__${traversalId}__${next}` },
|
|
663
|
+
)
|
|
664
|
+
}
|
|
665
|
+
```
|
|
302
666
|
|
|
303
667
|
## Verify with a plan
|
|
304
668
|
|
|
@@ -338,5 +702,11 @@ claim statement, and check the index scan's actual rows while several workers cl
|
|
|
338
702
|
| A key-only walk: select every account id, then check each | Touches every entity to find the few with work | Read work items, dirty markers, or a due timestamp |
|
|
339
703
|
| A literal `LIMIT 500` in SQL text | Cannot be tuned per environment or during an incident | A validated tunable with a hard maximum, bound as `LIMIT $1` |
|
|
340
704
|
| `Promise.all(accounts.map(sync))` over a result of any size | One task per row exhausts the pool or the upstream API | A bounded batch with bounded concurrency |
|
|
705
|
+
| An `OR` across unrelated signals in one walk | No index serves the `OR`, so every account is read | One dispatcher per signal, each with its own range |
|
|
706
|
+
| Comparing a fact column with a per-row interval column | The sum is not indexable, so every row is read | One bucket per interval value, bounded by a plain range |
|
|
707
|
+
| Job ids keyed only by entity | A running or retained job silently drops a new trigger | An id keyed by entity and triggering event |
|
|
708
|
+
| A new sweeper beside an existing reconciler | A second pass over rows the reconciler already reads | Add the idempotent repair to the existing reconciler |
|
|
709
|
+
| A scheduled membership filter rebuild | Reads every key on every run | Rebuild on operator action or detected failure only |
|
|
710
|
+
| Restarting an external listing at page one | Failing items at the front starve the rest | A continuation job carrying the provider's token |
|
|
341
711
|
| `OFFSET` pagination | Every page rescans and discards the rows before it | Keyset on the indexed key |
|
|
342
712
|
| `UPDATE` or `DELETE` with a predicate but no `LIMIT` | One statement locks and rewrites an unbounded row set in one transaction | A CTE that selects a capped, `SKIP LOCKED` id set |
|
package/skills/manifest.json
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
{
|
|
25
25
|
"name": "bounded-iteration",
|
|
26
26
|
"plugin": "vouchington-database",
|
|
27
|
-
"pluginVersion": "0.
|
|
27
|
+
"pluginVersion": "0.5.0",
|
|
28
28
|
"path": "bounded-iteration/SKILL.md"
|
|
29
29
|
},
|
|
30
30
|
{
|
|
@@ -109,19 +109,19 @@
|
|
|
109
109
|
{
|
|
110
110
|
"name": "postgres-node-performance-tuning",
|
|
111
111
|
"plugin": "vouchington-database",
|
|
112
|
-
"pluginVersion": "0.
|
|
112
|
+
"pluginVersion": "0.5.0",
|
|
113
113
|
"path": "postgres-node-performance-tuning/SKILL.md"
|
|
114
114
|
},
|
|
115
115
|
{
|
|
116
116
|
"name": "postgres-partitioning-uuid-v7",
|
|
117
117
|
"plugin": "vouchington-database",
|
|
118
|
-
"pluginVersion": "0.
|
|
118
|
+
"pluginVersion": "0.5.0",
|
|
119
119
|
"path": "postgres-partitioning-uuid-v7/SKILL.md"
|
|
120
120
|
},
|
|
121
121
|
{
|
|
122
122
|
"name": "postgres-schema-design",
|
|
123
123
|
"plugin": "vouchington-database",
|
|
124
|
-
"pluginVersion": "0.
|
|
124
|
+
"pluginVersion": "0.5.0",
|
|
125
125
|
"path": "postgres-schema-design/SKILL.md"
|
|
126
126
|
},
|
|
127
127
|
{
|
|
@@ -26,10 +26,13 @@ Use keyset pagination, never `OFFSET`. An optimizer fence is a `MATERIALIZED` CT
|
|
|
26
26
|
cross-partition query. Choose the partition key from dominant access patterns; a hot reader
|
|
27
27
|
of `RANGE (id)` needs an id or time bound.
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
For a `RANGE (id)` child read by parent id, a shared lower-bound helper equivalent to
|
|
30
|
+
`id >= min_uuidv7(uuid_extract_timestamp(parent_id) - interval '1 hour')` prunes earlier
|
|
31
|
+
partitions; the overlap allows clock skew. It is safe only when every writer sets the parent at
|
|
32
|
+
insert to an existing parent, never re-points it to a newer parent, and nothing mints child ids
|
|
33
|
+
from a historical time; see
|
|
34
|
+
[parent lower bound](../../postgres-partitioning-uuid-v7/references/partition-lifecycle.md#parent-lower-bound).
|
|
35
|
+
Rate limits query an actor-keyed table rather than
|
|
33
36
|
recipient-keyed fan-out rows. Verify actual pruning with EXPLAIN.
|
|
34
37
|
|
|
35
38
|
Generic `jonathanong/no-mistakes` checks include `postgres-no-offset`; inspect available rules
|
|
@@ -17,6 +17,15 @@ Partition only after lifecycle, retention, and query predicates show that it hel
|
|
|
17
17
|
- Treat a UUIDv7 as time ordered. Where the meaning matters, keep an explicit business timestamp.
|
|
18
18
|
- Align the partition key, primary key, indexes, constraints, and query predicates so partition
|
|
19
19
|
pruning is observable.
|
|
20
|
+
- Add a parent lower bound to a read of a `RANGE (id)` child table by its parent's UUIDv7 id
|
|
21
|
+
(`id >= min_uuidv7(parent time - skew)`) only when both hold: every writer sets the parent column
|
|
22
|
+
at insert, to a parent that already exists, and never re-points it to a newer parent; and nothing
|
|
23
|
+
mints child ids from a historical time (imports, backfills, seeds). Counterexample: articles
|
|
24
|
+
clustered into a story created later are older than their parent, and the bound silently drops
|
|
25
|
+
them. See [parent lower bound](references/partition-lifecycle.md#parent-lower-bound).
|
|
26
|
+
- A statement that must read every partition declares that, with a reason, on the statement. Do not
|
|
27
|
+
use a table-wide exemption: it also silences every future query on that table. See
|
|
28
|
+
[cross-partition reads](references/partition-lifecycle.md#declaring-cross-partition-reads).
|
|
20
29
|
- Plan creation, retention, migration, and verification as one deployable lifecycle. Include
|
|
21
30
|
rollback and independent-reader compatibility.
|
|
22
31
|
- Read [partition lifecycle](references/partition-lifecycle.md) before a schema or retention
|
|
@@ -12,3 +12,39 @@ Every primary or unique constraint on a partitioned table must include its parti
|
|
|
12
12
|
pruning with predicates on that key, including joins whose other side needs an equivalent range
|
|
13
13
|
condition. UUIDv7 values are time ordered, not a promise that independently generated identifiers
|
|
14
14
|
have a strict ordering relationship.
|
|
15
|
+
|
|
16
|
+
## Parent lower bound
|
|
17
|
+
|
|
18
|
+
A child table partitioned `RANGE (id)` prunes when a reader bounds the child id. A reader that
|
|
19
|
+
starts from the parent can derive that bound from the parent's UUIDv7 id, assuming a child is never
|
|
20
|
+
older than its parent:
|
|
21
|
+
|
|
22
|
+
```sql
|
|
23
|
+
SELECT item.id, item.body
|
|
24
|
+
FROM order_items AS item
|
|
25
|
+
WHERE item.order_id = $1
|
|
26
|
+
AND item.id >= min_uuidv7($2::timestamptz - $3::interval);
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`$2` is the parent's creation time and `$3` the clock skew allowance. The bound is safe only when
|
|
30
|
+
both hold:
|
|
31
|
+
|
|
32
|
+
- Every writer sets `order_id` at insert, to an order that already exists, and never re-points the
|
|
33
|
+
row to a newer order.
|
|
34
|
+
- Nothing mints child ids from a historical time: imports, backfills, and seeds.
|
|
35
|
+
|
|
36
|
+
Counterexample: `articles` clustered into a `stories` row created later. Each article id is older
|
|
37
|
+
than its story id, so `article.id >= min_uuidv7(story time - skew)` silently drops the older
|
|
38
|
+
articles. Bound such a read by the child's own window instead.
|
|
39
|
+
|
|
40
|
+
## Declaring cross-partition reads
|
|
41
|
+
|
|
42
|
+
A statement that must read every partition says so, with a reason, on the statement itself:
|
|
43
|
+
|
|
44
|
+
```sql
|
|
45
|
+
-- cross-partition: nightly integrity check compares order_items totals with orders
|
|
46
|
+
SELECT order_id, sum(quantity) FROM order_items GROUP BY order_id;
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Do not exempt the whole table. A table-wide exemption also silences the check for every future
|
|
50
|
+
query on that table, including one that forgot its partition key predicate.
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { AlterTableStmt, DoStmt } from '@libpg-query/parser';
|
|
2
|
-
import { type SqlMigrationConstraintMetadata } from './constraint-shared.mts';
|
|
3
|
-
/**
|
|
4
|
-
* Applies AT_AddConstraint / AT_ValidateConstraint commands from one ALTER TABLE to `result`.
|
|
5
|
-
*
|
|
6
|
-
* `trackConstraintValidation` gates NOT VALID/VALIDATE pairing bookkeeping. Callers that
|
|
7
|
-
* parse ALTER TABLE text out of a DO body should pass false so only `foreignKeys` is filled.
|
|
8
|
-
*/
|
|
9
|
-
export declare function processAlterTableStmt(statement: AlterTableStmt, baseOffset: number, result: SqlMigrationConstraintMetadata, trackConstraintValidation?: boolean): void;
|
|
10
|
-
/** Idempotent migrations wrap `ALTER TABLE ... ADD CONSTRAINT` in `DO $$ ... $$` catalog checks. */
|
|
11
|
-
export declare function collectDoStmtConstraints(content: string, statement: DoStmt, stmtLocation: number, result: SqlMigrationConstraintMetadata): void;
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
import { fkAttrNames, foreignKeyFromConstraint, } from "./constraint-shared.mjs";
|
|
2
|
-
import { parseSql } from "./parser.mjs";
|
|
3
|
-
/**
|
|
4
|
-
* Applies AT_AddConstraint / AT_ValidateConstraint commands from one ALTER TABLE to `result`.
|
|
5
|
-
*
|
|
6
|
-
* `trackConstraintValidation` gates NOT VALID/VALIDATE pairing bookkeeping. Callers that
|
|
7
|
-
* parse ALTER TABLE text out of a DO body should pass false so only `foreignKeys` is filled.
|
|
8
|
-
*/
|
|
9
|
-
export function processAlterTableStmt(statement, baseOffset, result, trackConstraintValidation = true) {
|
|
10
|
-
const tableName = statement.relation?.relname;
|
|
11
|
-
/* v8 ignore next */
|
|
12
|
-
if (!tableName)
|
|
13
|
-
return;
|
|
14
|
-
for (const rawCommand of statement.cmds ?? []) {
|
|
15
|
-
/* v8 ignore next */
|
|
16
|
-
if (!rawCommand || !('AlterTableCmd' in rawCommand))
|
|
17
|
-
continue;
|
|
18
|
-
const command = rawCommand.AlterTableCmd;
|
|
19
|
-
if (command.subtype === 'AT_ValidateConstraint' && command.name) {
|
|
20
|
-
if (trackConstraintValidation)
|
|
21
|
-
result.validatedConstraints.add(`${tableName}.${command.name}`);
|
|
22
|
-
continue;
|
|
23
|
-
}
|
|
24
|
-
/* v8 ignore next */
|
|
25
|
-
if (command.subtype !== 'AT_AddConstraint')
|
|
26
|
-
continue;
|
|
27
|
-
const definition = command.def;
|
|
28
|
-
/* v8 ignore next */
|
|
29
|
-
if (!definition || !('Constraint' in definition))
|
|
30
|
-
continue;
|
|
31
|
-
const constraint = definition.Constraint;
|
|
32
|
-
/* v8 ignore next */
|
|
33
|
-
if (!constraint)
|
|
34
|
-
continue;
|
|
35
|
-
if (constraint.contype === 'CONSTR_FOREIGN') {
|
|
36
|
-
// oxlint-disable-next-line no-mistakes/ts-no-const-aliases -- isolate the generated AST type escape at the foreign-key adapter boundary
|
|
37
|
-
const constraintRecord = constraint;
|
|
38
|
-
result.foreignKeys.push(foreignKeyFromConstraint(constraintRecord, tableName, fkAttrNames(constraintRecord), baseOffset));
|
|
39
|
-
}
|
|
40
|
-
if (!trackConstraintValidation)
|
|
41
|
-
continue;
|
|
42
|
-
/* v8 ignore next */
|
|
43
|
-
if (constraint.contype !== 'CONSTR_CHECK' && constraint.contype !== 'CONSTR_FOREIGN')
|
|
44
|
-
continue;
|
|
45
|
-
result.addedConstraints.push({
|
|
46
|
-
/* v8 ignore next 3 */
|
|
47
|
-
constraintType: constraint.contype ?? null,
|
|
48
|
-
location: baseOffset + (constraint.location ?? 0),
|
|
49
|
-
name: constraint.conname ?? null,
|
|
50
|
-
tableName,
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
const ALTER_TABLE_STATEMENT_RE = /ALTER\s+TABLE\b[^;]*;/gi;
|
|
55
|
-
/** Idempotent migrations wrap `ALTER TABLE ... ADD CONSTRAINT` in `DO $$ ... $$` catalog checks. */
|
|
56
|
-
/* v8 ignore start -- DO-body regex/parse fallbacks */
|
|
57
|
-
export function collectDoStmtConstraints(content, statement, stmtLocation, result) {
|
|
58
|
-
for (const arg of statement.args ?? []) {
|
|
59
|
-
const defElem = arg && 'DefElem' in arg ? arg.DefElem : undefined;
|
|
60
|
-
const stringArg = defElem?.arg && 'String' in defElem.arg ? defElem.arg.String : undefined;
|
|
61
|
-
const body = typeof stringArg?.sval === 'string' ? stringArg.sval : '';
|
|
62
|
-
for (const match of body.matchAll(ALTER_TABLE_STATEMENT_RE)) {
|
|
63
|
-
const baseOffset = content.indexOf(match[0], stmtLocation);
|
|
64
|
-
/* v8 ignore next */
|
|
65
|
-
if (baseOffset === -1)
|
|
66
|
-
continue;
|
|
67
|
-
let nested;
|
|
68
|
-
try {
|
|
69
|
-
nested = parseSql(match[0]);
|
|
70
|
-
}
|
|
71
|
-
catch {
|
|
72
|
-
/* v8 ignore next */
|
|
73
|
-
continue;
|
|
74
|
-
}
|
|
75
|
-
for (const nestedStmt of nested.stmts ?? []) {
|
|
76
|
-
const node = nestedStmt.stmt;
|
|
77
|
-
if (node && 'AlterTableStmt' in node) {
|
|
78
|
-
processAlterTableStmt(node.AlterTableStmt, baseOffset, result, false);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
/* v8 ignore stop */
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export type SqlMigrationConstraintMetadata = {
|
|
2
|
-
addedConstraints: Array<{
|
|
3
|
-
constraintType: string | null;
|
|
4
|
-
location: number;
|
|
5
|
-
name: string | null;
|
|
6
|
-
tableName: string;
|
|
7
|
-
}>;
|
|
8
|
-
foreignKeys: Array<{
|
|
9
|
-
columnNames: string[];
|
|
10
|
-
deleteAction: string | null;
|
|
11
|
-
location: number;
|
|
12
|
-
referencedTableName: string | null;
|
|
13
|
-
tableName: string | null;
|
|
14
|
-
}>;
|
|
15
|
-
validatedConstraints: Set<string>;
|
|
16
|
-
};
|
|
17
|
-
export type ForeignKey = SqlMigrationConstraintMetadata['foreignKeys'][number];
|
|
18
|
-
export declare function fkAttrNames(constraint: Record<string, unknown>): string[];
|
|
19
|
-
export declare function foreignKeyFromConstraint(constraint: Record<string, unknown>, tableName: string | null, columnNames: string[], baseOffset?: number): ForeignKey;
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { isRecord } from "./unknown-record.mjs";
|
|
2
|
-
export function fkAttrNames(constraint) {
|
|
3
|
-
const attrs = constraint.fk_attrs;
|
|
4
|
-
if (!Array.isArray(attrs))
|
|
5
|
-
return [];
|
|
6
|
-
return attrs.flatMap((attr) => {
|
|
7
|
-
const sval = isRecord(attr) && isRecord(attr.String) ? attr.String.sval : undefined;
|
|
8
|
-
return typeof sval === 'string' ? [sval] : [];
|
|
9
|
-
});
|
|
10
|
-
}
|
|
11
|
-
export function foreignKeyFromConstraint(constraint, tableName, columnNames, baseOffset = 0) {
|
|
12
|
-
const pktable = constraint.pktable;
|
|
13
|
-
return {
|
|
14
|
-
columnNames,
|
|
15
|
-
deleteAction: typeof constraint.fk_del_action === 'string' ? constraint.fk_del_action : null,
|
|
16
|
-
location: baseOffset + (typeof constraint.location === 'number' ? constraint.location : 0),
|
|
17
|
-
referencedTableName: isRecord(pktable) && typeof pktable.relname === 'string' ? pktable.relname : null,
|
|
18
|
-
tableName,
|
|
19
|
-
};
|
|
20
|
-
}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { collectDoStmtConstraints, processAlterTableStmt } from "./constraint-do-block.mjs";
|
|
2
|
-
import { fkAttrNames, foreignKeyFromConstraint, } from "./constraint-shared.mjs";
|
|
3
|
-
import { parseSql } from "./parser.mjs";
|
|
4
|
-
import { isRecord } from "./unknown-record.mjs";
|
|
5
|
-
/** Inline `col UUID REFERENCES ...` and table-level `FOREIGN KEY (...) REFERENCES ...` forms. */
|
|
6
|
-
/* v8 ignore start -- defensive parse-tree walks */
|
|
7
|
-
function collectCreateStmtForeignKeys(statement, foreignKeys) {
|
|
8
|
-
const relation = statement.relation;
|
|
9
|
-
const tableName = isRecord(relation) && typeof relation.relname === 'string' ? relation.relname : null;
|
|
10
|
-
const tableElts = statement.tableElts;
|
|
11
|
-
if (!Array.isArray(tableElts))
|
|
12
|
-
return;
|
|
13
|
-
for (const element of tableElts) {
|
|
14
|
-
if (!isRecord(element))
|
|
15
|
-
continue;
|
|
16
|
-
if (isRecord(element.ColumnDef)) {
|
|
17
|
-
const column = element.ColumnDef;
|
|
18
|
-
const colname = typeof column.colname === 'string' ? column.colname : null;
|
|
19
|
-
for (const constraintNode of column.constraints ?? []) {
|
|
20
|
-
/* v8 ignore next 3 */
|
|
21
|
-
if (!isRecord(constraintNode) || !isRecord(constraintNode.Constraint))
|
|
22
|
-
continue;
|
|
23
|
-
const constraint = constraintNode.Constraint;
|
|
24
|
-
if (constraint.contype !== 'CONSTR_FOREIGN')
|
|
25
|
-
continue;
|
|
26
|
-
foreignKeys.push(foreignKeyFromConstraint(constraint, tableName, colname ? [colname] : []));
|
|
27
|
-
}
|
|
28
|
-
continue;
|
|
29
|
-
}
|
|
30
|
-
if (isRecord(element.Constraint) && element.Constraint.contype === 'CONSTR_FOREIGN') {
|
|
31
|
-
foreignKeys.push(foreignKeyFromConstraint(element.Constraint, tableName, fkAttrNames(element.Constraint)));
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
/* v8 ignore stop */
|
|
36
|
-
export function extractMigrationConstraintMetadata(content) {
|
|
37
|
-
const result = {
|
|
38
|
-
addedConstraints: [],
|
|
39
|
-
foreignKeys: [],
|
|
40
|
-
validatedConstraints: new Set(),
|
|
41
|
-
};
|
|
42
|
-
const parseResult = parseSql(content);
|
|
43
|
-
for (const rawStmt of parseResult.stmts ?? []) {
|
|
44
|
-
const node = rawStmt.stmt;
|
|
45
|
-
/* v8 ignore next */
|
|
46
|
-
if (!node)
|
|
47
|
-
continue;
|
|
48
|
-
const stmtLocation = rawStmt.stmt_location ?? 0;
|
|
49
|
-
if ('CreateStmt' in node) {
|
|
50
|
-
collectCreateStmtForeignKeys(node.CreateStmt, result.foreignKeys);
|
|
51
|
-
continue;
|
|
52
|
-
}
|
|
53
|
-
if ('DoStmt' in node) {
|
|
54
|
-
collectDoStmtConstraints(content, node.DoStmt, stmtLocation, result);
|
|
55
|
-
continue;
|
|
56
|
-
}
|
|
57
|
-
/* v8 ignore next */
|
|
58
|
-
if (!('AlterTableStmt' in node))
|
|
59
|
-
continue;
|
|
60
|
-
processAlterTableStmt(node.AlterTableStmt, 0, result);
|
|
61
|
-
}
|
|
62
|
-
return result;
|
|
63
|
-
}
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
import type { SqlCreateIndexMetadata } from './index-metadata.mts';
|
|
2
|
-
/** UNIQUE/PRIMARY KEY create implicit unique indexes that never appear as CREATE INDEX. */
|
|
3
|
-
export declare function collectCreateStmtImplicitIndexes(statement: Record<string, unknown>, location: number, indexes: SqlCreateIndexMetadata[]): void;
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
import { isRecord } from "./unknown-record.mjs";
|
|
2
|
-
function implicitIndexParam(name) {
|
|
3
|
-
return { name, opclass: null, ordering: null, nullsOrdering: null };
|
|
4
|
-
}
|
|
5
|
-
function pushImplicitIndex(indexes, relname, location, columnNames) {
|
|
6
|
-
indexes.push({
|
|
7
|
-
relname,
|
|
8
|
-
idxname: null,
|
|
9
|
-
unique: true,
|
|
10
|
-
indexParams: columnNames,
|
|
11
|
-
indexParamDetails: columnNames.map(implicitIndexParam),
|
|
12
|
-
includeParams: [],
|
|
13
|
-
whereClause: null,
|
|
14
|
-
whereClauseKey: null,
|
|
15
|
-
accessMethod: 'btree',
|
|
16
|
-
location,
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
function constraintKeyNames(constraint) {
|
|
20
|
-
const keys = constraint.keys;
|
|
21
|
-
if (!Array.isArray(keys))
|
|
22
|
-
return [];
|
|
23
|
-
return keys.flatMap((key) => {
|
|
24
|
-
const sval = isRecord(key) && isRecord(key.String) ? key.String.sval : undefined;
|
|
25
|
-
/* v8 ignore next */
|
|
26
|
-
return typeof sval === 'string' ? [sval] : [];
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
/** UNIQUE/PRIMARY KEY create implicit unique indexes that never appear as CREATE INDEX. */
|
|
30
|
-
export function collectCreateStmtImplicitIndexes(statement, location, indexes) {
|
|
31
|
-
const relation = statement.relation;
|
|
32
|
-
const relname = isRecord(relation) && typeof relation.relname === 'string' ? relation.relname : null;
|
|
33
|
-
const tableElts = statement.tableElts;
|
|
34
|
-
if (!relname || !Array.isArray(tableElts))
|
|
35
|
-
return;
|
|
36
|
-
for (const element of tableElts) {
|
|
37
|
-
if (!isRecord(element))
|
|
38
|
-
continue;
|
|
39
|
-
if (isRecord(element.ColumnDef)) {
|
|
40
|
-
const column = element.ColumnDef;
|
|
41
|
-
const colname = typeof column.colname === 'string' ? column.colname : null;
|
|
42
|
-
if (!colname)
|
|
43
|
-
continue;
|
|
44
|
-
for (const constraintNode of column.constraints ?? []) {
|
|
45
|
-
/* v8 ignore next */
|
|
46
|
-
if (!isRecord(constraintNode) || !isRecord(constraintNode.Constraint))
|
|
47
|
-
continue;
|
|
48
|
-
const contype = constraintNode.Constraint.contype;
|
|
49
|
-
if (contype !== 'CONSTR_UNIQUE' && contype !== 'CONSTR_PRIMARY')
|
|
50
|
-
continue;
|
|
51
|
-
pushImplicitIndex(indexes, relname, location, [colname]);
|
|
52
|
-
}
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
/* v8 ignore next */
|
|
56
|
-
if (!isRecord(element.Constraint))
|
|
57
|
-
continue;
|
|
58
|
-
const contype = element.Constraint.contype;
|
|
59
|
-
if (contype !== 'CONSTR_UNIQUE' && contype !== 'CONSTR_PRIMARY')
|
|
60
|
-
continue;
|
|
61
|
-
const columnNames = constraintKeyNames(element.Constraint);
|
|
62
|
-
if (columnNames.length === 0)
|
|
63
|
-
continue;
|
|
64
|
-
pushImplicitIndex(indexes, relname, location, columnNames);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export type SqlIndexParam = {
|
|
2
|
-
name: string | null;
|
|
3
|
-
opclass: string | null;
|
|
4
|
-
ordering: string | null;
|
|
5
|
-
nullsOrdering: string | null;
|
|
6
|
-
};
|
|
7
|
-
export type SqlCreateIndexMetadata = {
|
|
8
|
-
relname: string;
|
|
9
|
-
idxname: string | null;
|
|
10
|
-
unique: boolean;
|
|
11
|
-
indexParams: Array<string | null>;
|
|
12
|
-
indexParamDetails: SqlIndexParam[];
|
|
13
|
-
includeParams: string[];
|
|
14
|
-
whereClause: unknown;
|
|
15
|
-
whereClauseKey: string | null;
|
|
16
|
-
accessMethod: string;
|
|
17
|
-
location: number;
|
|
18
|
-
};
|
|
19
|
-
export declare function extractCreateIndexMetadata(content: string): SqlCreateIndexMetadata[];
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
import { collectCreateStmtImplicitIndexes } from "./implicit-indexes.mjs";
|
|
2
|
-
import { parseSql } from "./parser.mjs";
|
|
3
|
-
import { isRecord } from "./unknown-record.mjs";
|
|
4
|
-
function indexElemName(param) {
|
|
5
|
-
if (!isRecord(param) || !isRecord(param.IndexElem))
|
|
6
|
-
return null;
|
|
7
|
-
const name = param.IndexElem.name;
|
|
8
|
-
return typeof name === 'string' ? name : null;
|
|
9
|
-
}
|
|
10
|
-
function indexElemOpclass(param) {
|
|
11
|
-
if (!isRecord(param) || !isRecord(param.IndexElem))
|
|
12
|
-
return null;
|
|
13
|
-
const opclass = param.IndexElem.opclass;
|
|
14
|
-
if (!Array.isArray(opclass))
|
|
15
|
-
return null;
|
|
16
|
-
const names = opclass.flatMap((node) => {
|
|
17
|
-
const sval = isRecord(node) && isRecord(node.String) ? node.String.sval : undefined;
|
|
18
|
-
/* v8 ignore next */
|
|
19
|
-
return typeof sval === 'string' ? [sval] : [];
|
|
20
|
-
});
|
|
21
|
-
return names.length > 0 ? names.join('.') : null;
|
|
22
|
-
}
|
|
23
|
-
/** Sort direction and NULLS placement together — PostgreSQL resolves both from the same `IndexElem`. */
|
|
24
|
-
function indexElemSortSpec(param) {
|
|
25
|
-
if (!isRecord(param) || !isRecord(param.IndexElem))
|
|
26
|
-
return { ordering: null, nullsOrdering: null };
|
|
27
|
-
const { ordering, nulls_ordering: nullsOrdering } = param.IndexElem;
|
|
28
|
-
return {
|
|
29
|
-
ordering: typeof ordering === 'string' ? ordering : null,
|
|
30
|
-
nullsOrdering: typeof nullsOrdering === 'string' ? nullsOrdering : null,
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
function indexParamDetail(param) {
|
|
34
|
-
return {
|
|
35
|
-
name: indexElemName(param),
|
|
36
|
-
opclass: indexElemOpclass(param),
|
|
37
|
-
...indexElemSortSpec(param),
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Deep-clones a parser AST node with `location` fields stripped, so two
|
|
42
|
-
* predicates that are byte-identical apart from source position compare
|
|
43
|
-
* equal via JSON.stringify.
|
|
44
|
-
*/
|
|
45
|
-
function stripLocations(value) {
|
|
46
|
-
if (Array.isArray(value))
|
|
47
|
-
return value.map(stripLocations);
|
|
48
|
-
if (!isRecord(value))
|
|
49
|
-
return value;
|
|
50
|
-
const result = {};
|
|
51
|
-
for (const [key, child] of Object.entries(value)) {
|
|
52
|
-
if (key === 'location')
|
|
53
|
-
continue;
|
|
54
|
-
result[key] = stripLocations(child);
|
|
55
|
-
}
|
|
56
|
-
return result;
|
|
57
|
-
}
|
|
58
|
-
function whereClauseKey(whereClause) {
|
|
59
|
-
if (whereClause === undefined || whereClause === null)
|
|
60
|
-
return null;
|
|
61
|
-
return JSON.stringify(stripLocations(whereClause));
|
|
62
|
-
}
|
|
63
|
-
export function extractCreateIndexMetadata(content) {
|
|
64
|
-
const indexes = [];
|
|
65
|
-
const parseResult = parseSql(content);
|
|
66
|
-
for (const rawStmt of parseResult.stmts ?? []) {
|
|
67
|
-
const node = rawStmt.stmt;
|
|
68
|
-
/* v8 ignore next */
|
|
69
|
-
if (!node)
|
|
70
|
-
continue;
|
|
71
|
-
if ('CreateStmt' in node) {
|
|
72
|
-
collectCreateStmtImplicitIndexes(node.CreateStmt, rawStmt.stmt_location ?? 0, indexes);
|
|
73
|
-
continue;
|
|
74
|
-
}
|
|
75
|
-
/* v8 ignore next */
|
|
76
|
-
if (!('IndexStmt' in node))
|
|
77
|
-
continue;
|
|
78
|
-
const indexStmt = node.IndexStmt;
|
|
79
|
-
const relname = indexStmt.relation?.relname;
|
|
80
|
-
/* v8 ignore next */
|
|
81
|
-
if (!relname)
|
|
82
|
-
continue;
|
|
83
|
-
const indexParams = indexStmt.indexParams ?? [];
|
|
84
|
-
indexes.push({
|
|
85
|
-
relname,
|
|
86
|
-
idxname: indexStmt.idxname ?? null,
|
|
87
|
-
unique: indexStmt.unique === true,
|
|
88
|
-
indexParams: indexParams.map(indexElemName),
|
|
89
|
-
indexParamDetails: indexParams.map(indexParamDetail),
|
|
90
|
-
includeParams: (indexStmt.indexIncludingParams ?? [])
|
|
91
|
-
.map(indexElemName)
|
|
92
|
-
.filter((name) => name !== null),
|
|
93
|
-
whereClause: indexStmt.whereClause ?? null,
|
|
94
|
-
whereClauseKey: whereClauseKey(indexStmt.whereClause),
|
|
95
|
-
accessMethod: indexStmt.accessMethod ?? 'btree',
|
|
96
|
-
location: rawStmt.stmt_location ?? 0,
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
return indexes;
|
|
100
|
-
}
|