turbine-orm 0.57.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/cli/index.js +30 -1
- package/dist/cjs/plan-divergence.d.ts +15 -0
- package/dist/cjs/plan-flip-probe.d.ts +165 -0
- package/dist/cjs/plan-flip-probe.js +304 -0
- package/dist/cli/index.js +30 -1
- package/dist/plan-divergence.d.ts +15 -0
- package/dist/plan-flip-probe.d.ts +165 -0
- package/dist/plan-flip-probe.js +262 -0
- package/package.json +1 -1
package/dist/cjs/cli/index.js
CHANGED
|
@@ -83,6 +83,7 @@ const index_advisor_js_1 = require("../index-advisor.js");
|
|
|
83
83
|
const index_stats_js_1 = require("../index-stats.js");
|
|
84
84
|
const introspect_js_1 = require("../introspect.js");
|
|
85
85
|
const plan_divergence_js_1 = require("../plan-divergence.js");
|
|
86
|
+
const plan_flip_probe_js_1 = require("../plan-flip-probe.js");
|
|
86
87
|
const schema_js_1 = require("../schema.js");
|
|
87
88
|
const schema_sql_js_1 = require("../schema-sql.js");
|
|
88
89
|
const config_js_1 = require("./config.js");
|
|
@@ -2383,9 +2384,18 @@ async function cmdDoctor(args, config) {
|
|
|
2383
2384
|
// trustworthy stats_reset age because they normalize write counters by it;
|
|
2384
2385
|
// this check reads no counter, only pg_stats, whose freshness is ANALYZE. A
|
|
2385
2386
|
// cluster with a NULL stats_reset (the default) must still get the check.
|
|
2386
|
-
const
|
|
2387
|
+
const scored = divergenceOn && snapshot.available
|
|
2387
2388
|
? (0, plan_divergence_js_1.findPlanDivergence)(schema, snapshot)
|
|
2388
2389
|
: { findings: [], notices: [], candidatesConsidered: 0, consideredIndexed: 0, consideredUnindexed: 0 };
|
|
2390
|
+
// Statistics can say how bad a flip WOULD be; only the planner can say whether
|
|
2391
|
+
// it is reachable. The `unindexed-filter` branch shipped in 0.57 without that
|
|
2392
|
+
// question answered and was right 6 times in 13 on a real schema, so every one
|
|
2393
|
+
// of its findings is now put to a plan-only EXPLAIN. Nothing is executed, and a
|
|
2394
|
+
// probe that fails keeps its finding rather than dropping it.
|
|
2395
|
+
const flipProbe = divergenceOn && scored.findings.some(plan_flip_probe_js_1.needsFlipProbe)
|
|
2396
|
+
? await (0, plan_flip_probe_js_1.probePlanFlips)({ connectionString: url, schema: config.schema, findings: scored.findings })
|
|
2397
|
+
: (0, plan_flip_probe_js_1.emptyFlipProbeResult)();
|
|
2398
|
+
const divergence = (0, plan_flip_probe_js_1.applyFlipVerdicts)(scored, flipProbe);
|
|
2389
2399
|
if (jsonMode) {
|
|
2390
2400
|
spinner?.stop();
|
|
2391
2401
|
console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, heat, subtract, divergence, args }), null, 2));
|
|
@@ -2479,6 +2489,12 @@ function buildDoctorJson(ctx) {
|
|
|
2479
2489
|
considered: ctx.divergence.candidatesConsidered,
|
|
2480
2490
|
indexed: ctx.divergence.consideredIndexed,
|
|
2481
2491
|
unindexed: ctx.divergence.consideredUnindexed,
|
|
2492
|
+
// Whether the unindexed findings above were put to the planner, and how many
|
|
2493
|
+
// it refuted. `flipProbed: false` means they are statistics-only and carry
|
|
2494
|
+
// 0.57's precision, so a consumer can tell a verified list from an unverified
|
|
2495
|
+
// one instead of inferring it from the count.
|
|
2496
|
+
flipProbed: ctx.divergence.flipProbed === true,
|
|
2497
|
+
flipRefuted: ctx.divergence.flipRefuted ?? 0,
|
|
2482
2498
|
};
|
|
2483
2499
|
return out;
|
|
2484
2500
|
}
|
|
@@ -2801,6 +2817,19 @@ function renderDivergenceGates(divergence) {
|
|
|
2801
2817
|
console.log(` ${(0, ui_js_1.dim)(`${divergence.candidatesConsidered} column(s) were scored (${divergence.consideredIndexed} indexed, ${divergence.consideredUnindexed} unindexed). That population is relation-probe`)}`);
|
|
2802
2818
|
console.log(` ${(0, ui_js_1.dim)('and leading-index columns only: a filter column that is neither is not covered. Skip this')}`);
|
|
2803
2819
|
console.log(` ${(0, ui_js_1.dim)('section with --no-plan-divergence.')}`);
|
|
2820
|
+
// Say whether the unindexed findings were verified, and what verification
|
|
2821
|
+
// removed. Statistics can only say how bad a flip would be; a plan-only EXPLAIN
|
|
2822
|
+
// says whether the planner can reach it at all.
|
|
2823
|
+
if (divergence.flipProbed === true) {
|
|
2824
|
+
const refuted = divergence.flipRefuted ?? 0;
|
|
2825
|
+
const wereRefuted = refuted === 1 ? '1 was refuted' : `${refuted} were refuted`;
|
|
2826
|
+
console.log(` ${(0, ui_js_1.dim)(`Every unindexed finding was put to the planner (EXPLAIN, nothing executed); ${wereRefuted}`)}`);
|
|
2827
|
+
console.log(` ${(0, ui_js_1.dim)('because the generic plan keeps the same sequential scan, so no flip is reachable.')}`);
|
|
2828
|
+
}
|
|
2829
|
+
else if (divergence.consideredUnindexed > 0) {
|
|
2830
|
+
console.log(` ${(0, ui_js_1.dim)('Unindexed findings are UNVERIFIED here: the planner probe did not run, so some may name a')}`);
|
|
2831
|
+
console.log(` ${(0, ui_js_1.dim)('divergence the planner would never choose.')}`);
|
|
2832
|
+
}
|
|
2804
2833
|
}
|
|
2805
2834
|
/** Round to a whole number and group it, for the divergence report's estimates. */
|
|
2806
2835
|
function divInt(n) {
|
|
@@ -424,6 +424,21 @@ export interface PlanDivergenceReport {
|
|
|
424
424
|
*/
|
|
425
425
|
consideredIndexed: number;
|
|
426
426
|
consideredUnindexed: number;
|
|
427
|
+
/**
|
|
428
|
+
* True when every `unindexed-filter` finding was put to the planner via
|
|
429
|
+
* `plan-flip-probe.ts` rather than reported on statistics alone.
|
|
430
|
+
*
|
|
431
|
+
* Absent or false means the findings below are UNVERIFIED: the probe was
|
|
432
|
+
* skipped, the engine is not Postgres, or the connection refused it.
|
|
433
|
+
*/
|
|
434
|
+
flipProbed?: boolean;
|
|
435
|
+
/**
|
|
436
|
+
* How many candidate findings the planner refuted, i.e. columns where the
|
|
437
|
+
* generic plan keeps the same sequential scan the good plan uses so there is
|
|
438
|
+
* no divergence to reach. Reported because a check that silently discards two
|
|
439
|
+
* thirds of what it found should say so.
|
|
440
|
+
*/
|
|
441
|
+
flipRefuted?: number;
|
|
427
442
|
}
|
|
428
443
|
/** A (table, column) pair whose distribution statistics the collector should read. */
|
|
429
444
|
export interface DivergenceCandidate {
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plan-flip probe: does the generic plan actually differ from the good one?
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* `plan-divergence.ts` scores a column from statistics alone and answers "IF the
|
|
7
|
+
* cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
|
|
8
|
+
* without an answer to the prior question, "CAN it flip at all", and that turned
|
|
9
|
+
* out to be the majority case: on a real 118-model schema the branch produced 39
|
|
10
|
+
* findings of which a measured sample was right 6 times in 13.
|
|
11
|
+
*
|
|
12
|
+
* Every false positive had one signature: **the generic plan kept the same
|
|
13
|
+
* sequential scan the custom plan chose.** There was no flip to be had, so the
|
|
14
|
+
* amplification the finding printed described a plan the planner would never
|
|
15
|
+
* pick.
|
|
16
|
+
*
|
|
17
|
+
* ## Why this is a probe and not another rule
|
|
18
|
+
*
|
|
19
|
+
* The obvious gate is arithmetic: require the generic row estimate
|
|
20
|
+
* (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
|
|
21
|
+
* Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
|
|
22
|
+
* estimate at or below the limit earns no discount at all.
|
|
23
|
+
*
|
|
24
|
+
* That rule is wrong, and it was measured wrong before it was written down here.
|
|
25
|
+
* The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
|
|
26
|
+
* which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
|
|
27
|
+
* that, and the plan is then chosen by comparing that fraction of the full index
|
|
28
|
+
* scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
|
|
29
|
+
* between estimates 3 and 4, not at the limit of 20:
|
|
30
|
+
*
|
|
31
|
+
* ```txt
|
|
32
|
+
* generic estimate 2 3 4 10 20 500
|
|
33
|
+
* generic plan seq seq index index index index
|
|
34
|
+
* buffers 250 250 20,074 20,074 19,071 765
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
|
|
38
|
+
* would discard. The boundary is also not a fixed number: it is a cost
|
|
39
|
+
* comparison, so it moves with the table. Reproducing the same estimate on a
|
|
40
|
+
* 1976-page table and on an 89-page narrow table put the flip in a different
|
|
41
|
+
* place each time.
|
|
42
|
+
*
|
|
43
|
+
* So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
|
|
44
|
+
* WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
|
|
45
|
+
* planner the exact question the rule was trying to predict. This module runs it.
|
|
46
|
+
*
|
|
47
|
+
* ## What it asks, and why only half the pair
|
|
48
|
+
*
|
|
49
|
+
* It plans ONE statement, under `force_generic_plan` only:
|
|
50
|
+
*
|
|
51
|
+
* ```sql
|
|
52
|
+
* PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
|
|
53
|
+
* EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* The custom plan is not needed. The finding's whole claim is that a promoted
|
|
57
|
+
* generic plan abandons the seq scan for an ordered index walk, so if the generic
|
|
58
|
+
* plan IS a seq scan on the target table, the claim is refuted no matter what the
|
|
59
|
+
* custom plan does. Asking one question instead of two halves the work and
|
|
60
|
+
* removes the need for a representative rare value, which statistics do not
|
|
61
|
+
* carry.
|
|
62
|
+
*
|
|
63
|
+
* `NULL` is a safe argument precisely because the plan is generic: a generic plan
|
|
64
|
+
* is built without looking at the value, which is the property the whole check is
|
|
65
|
+
* about. The LIMIT is bound as `$2` rather than inlined because that is the shape
|
|
66
|
+
* Turbine emits, and an inlined limit takes a different code path in the planner.
|
|
67
|
+
*
|
|
68
|
+
* ## Failure is never a silent drop
|
|
69
|
+
*
|
|
70
|
+
* A probe that errors, times out, or returns an unparseable plan yields
|
|
71
|
+
* `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
|
|
72
|
+
* findings when the database is uncooperative would be worse than one that
|
|
73
|
+
* over-reports, because the failure would be invisible in exactly the
|
|
74
|
+
* environments (restricted roles, non-Postgres engines) where a human is least
|
|
75
|
+
* able to check.
|
|
76
|
+
*
|
|
77
|
+
* @module
|
|
78
|
+
*/
|
|
79
|
+
import type { PlanDivergenceFinding, PlanDivergenceReport } from './plan-divergence.js';
|
|
80
|
+
/**
|
|
81
|
+
* The planner's answer for one finding.
|
|
82
|
+
*
|
|
83
|
+
* - `'flip-reachable'`, the generic plan is NOT a plain seq scan of the target
|
|
84
|
+
* table, so the divergence the finding describes is one the planner can
|
|
85
|
+
* actually choose.
|
|
86
|
+
* - `'no-flip'`, the generic plan sequentially scans the target table, the same
|
|
87
|
+
* access the good plan uses. Nothing to diverge to.
|
|
88
|
+
* - `'unknown'`, the probe did not produce an answer. The finding is kept.
|
|
89
|
+
*/
|
|
90
|
+
export type FlipVerdict = 'flip-reachable' | 'no-flip' | 'unknown';
|
|
91
|
+
/** Outcome of a probe pass, keyed by {@link flipProbeKey}. */
|
|
92
|
+
export interface FlipProbeResult {
|
|
93
|
+
/** True when the probe pass ran at all (Postgres, connection succeeded). */
|
|
94
|
+
available: boolean;
|
|
95
|
+
verdicts: Record<string, FlipVerdict>;
|
|
96
|
+
notices: string[];
|
|
97
|
+
}
|
|
98
|
+
/** An empty result, which keeps every finding. Used when probing is off. */
|
|
99
|
+
export declare function emptyFlipProbeResult(): FlipProbeResult;
|
|
100
|
+
/**
|
|
101
|
+
* Map key for a (table, column) pair.
|
|
102
|
+
*
|
|
103
|
+
* `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
|
|
104
|
+
* literal NUL in a source file makes `grep` treat the whole file as binary, which
|
|
105
|
+
* is how four of them survived a release in `cli/index.ts`.
|
|
106
|
+
*/
|
|
107
|
+
export declare function flipProbeKey(table: string, column: string): string;
|
|
108
|
+
/**
|
|
109
|
+
* Which findings are worth probing.
|
|
110
|
+
*
|
|
111
|
+
* `unindexed-filter` only. The `sparse-value` branch already requires an index
|
|
112
|
+
* that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
|
|
113
|
+
* that later produced 6 of 13 here, so there is no measured precision problem to
|
|
114
|
+
* spend a round trip on.
|
|
115
|
+
*/
|
|
116
|
+
export declare function needsFlipProbe(finding: PlanDivergenceFinding): boolean;
|
|
117
|
+
/**
|
|
118
|
+
* The SQL for one probe. Pure, so the exact text is unit-testable without a
|
|
119
|
+
* database.
|
|
120
|
+
*
|
|
121
|
+
* Every identifier goes through {@link quoteIdent}; the only values in the
|
|
122
|
+
* statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
|
|
123
|
+
* caller as `tpf_<index>` and is never caller-controlled text.
|
|
124
|
+
*/
|
|
125
|
+
export declare function buildFlipProbeSql(finding: PlanDivergenceFinding, name: string, searchSchema?: string): {
|
|
126
|
+
prepare: string;
|
|
127
|
+
explain: string;
|
|
128
|
+
deallocate: string;
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
|
|
132
|
+
*
|
|
133
|
+
* Exported for unit tests: the plan shapes this has to classify are exactly the
|
|
134
|
+
* ones that are tedious to produce live.
|
|
135
|
+
*
|
|
136
|
+
* The rule is deliberately narrow. Only a `Seq Scan` ON THE TARGET TABLE refutes
|
|
137
|
+
* a finding. A seq scan of some other relation in a more complex plan says
|
|
138
|
+
* nothing about this column, and anything that is not a plain sequential scan of
|
|
139
|
+
* the target (index scan, bitmap heap scan, index-only scan) leaves the flip
|
|
140
|
+
* reachable.
|
|
141
|
+
*/
|
|
142
|
+
export declare function verdictFromPlanJson(payload: unknown, table: string): FlipVerdict;
|
|
143
|
+
export interface ProbePlanFlipsOptions {
|
|
144
|
+
connectionString: string;
|
|
145
|
+
schema?: string;
|
|
146
|
+
findings: PlanDivergenceFinding[];
|
|
147
|
+
statementTimeoutMs?: number;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Ask the planner, once per candidate finding, whether the flip is reachable.
|
|
151
|
+
*
|
|
152
|
+
* Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
|
|
153
|
+
* executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
|
|
154
|
+
* INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
|
|
155
|
+
* unprobeable column degrades that column's verdict to `'unknown'` and never the
|
|
156
|
+
* pass.
|
|
157
|
+
*/
|
|
158
|
+
export declare function probePlanFlips(options: ProbePlanFlipsOptions): Promise<FlipProbeResult>;
|
|
159
|
+
/**
|
|
160
|
+
* Drop the findings the planner refuted, and record how many.
|
|
161
|
+
*
|
|
162
|
+
* Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
|
|
163
|
+
* contract in the module header.
|
|
164
|
+
*/
|
|
165
|
+
export declare function applyFlipVerdicts(report: PlanDivergenceReport, probe: FlipProbeResult): PlanDivergenceReport;
|
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Plan-flip probe: does the generic plan actually differ from the good one?
|
|
4
|
+
*
|
|
5
|
+
* ## Why this exists
|
|
6
|
+
*
|
|
7
|
+
* `plan-divergence.ts` scores a column from statistics alone and answers "IF the
|
|
8
|
+
* cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
|
|
9
|
+
* without an answer to the prior question, "CAN it flip at all", and that turned
|
|
10
|
+
* out to be the majority case: on a real 118-model schema the branch produced 39
|
|
11
|
+
* findings of which a measured sample was right 6 times in 13.
|
|
12
|
+
*
|
|
13
|
+
* Every false positive had one signature: **the generic plan kept the same
|
|
14
|
+
* sequential scan the custom plan chose.** There was no flip to be had, so the
|
|
15
|
+
* amplification the finding printed described a plan the planner would never
|
|
16
|
+
* pick.
|
|
17
|
+
*
|
|
18
|
+
* ## Why this is a probe and not another rule
|
|
19
|
+
*
|
|
20
|
+
* The obvious gate is arithmetic: require the generic row estimate
|
|
21
|
+
* (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
|
|
22
|
+
* Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
|
|
23
|
+
* estimate at or below the limit earns no discount at all.
|
|
24
|
+
*
|
|
25
|
+
* That rule is wrong, and it was measured wrong before it was written down here.
|
|
26
|
+
* The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
|
|
27
|
+
* which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
|
|
28
|
+
* that, and the plan is then chosen by comparing that fraction of the full index
|
|
29
|
+
* scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
|
|
30
|
+
* between estimates 3 and 4, not at the limit of 20:
|
|
31
|
+
*
|
|
32
|
+
* ```txt
|
|
33
|
+
* generic estimate 2 3 4 10 20 500
|
|
34
|
+
* generic plan seq seq index index index index
|
|
35
|
+
* buffers 250 250 20,074 20,074 19,071 765
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
|
|
39
|
+
* would discard. The boundary is also not a fixed number: it is a cost
|
|
40
|
+
* comparison, so it moves with the table. Reproducing the same estimate on a
|
|
41
|
+
* 1976-page table and on an 89-page narrow table put the flip in a different
|
|
42
|
+
* place each time.
|
|
43
|
+
*
|
|
44
|
+
* So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
|
|
45
|
+
* WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
|
|
46
|
+
* planner the exact question the rule was trying to predict. This module runs it.
|
|
47
|
+
*
|
|
48
|
+
* ## What it asks, and why only half the pair
|
|
49
|
+
*
|
|
50
|
+
* It plans ONE statement, under `force_generic_plan` only:
|
|
51
|
+
*
|
|
52
|
+
* ```sql
|
|
53
|
+
* PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
|
|
54
|
+
* EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
|
|
55
|
+
* ```
|
|
56
|
+
*
|
|
57
|
+
* The custom plan is not needed. The finding's whole claim is that a promoted
|
|
58
|
+
* generic plan abandons the seq scan for an ordered index walk, so if the generic
|
|
59
|
+
* plan IS a seq scan on the target table, the claim is refuted no matter what the
|
|
60
|
+
* custom plan does. Asking one question instead of two halves the work and
|
|
61
|
+
* removes the need for a representative rare value, which statistics do not
|
|
62
|
+
* carry.
|
|
63
|
+
*
|
|
64
|
+
* `NULL` is a safe argument precisely because the plan is generic: a generic plan
|
|
65
|
+
* is built without looking at the value, which is the property the whole check is
|
|
66
|
+
* about. The LIMIT is bound as `$2` rather than inlined because that is the shape
|
|
67
|
+
* Turbine emits, and an inlined limit takes a different code path in the planner.
|
|
68
|
+
*
|
|
69
|
+
* ## Failure is never a silent drop
|
|
70
|
+
*
|
|
71
|
+
* A probe that errors, times out, or returns an unparseable plan yields
|
|
72
|
+
* `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
|
|
73
|
+
* findings when the database is uncooperative would be worse than one that
|
|
74
|
+
* over-reports, because the failure would be invisible in exactly the
|
|
75
|
+
* environments (restricted roles, non-Postgres engines) where a human is least
|
|
76
|
+
* able to check.
|
|
77
|
+
*
|
|
78
|
+
* @module
|
|
79
|
+
*/
|
|
80
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
81
|
+
if (k2 === undefined) k2 = k;
|
|
82
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
83
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
84
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
85
|
+
}
|
|
86
|
+
Object.defineProperty(o, k2, desc);
|
|
87
|
+
}) : (function(o, m, k, k2) {
|
|
88
|
+
if (k2 === undefined) k2 = k;
|
|
89
|
+
o[k2] = m[k];
|
|
90
|
+
}));
|
|
91
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
92
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
93
|
+
}) : function(o, v) {
|
|
94
|
+
o["default"] = v;
|
|
95
|
+
});
|
|
96
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
97
|
+
var ownKeys = function(o) {
|
|
98
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
99
|
+
var ar = [];
|
|
100
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
101
|
+
return ar;
|
|
102
|
+
};
|
|
103
|
+
return ownKeys(o);
|
|
104
|
+
};
|
|
105
|
+
return function (mod) {
|
|
106
|
+
if (mod && mod.__esModule) return mod;
|
|
107
|
+
var result = {};
|
|
108
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
109
|
+
__setModuleDefault(result, mod);
|
|
110
|
+
return result;
|
|
111
|
+
};
|
|
112
|
+
})();
|
|
113
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
114
|
+
exports.emptyFlipProbeResult = emptyFlipProbeResult;
|
|
115
|
+
exports.flipProbeKey = flipProbeKey;
|
|
116
|
+
exports.needsFlipProbe = needsFlipProbe;
|
|
117
|
+
exports.buildFlipProbeSql = buildFlipProbeSql;
|
|
118
|
+
exports.verdictFromPlanJson = verdictFromPlanJson;
|
|
119
|
+
exports.probePlanFlips = probePlanFlips;
|
|
120
|
+
exports.applyFlipVerdicts = applyFlipVerdicts;
|
|
121
|
+
const utils_js_1 = require("./query/utils.js");
|
|
122
|
+
/** An empty result, which keeps every finding. Used when probing is off. */
|
|
123
|
+
function emptyFlipProbeResult() {
|
|
124
|
+
return { available: false, verdicts: {}, notices: [] };
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Map key for a (table, column) pair.
|
|
128
|
+
*
|
|
129
|
+
* `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
|
|
130
|
+
* literal NUL in a source file makes `grep` treat the whole file as binary, which
|
|
131
|
+
* is how four of them survived a release in `cli/index.ts`.
|
|
132
|
+
*/
|
|
133
|
+
function flipProbeKey(table, column) {
|
|
134
|
+
return `${table}\u0000${column}`;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Which findings are worth probing.
|
|
138
|
+
*
|
|
139
|
+
* `unindexed-filter` only. The `sparse-value` branch already requires an index
|
|
140
|
+
* that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
|
|
141
|
+
* that later produced 6 of 13 here, so there is no measured precision problem to
|
|
142
|
+
* spend a round trip on.
|
|
143
|
+
*/
|
|
144
|
+
function needsFlipProbe(finding) {
|
|
145
|
+
return finding.branch === 'unindexed-filter';
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* The SQL for one probe. Pure, so the exact text is unit-testable without a
|
|
149
|
+
* database.
|
|
150
|
+
*
|
|
151
|
+
* Every identifier goes through {@link quoteIdent}; the only values in the
|
|
152
|
+
* statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
|
|
153
|
+
* caller as `tpf_<index>` and is never caller-controlled text.
|
|
154
|
+
*/
|
|
155
|
+
function buildFlipProbeSql(finding, name, searchSchema) {
|
|
156
|
+
const rel = searchSchema ? `${(0, utils_js_1.quoteIdent)(searchSchema)}.${(0, utils_js_1.quoteIdent)(finding.table)}` : (0, utils_js_1.quoteIdent)(finding.table);
|
|
157
|
+
// No declared parameter types: Postgres infers both from context, which avoids
|
|
158
|
+
// maintaining a second pg-type mapping that could disagree with the column's
|
|
159
|
+
// real type and turn a diagnostic into an error.
|
|
160
|
+
const prepare = `PREPARE ${name} AS SELECT * FROM ${rel} ` +
|
|
161
|
+
`WHERE ${(0, utils_js_1.quoteIdent)(finding.column)} = $1 ` +
|
|
162
|
+
`ORDER BY ${(0, utils_js_1.quoteIdent)(finding.orderColumn)} LIMIT $2`;
|
|
163
|
+
return {
|
|
164
|
+
prepare,
|
|
165
|
+
explain: `EXPLAIN (FORMAT JSON) EXECUTE ${name}(NULL, ${Number(finding.assumedLimit)})`,
|
|
166
|
+
deallocate: `DEALLOCATE ${name}`,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
function* walkPlan(node) {
|
|
170
|
+
yield node;
|
|
171
|
+
for (const child of node.Plans ?? [])
|
|
172
|
+
yield* walkPlan(child);
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
|
|
176
|
+
*
|
|
177
|
+
* Exported for unit tests: the plan shapes this has to classify are exactly the
|
|
178
|
+
* ones that are tedious to produce live.
|
|
179
|
+
*
|
|
180
|
+
* The rule is deliberately narrow. Only a `Seq Scan` ON THE TARGET TABLE refutes
|
|
181
|
+
* a finding. A seq scan of some other relation in a more complex plan says
|
|
182
|
+
* nothing about this column, and anything that is not a plain sequential scan of
|
|
183
|
+
* the target (index scan, bitmap heap scan, index-only scan) leaves the flip
|
|
184
|
+
* reachable.
|
|
185
|
+
*/
|
|
186
|
+
function verdictFromPlanJson(payload, table) {
|
|
187
|
+
const root = Array.isArray(payload) ? payload[0] : undefined;
|
|
188
|
+
const plan = root?.Plan;
|
|
189
|
+
if (!plan)
|
|
190
|
+
return 'unknown';
|
|
191
|
+
for (const node of walkPlan(plan)) {
|
|
192
|
+
if (node['Relation Name'] !== table)
|
|
193
|
+
continue;
|
|
194
|
+
const type = node['Node Type'];
|
|
195
|
+
if (type === undefined)
|
|
196
|
+
continue;
|
|
197
|
+
return type === 'Seq Scan' ? 'no-flip' : 'flip-reachable';
|
|
198
|
+
}
|
|
199
|
+
// The target table is not in the plan at all, which should not happen for a
|
|
200
|
+
// statement that selects from it. Treated as unknown rather than as a refutation.
|
|
201
|
+
return 'unknown';
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Ask the planner, once per candidate finding, whether the flip is reachable.
|
|
205
|
+
*
|
|
206
|
+
* Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
|
|
207
|
+
* executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
|
|
208
|
+
* INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
|
|
209
|
+
* unprobeable column degrades that column's verdict to `'unknown'` and never the
|
|
210
|
+
* pass.
|
|
211
|
+
*/
|
|
212
|
+
async function probePlanFlips(options) {
|
|
213
|
+
const targets = options.findings.filter(needsFlipProbe);
|
|
214
|
+
const result = { available: false, verdicts: {}, notices: [] };
|
|
215
|
+
if (targets.length === 0) {
|
|
216
|
+
result.available = true;
|
|
217
|
+
return result;
|
|
218
|
+
}
|
|
219
|
+
const { Client } = (await Promise.resolve().then(() => __importStar(require('pg')))).default;
|
|
220
|
+
const client = new Client({ connectionString: options.connectionString });
|
|
221
|
+
try {
|
|
222
|
+
await client.connect();
|
|
223
|
+
await client.query(`SET statement_timeout = ${Number(options.statementTimeoutMs ?? 5000)}`);
|
|
224
|
+
// READ ONLY is belt-and-braces: EXPLAIN without ANALYZE cannot write, and the
|
|
225
|
+
// transaction is rolled back regardless. It costs nothing and makes the
|
|
226
|
+
// read-only intent checkable from a server-side log.
|
|
227
|
+
await client.query('BEGIN READ ONLY');
|
|
228
|
+
result.available = true;
|
|
229
|
+
for (let i = 0; i < targets.length; i++) {
|
|
230
|
+
const finding = targets[i];
|
|
231
|
+
const key = flipProbeKey(finding.table, finding.column);
|
|
232
|
+
const name = `tpf_${i}`;
|
|
233
|
+
const sql = buildFlipProbeSql(finding, name, options.schema);
|
|
234
|
+
try {
|
|
235
|
+
// A failed probe must not poison the surrounding transaction for the
|
|
236
|
+
// probes after it, so each one gets its own savepoint.
|
|
237
|
+
await client.query(`SAVEPOINT ${name}`);
|
|
238
|
+
await client.query(sql.prepare);
|
|
239
|
+
await client.query('SET LOCAL plan_cache_mode = force_generic_plan');
|
|
240
|
+
const res = await client.query(sql.explain);
|
|
241
|
+
const row = res.rows[0];
|
|
242
|
+
const payload = row ? Object.values(row)[0] : undefined;
|
|
243
|
+
result.verdicts[key] =
|
|
244
|
+
typeof payload === 'string'
|
|
245
|
+
? verdictFromPlanJson(JSON.parse(payload), finding.table)
|
|
246
|
+
: verdictFromPlanJson(payload, finding.table);
|
|
247
|
+
await client.query(sql.deallocate);
|
|
248
|
+
await client.query(`RELEASE SAVEPOINT ${name}`);
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
result.verdicts[key] = 'unknown';
|
|
252
|
+
result.notices.push(`flip probe on ${finding.table}.${finding.column} was inconclusive (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); the finding is kept`);
|
|
253
|
+
try {
|
|
254
|
+
await client.query(`ROLLBACK TO SAVEPOINT ${name}`);
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
// The transaction itself is gone; the remaining probes will each record
|
|
258
|
+
// their own notice and the pass still returns what it has.
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
catch (err) {
|
|
264
|
+
result.notices.push(`plan-flip probing unavailable (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); findings are reported unverified`);
|
|
265
|
+
}
|
|
266
|
+
finally {
|
|
267
|
+
try {
|
|
268
|
+
await client.query('ROLLBACK');
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
// Nothing to roll back.
|
|
272
|
+
}
|
|
273
|
+
await client.end().catch(() => { });
|
|
274
|
+
}
|
|
275
|
+
return result;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Drop the findings the planner refuted, and record how many.
|
|
279
|
+
*
|
|
280
|
+
* Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
|
|
281
|
+
* contract in the module header.
|
|
282
|
+
*/
|
|
283
|
+
function applyFlipVerdicts(report, probe) {
|
|
284
|
+
if (!probe.available)
|
|
285
|
+
return report;
|
|
286
|
+
let refuted = 0;
|
|
287
|
+
const findings = report.findings.filter((f) => {
|
|
288
|
+
if (!needsFlipProbe(f))
|
|
289
|
+
return true;
|
|
290
|
+
const verdict = probe.verdicts[flipProbeKey(f.table, f.column)];
|
|
291
|
+
if (verdict === 'no-flip') {
|
|
292
|
+
refuted++;
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
return true;
|
|
296
|
+
});
|
|
297
|
+
return {
|
|
298
|
+
...report,
|
|
299
|
+
findings,
|
|
300
|
+
flipProbed: true,
|
|
301
|
+
flipRefuted: refuted,
|
|
302
|
+
notices: [...report.notices, ...probe.notices.map((n) => ({ table: '', column: '', reason: n }))],
|
|
303
|
+
};
|
|
304
|
+
}
|
package/dist/cli/index.js
CHANGED
|
@@ -33,6 +33,7 @@ import { buildCreateIndexSql, buildDropIndexSql, collectDoctorProbeIndexNames, c
|
|
|
33
33
|
import { auditDoctorIndexes, collectStatsSnapshot, collectTableHeat, findInvalidIndexes, findRedundantIndexes, findUnusedIndexes, formatBytes, isSnapshotUsable, STATS_THRESHOLDS, scoreMissingIndex, } from '../index-stats.js';
|
|
34
34
|
import { introspect } from '../introspect.js';
|
|
35
35
|
import { collectDivergenceCandidateColumns, collectDivergenceOrderColumns, findPlanDivergence, PLAN_DIVERGENCE_THRESHOLDS, } from '../plan-divergence.js';
|
|
36
|
+
import { applyFlipVerdicts, emptyFlipProbeResult, needsFlipProbe, probePlanFlips } from '../plan-flip-probe.js';
|
|
36
37
|
import { snakeToCamel } from '../schema.js';
|
|
37
38
|
import { DestructivePushRefusal, schemaDiff, schemaPush } from '../schema-sql.js';
|
|
38
39
|
import { configTemplate, DEFAULT_INIT_SEED_FILE, findConfigFile, loadConfigResult, looksLikeSchemaFilePath, resolveConfig, resolveSeedFile, unwrapModuleDefault, } from './config.js';
|
|
@@ -2333,9 +2334,18 @@ async function cmdDoctor(args, config) {
|
|
|
2333
2334
|
// trustworthy stats_reset age because they normalize write counters by it;
|
|
2334
2335
|
// this check reads no counter, only pg_stats, whose freshness is ANALYZE. A
|
|
2335
2336
|
// cluster with a NULL stats_reset (the default) must still get the check.
|
|
2336
|
-
const
|
|
2337
|
+
const scored = divergenceOn && snapshot.available
|
|
2337
2338
|
? findPlanDivergence(schema, snapshot)
|
|
2338
2339
|
: { findings: [], notices: [], candidatesConsidered: 0, consideredIndexed: 0, consideredUnindexed: 0 };
|
|
2340
|
+
// Statistics can say how bad a flip WOULD be; only the planner can say whether
|
|
2341
|
+
// it is reachable. The `unindexed-filter` branch shipped in 0.57 without that
|
|
2342
|
+
// question answered and was right 6 times in 13 on a real schema, so every one
|
|
2343
|
+
// of its findings is now put to a plan-only EXPLAIN. Nothing is executed, and a
|
|
2344
|
+
// probe that fails keeps its finding rather than dropping it.
|
|
2345
|
+
const flipProbe = divergenceOn && scored.findings.some(needsFlipProbe)
|
|
2346
|
+
? await probePlanFlips({ connectionString: url, schema: config.schema, findings: scored.findings })
|
|
2347
|
+
: emptyFlipProbeResult();
|
|
2348
|
+
const divergence = applyFlipVerdicts(scored, flipProbe);
|
|
2339
2349
|
if (jsonMode) {
|
|
2340
2350
|
spinner?.stop();
|
|
2341
2351
|
console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, heat, subtract, divergence, args }), null, 2));
|
|
@@ -2429,6 +2439,12 @@ function buildDoctorJson(ctx) {
|
|
|
2429
2439
|
considered: ctx.divergence.candidatesConsidered,
|
|
2430
2440
|
indexed: ctx.divergence.consideredIndexed,
|
|
2431
2441
|
unindexed: ctx.divergence.consideredUnindexed,
|
|
2442
|
+
// Whether the unindexed findings above were put to the planner, and how many
|
|
2443
|
+
// it refuted. `flipProbed: false` means they are statistics-only and carry
|
|
2444
|
+
// 0.57's precision, so a consumer can tell a verified list from an unverified
|
|
2445
|
+
// one instead of inferring it from the count.
|
|
2446
|
+
flipProbed: ctx.divergence.flipProbed === true,
|
|
2447
|
+
flipRefuted: ctx.divergence.flipRefuted ?? 0,
|
|
2432
2448
|
};
|
|
2433
2449
|
return out;
|
|
2434
2450
|
}
|
|
@@ -2751,6 +2767,19 @@ function renderDivergenceGates(divergence) {
|
|
|
2751
2767
|
console.log(` ${dim(`${divergence.candidatesConsidered} column(s) were scored (${divergence.consideredIndexed} indexed, ${divergence.consideredUnindexed} unindexed). That population is relation-probe`)}`);
|
|
2752
2768
|
console.log(` ${dim('and leading-index columns only: a filter column that is neither is not covered. Skip this')}`);
|
|
2753
2769
|
console.log(` ${dim('section with --no-plan-divergence.')}`);
|
|
2770
|
+
// Say whether the unindexed findings were verified, and what verification
|
|
2771
|
+
// removed. Statistics can only say how bad a flip would be; a plan-only EXPLAIN
|
|
2772
|
+
// says whether the planner can reach it at all.
|
|
2773
|
+
if (divergence.flipProbed === true) {
|
|
2774
|
+
const refuted = divergence.flipRefuted ?? 0;
|
|
2775
|
+
const wereRefuted = refuted === 1 ? '1 was refuted' : `${refuted} were refuted`;
|
|
2776
|
+
console.log(` ${dim(`Every unindexed finding was put to the planner (EXPLAIN, nothing executed); ${wereRefuted}`)}`);
|
|
2777
|
+
console.log(` ${dim('because the generic plan keeps the same sequential scan, so no flip is reachable.')}`);
|
|
2778
|
+
}
|
|
2779
|
+
else if (divergence.consideredUnindexed > 0) {
|
|
2780
|
+
console.log(` ${dim('Unindexed findings are UNVERIFIED here: the planner probe did not run, so some may name a')}`);
|
|
2781
|
+
console.log(` ${dim('divergence the planner would never choose.')}`);
|
|
2782
|
+
}
|
|
2754
2783
|
}
|
|
2755
2784
|
/** Round to a whole number and group it, for the divergence report's estimates. */
|
|
2756
2785
|
function divInt(n) {
|
|
@@ -424,6 +424,21 @@ export interface PlanDivergenceReport {
|
|
|
424
424
|
*/
|
|
425
425
|
consideredIndexed: number;
|
|
426
426
|
consideredUnindexed: number;
|
|
427
|
+
/**
|
|
428
|
+
* True when every `unindexed-filter` finding was put to the planner via
|
|
429
|
+
* `plan-flip-probe.ts` rather than reported on statistics alone.
|
|
430
|
+
*
|
|
431
|
+
* Absent or false means the findings below are UNVERIFIED: the probe was
|
|
432
|
+
* skipped, the engine is not Postgres, or the connection refused it.
|
|
433
|
+
*/
|
|
434
|
+
flipProbed?: boolean;
|
|
435
|
+
/**
|
|
436
|
+
* How many candidate findings the planner refuted, i.e. columns where the
|
|
437
|
+
* generic plan keeps the same sequential scan the good plan uses so there is
|
|
438
|
+
* no divergence to reach. Reported because a check that silently discards two
|
|
439
|
+
* thirds of what it found should say so.
|
|
440
|
+
*/
|
|
441
|
+
flipRefuted?: number;
|
|
427
442
|
}
|
|
428
443
|
/** A (table, column) pair whose distribution statistics the collector should read. */
|
|
429
444
|
export interface DivergenceCandidate {
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plan-flip probe: does the generic plan actually differ from the good one?
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* `plan-divergence.ts` scores a column from statistics alone and answers "IF the
|
|
7
|
+
* cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
|
|
8
|
+
* without an answer to the prior question, "CAN it flip at all", and that turned
|
|
9
|
+
* out to be the majority case: on a real 118-model schema the branch produced 39
|
|
10
|
+
* findings of which a measured sample was right 6 times in 13.
|
|
11
|
+
*
|
|
12
|
+
* Every false positive had one signature: **the generic plan kept the same
|
|
13
|
+
* sequential scan the custom plan chose.** There was no flip to be had, so the
|
|
14
|
+
* amplification the finding printed described a plan the planner would never
|
|
15
|
+
* pick.
|
|
16
|
+
*
|
|
17
|
+
* ## Why this is a probe and not another rule
|
|
18
|
+
*
|
|
19
|
+
* The obvious gate is arithmetic: require the generic row estimate
|
|
20
|
+
* (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
|
|
21
|
+
* Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
|
|
22
|
+
* estimate at or below the limit earns no discount at all.
|
|
23
|
+
*
|
|
24
|
+
* That rule is wrong, and it was measured wrong before it was written down here.
|
|
25
|
+
* The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
|
|
26
|
+
* which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
|
|
27
|
+
* that, and the plan is then chosen by comparing that fraction of the full index
|
|
28
|
+
* scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
|
|
29
|
+
* between estimates 3 and 4, not at the limit of 20:
|
|
30
|
+
*
|
|
31
|
+
* ```txt
|
|
32
|
+
* generic estimate 2 3 4 10 20 500
|
|
33
|
+
* generic plan seq seq index index index index
|
|
34
|
+
* buffers 250 250 20,074 20,074 19,071 765
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
|
|
38
|
+
* would discard. The boundary is also not a fixed number: it is a cost
|
|
39
|
+
* comparison, so it moves with the table. Reproducing the same estimate on a
|
|
40
|
+
* 1976-page table and on an 89-page narrow table put the flip in a different
|
|
41
|
+
* place each time.
|
|
42
|
+
*
|
|
43
|
+
* So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
|
|
44
|
+
* WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
|
|
45
|
+
* planner the exact question the rule was trying to predict. This module runs it.
|
|
46
|
+
*
|
|
47
|
+
* ## What it asks, and why only half the pair
|
|
48
|
+
*
|
|
49
|
+
* It plans ONE statement, under `force_generic_plan` only:
|
|
50
|
+
*
|
|
51
|
+
* ```sql
|
|
52
|
+
* PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
|
|
53
|
+
* EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* The custom plan is not needed. The finding's whole claim is that a promoted
|
|
57
|
+
* generic plan abandons the seq scan for an ordered index walk, so if the generic
|
|
58
|
+
* plan IS a seq scan on the target table, the claim is refuted no matter what the
|
|
59
|
+
* custom plan does. Asking one question instead of two halves the work and
|
|
60
|
+
* removes the need for a representative rare value, which statistics do not
|
|
61
|
+
* carry.
|
|
62
|
+
*
|
|
63
|
+
* `NULL` is a safe argument precisely because the plan is generic: a generic plan
|
|
64
|
+
* is built without looking at the value, which is the property the whole check is
|
|
65
|
+
* about. The LIMIT is bound as `$2` rather than inlined because that is the shape
|
|
66
|
+
* Turbine emits, and an inlined limit takes a different code path in the planner.
|
|
67
|
+
*
|
|
68
|
+
* ## Failure is never a silent drop
|
|
69
|
+
*
|
|
70
|
+
* A probe that errors, times out, or returns an unparseable plan yields
|
|
71
|
+
* `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
|
|
72
|
+
* findings when the database is uncooperative would be worse than one that
|
|
73
|
+
* over-reports, because the failure would be invisible in exactly the
|
|
74
|
+
* environments (restricted roles, non-Postgres engines) where a human is least
|
|
75
|
+
* able to check.
|
|
76
|
+
*
|
|
77
|
+
* @module
|
|
78
|
+
*/
|
|
79
|
+
import type { PlanDivergenceFinding, PlanDivergenceReport } from './plan-divergence.js';
|
|
80
|
+
/**
|
|
81
|
+
* The planner's answer for one finding.
|
|
82
|
+
*
|
|
83
|
+
* - `'flip-reachable'`, the generic plan is NOT a plain seq scan of the target
|
|
84
|
+
* table, so the divergence the finding describes is one the planner can
|
|
85
|
+
* actually choose.
|
|
86
|
+
* - `'no-flip'`, the generic plan sequentially scans the target table, the same
|
|
87
|
+
* access the good plan uses. Nothing to diverge to.
|
|
88
|
+
* - `'unknown'`, the probe did not produce an answer. The finding is kept.
|
|
89
|
+
*/
|
|
90
|
+
export type FlipVerdict = 'flip-reachable' | 'no-flip' | 'unknown';
|
|
91
|
+
/** Outcome of a probe pass, keyed by {@link flipProbeKey}. */
|
|
92
|
+
export interface FlipProbeResult {
|
|
93
|
+
/** True when the probe pass ran at all (Postgres, connection succeeded). */
|
|
94
|
+
available: boolean;
|
|
95
|
+
verdicts: Record<string, FlipVerdict>;
|
|
96
|
+
notices: string[];
|
|
97
|
+
}
|
|
98
|
+
/** An empty result, which keeps every finding. Used when probing is off. */
|
|
99
|
+
export declare function emptyFlipProbeResult(): FlipProbeResult;
|
|
100
|
+
/**
|
|
101
|
+
* Map key for a (table, column) pair.
|
|
102
|
+
*
|
|
103
|
+
* `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
|
|
104
|
+
* literal NUL in a source file makes `grep` treat the whole file as binary, which
|
|
105
|
+
* is how four of them survived a release in `cli/index.ts`.
|
|
106
|
+
*/
|
|
107
|
+
export declare function flipProbeKey(table: string, column: string): string;
|
|
108
|
+
/**
|
|
109
|
+
* Which findings are worth probing.
|
|
110
|
+
*
|
|
111
|
+
* `unindexed-filter` only. The `sparse-value` branch already requires an index
|
|
112
|
+
* that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
|
|
113
|
+
* that later produced 6 of 13 here, so there is no measured precision problem to
|
|
114
|
+
* spend a round trip on.
|
|
115
|
+
*/
|
|
116
|
+
export declare function needsFlipProbe(finding: PlanDivergenceFinding): boolean;
|
|
117
|
+
/**
|
|
118
|
+
* The SQL for one probe. Pure, so the exact text is unit-testable without a
|
|
119
|
+
* database.
|
|
120
|
+
*
|
|
121
|
+
* Every identifier goes through {@link quoteIdent}; the only values in the
|
|
122
|
+
* statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
|
|
123
|
+
* caller as `tpf_<index>` and is never caller-controlled text.
|
|
124
|
+
*/
|
|
125
|
+
export declare function buildFlipProbeSql(finding: PlanDivergenceFinding, name: string, searchSchema?: string): {
|
|
126
|
+
prepare: string;
|
|
127
|
+
explain: string;
|
|
128
|
+
deallocate: string;
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
|
|
132
|
+
*
|
|
133
|
+
* Exported for unit tests: the plan shapes this has to classify are exactly the
|
|
134
|
+
* ones that are tedious to produce live.
|
|
135
|
+
*
|
|
136
|
+
* The rule is deliberately narrow. Only a `Seq Scan` ON THE TARGET TABLE refutes
|
|
137
|
+
* a finding. A seq scan of some other relation in a more complex plan says
|
|
138
|
+
* nothing about this column, and anything that is not a plain sequential scan of
|
|
139
|
+
* the target (index scan, bitmap heap scan, index-only scan) leaves the flip
|
|
140
|
+
* reachable.
|
|
141
|
+
*/
|
|
142
|
+
export declare function verdictFromPlanJson(payload: unknown, table: string): FlipVerdict;
|
|
143
|
+
export interface ProbePlanFlipsOptions {
|
|
144
|
+
connectionString: string;
|
|
145
|
+
schema?: string;
|
|
146
|
+
findings: PlanDivergenceFinding[];
|
|
147
|
+
statementTimeoutMs?: number;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Ask the planner, once per candidate finding, whether the flip is reachable.
|
|
151
|
+
*
|
|
152
|
+
* Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
|
|
153
|
+
* executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
|
|
154
|
+
* INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
|
|
155
|
+
* unprobeable column degrades that column's verdict to `'unknown'` and never the
|
|
156
|
+
* pass.
|
|
157
|
+
*/
|
|
158
|
+
export declare function probePlanFlips(options: ProbePlanFlipsOptions): Promise<FlipProbeResult>;
|
|
159
|
+
/**
|
|
160
|
+
* Drop the findings the planner refuted, and record how many.
|
|
161
|
+
*
|
|
162
|
+
* Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
|
|
163
|
+
* contract in the module header.
|
|
164
|
+
*/
|
|
165
|
+
export declare function applyFlipVerdicts(report: PlanDivergenceReport, probe: FlipProbeResult): PlanDivergenceReport;
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plan-flip probe: does the generic plan actually differ from the good one?
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* `plan-divergence.ts` scores a column from statistics alone and answers "IF the
|
|
7
|
+
* cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
|
|
8
|
+
* without an answer to the prior question, "CAN it flip at all", and that turned
|
|
9
|
+
* out to be the majority case: on a real 118-model schema the branch produced 39
|
|
10
|
+
* findings of which a measured sample was right 6 times in 13.
|
|
11
|
+
*
|
|
12
|
+
* Every false positive had one signature: **the generic plan kept the same
|
|
13
|
+
* sequential scan the custom plan chose.** There was no flip to be had, so the
|
|
14
|
+
* amplification the finding printed described a plan the planner would never
|
|
15
|
+
* pick.
|
|
16
|
+
*
|
|
17
|
+
* ## Why this is a probe and not another rule
|
|
18
|
+
*
|
|
19
|
+
* The obvious gate is arithmetic: require the generic row estimate
|
|
20
|
+
* (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
|
|
21
|
+
* Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
|
|
22
|
+
* estimate at or below the limit earns no discount at all.
|
|
23
|
+
*
|
|
24
|
+
* That rule is wrong, and it was measured wrong before it was written down here.
|
|
25
|
+
* The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
|
|
26
|
+
* which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
|
|
27
|
+
* that, and the plan is then chosen by comparing that fraction of the full index
|
|
28
|
+
* scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
|
|
29
|
+
* between estimates 3 and 4, not at the limit of 20:
|
|
30
|
+
*
|
|
31
|
+
* ```txt
|
|
32
|
+
* generic estimate 2 3 4 10 20 500
|
|
33
|
+
* generic plan seq seq index index index index
|
|
34
|
+
* buffers 250 250 20,074 20,074 19,071 765
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
|
|
38
|
+
* would discard. The boundary is also not a fixed number: it is a cost
|
|
39
|
+
* comparison, so it moves with the table. Reproducing the same estimate on a
|
|
40
|
+
* 1976-page table and on an 89-page narrow table put the flip in a different
|
|
41
|
+
* place each time.
|
|
42
|
+
*
|
|
43
|
+
* So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
|
|
44
|
+
* WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
|
|
45
|
+
* planner the exact question the rule was trying to predict. This module runs it.
|
|
46
|
+
*
|
|
47
|
+
* ## What it asks, and why only half the pair
|
|
48
|
+
*
|
|
49
|
+
* It plans ONE statement, under `force_generic_plan` only:
|
|
50
|
+
*
|
|
51
|
+
* ```sql
|
|
52
|
+
* PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
|
|
53
|
+
* EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
|
|
54
|
+
* ```
|
|
55
|
+
*
|
|
56
|
+
* The custom plan is not needed. The finding's whole claim is that a promoted
|
|
57
|
+
* generic plan abandons the seq scan for an ordered index walk, so if the generic
|
|
58
|
+
* plan IS a seq scan on the target table, the claim is refuted no matter what the
|
|
59
|
+
* custom plan does. Asking one question instead of two halves the work and
|
|
60
|
+
* removes the need for a representative rare value, which statistics do not
|
|
61
|
+
* carry.
|
|
62
|
+
*
|
|
63
|
+
* `NULL` is a safe argument precisely because the plan is generic: a generic plan
|
|
64
|
+
* is built without looking at the value, which is the property the whole check is
|
|
65
|
+
* about. The LIMIT is bound as `$2` rather than inlined because that is the shape
|
|
66
|
+
* Turbine emits, and an inlined limit takes a different code path in the planner.
|
|
67
|
+
*
|
|
68
|
+
* ## Failure is never a silent drop
|
|
69
|
+
*
|
|
70
|
+
* A probe that errors, times out, or returns an unparseable plan yields
|
|
71
|
+
* `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
|
|
72
|
+
* findings when the database is uncooperative would be worse than one that
|
|
73
|
+
* over-reports, because the failure would be invisible in exactly the
|
|
74
|
+
* environments (restricted roles, non-Postgres engines) where a human is least
|
|
75
|
+
* able to check.
|
|
76
|
+
*
|
|
77
|
+
* @module
|
|
78
|
+
*/
|
|
79
|
+
import { quoteIdent } from './query/utils.js';
|
|
80
|
+
/** An empty result, which keeps every finding. Used when probing is off. */
|
|
81
|
+
export function emptyFlipProbeResult() {
|
|
82
|
+
return { available: false, verdicts: {}, notices: [] };
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Map key for a (table, column) pair.
|
|
86
|
+
*
|
|
87
|
+
* `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
|
|
88
|
+
* literal NUL in a source file makes `grep` treat the whole file as binary, which
|
|
89
|
+
* is how four of them survived a release in `cli/index.ts`.
|
|
90
|
+
*/
|
|
91
|
+
export function flipProbeKey(table, column) {
|
|
92
|
+
return `${table}\u0000${column}`;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Which findings are worth probing.
|
|
96
|
+
*
|
|
97
|
+
* `unindexed-filter` only. The `sparse-value` branch already requires an index
|
|
98
|
+
* that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
|
|
99
|
+
* that later produced 6 of 13 here, so there is no measured precision problem to
|
|
100
|
+
* spend a round trip on.
|
|
101
|
+
*/
|
|
102
|
+
export function needsFlipProbe(finding) {
|
|
103
|
+
return finding.branch === 'unindexed-filter';
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The SQL for one probe. Pure, so the exact text is unit-testable without a
|
|
107
|
+
* database.
|
|
108
|
+
*
|
|
109
|
+
* Every identifier goes through {@link quoteIdent}; the only values in the
|
|
110
|
+
* statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
|
|
111
|
+
* caller as `tpf_<index>` and is never caller-controlled text.
|
|
112
|
+
*/
|
|
113
|
+
export function buildFlipProbeSql(finding, name, searchSchema) {
|
|
114
|
+
const rel = searchSchema ? `${quoteIdent(searchSchema)}.${quoteIdent(finding.table)}` : quoteIdent(finding.table);
|
|
115
|
+
// No declared parameter types: Postgres infers both from context, which avoids
|
|
116
|
+
// maintaining a second pg-type mapping that could disagree with the column's
|
|
117
|
+
// real type and turn a diagnostic into an error.
|
|
118
|
+
const prepare = `PREPARE ${name} AS SELECT * FROM ${rel} ` +
|
|
119
|
+
`WHERE ${quoteIdent(finding.column)} = $1 ` +
|
|
120
|
+
`ORDER BY ${quoteIdent(finding.orderColumn)} LIMIT $2`;
|
|
121
|
+
return {
|
|
122
|
+
prepare,
|
|
123
|
+
explain: `EXPLAIN (FORMAT JSON) EXECUTE ${name}(NULL, ${Number(finding.assumedLimit)})`,
|
|
124
|
+
deallocate: `DEALLOCATE ${name}`,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function* walkPlan(node) {
|
|
128
|
+
yield node;
|
|
129
|
+
for (const child of node.Plans ?? [])
|
|
130
|
+
yield* walkPlan(child);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
|
|
134
|
+
*
|
|
135
|
+
* Exported for unit tests: the plan shapes this has to classify are exactly the
|
|
136
|
+
* ones that are tedious to produce live.
|
|
137
|
+
*
|
|
138
|
+
* The rule is deliberately narrow. Only a `Seq Scan` ON THE TARGET TABLE refutes
|
|
139
|
+
* a finding. A seq scan of some other relation in a more complex plan says
|
|
140
|
+
* nothing about this column, and anything that is not a plain sequential scan of
|
|
141
|
+
* the target (index scan, bitmap heap scan, index-only scan) leaves the flip
|
|
142
|
+
* reachable.
|
|
143
|
+
*/
|
|
144
|
+
export function verdictFromPlanJson(payload, table) {
|
|
145
|
+
const root = Array.isArray(payload) ? payload[0] : undefined;
|
|
146
|
+
const plan = root?.Plan;
|
|
147
|
+
if (!plan)
|
|
148
|
+
return 'unknown';
|
|
149
|
+
for (const node of walkPlan(plan)) {
|
|
150
|
+
if (node['Relation Name'] !== table)
|
|
151
|
+
continue;
|
|
152
|
+
const type = node['Node Type'];
|
|
153
|
+
if (type === undefined)
|
|
154
|
+
continue;
|
|
155
|
+
return type === 'Seq Scan' ? 'no-flip' : 'flip-reachable';
|
|
156
|
+
}
|
|
157
|
+
// The target table is not in the plan at all, which should not happen for a
|
|
158
|
+
// statement that selects from it. Treated as unknown rather than as a refutation.
|
|
159
|
+
return 'unknown';
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Ask the planner, once per candidate finding, whether the flip is reachable.
|
|
163
|
+
*
|
|
164
|
+
* Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
|
|
165
|
+
* executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
|
|
166
|
+
* INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
|
|
167
|
+
* unprobeable column degrades that column's verdict to `'unknown'` and never the
|
|
168
|
+
* pass.
|
|
169
|
+
*/
|
|
170
|
+
export async function probePlanFlips(options) {
|
|
171
|
+
const targets = options.findings.filter(needsFlipProbe);
|
|
172
|
+
const result = { available: false, verdicts: {}, notices: [] };
|
|
173
|
+
if (targets.length === 0) {
|
|
174
|
+
result.available = true;
|
|
175
|
+
return result;
|
|
176
|
+
}
|
|
177
|
+
const { Client } = (await import('pg')).default;
|
|
178
|
+
const client = new Client({ connectionString: options.connectionString });
|
|
179
|
+
try {
|
|
180
|
+
await client.connect();
|
|
181
|
+
await client.query(`SET statement_timeout = ${Number(options.statementTimeoutMs ?? 5000)}`);
|
|
182
|
+
// READ ONLY is belt-and-braces: EXPLAIN without ANALYZE cannot write, and the
|
|
183
|
+
// transaction is rolled back regardless. It costs nothing and makes the
|
|
184
|
+
// read-only intent checkable from a server-side log.
|
|
185
|
+
await client.query('BEGIN READ ONLY');
|
|
186
|
+
result.available = true;
|
|
187
|
+
for (let i = 0; i < targets.length; i++) {
|
|
188
|
+
const finding = targets[i];
|
|
189
|
+
const key = flipProbeKey(finding.table, finding.column);
|
|
190
|
+
const name = `tpf_${i}`;
|
|
191
|
+
const sql = buildFlipProbeSql(finding, name, options.schema);
|
|
192
|
+
try {
|
|
193
|
+
// A failed probe must not poison the surrounding transaction for the
|
|
194
|
+
// probes after it, so each one gets its own savepoint.
|
|
195
|
+
await client.query(`SAVEPOINT ${name}`);
|
|
196
|
+
await client.query(sql.prepare);
|
|
197
|
+
await client.query('SET LOCAL plan_cache_mode = force_generic_plan');
|
|
198
|
+
const res = await client.query(sql.explain);
|
|
199
|
+
const row = res.rows[0];
|
|
200
|
+
const payload = row ? Object.values(row)[0] : undefined;
|
|
201
|
+
result.verdicts[key] =
|
|
202
|
+
typeof payload === 'string'
|
|
203
|
+
? verdictFromPlanJson(JSON.parse(payload), finding.table)
|
|
204
|
+
: verdictFromPlanJson(payload, finding.table);
|
|
205
|
+
await client.query(sql.deallocate);
|
|
206
|
+
await client.query(`RELEASE SAVEPOINT ${name}`);
|
|
207
|
+
}
|
|
208
|
+
catch (err) {
|
|
209
|
+
result.verdicts[key] = 'unknown';
|
|
210
|
+
result.notices.push(`flip probe on ${finding.table}.${finding.column} was inconclusive (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); the finding is kept`);
|
|
211
|
+
try {
|
|
212
|
+
await client.query(`ROLLBACK TO SAVEPOINT ${name}`);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// The transaction itself is gone; the remaining probes will each record
|
|
216
|
+
// their own notice and the pass still returns what it has.
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch (err) {
|
|
222
|
+
result.notices.push(`plan-flip probing unavailable (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); findings are reported unverified`);
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
try {
|
|
226
|
+
await client.query('ROLLBACK');
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
// Nothing to roll back.
|
|
230
|
+
}
|
|
231
|
+
await client.end().catch(() => { });
|
|
232
|
+
}
|
|
233
|
+
return result;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Drop the findings the planner refuted, and record how many.
|
|
237
|
+
*
|
|
238
|
+
* Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
|
|
239
|
+
* contract in the module header.
|
|
240
|
+
*/
|
|
241
|
+
export function applyFlipVerdicts(report, probe) {
|
|
242
|
+
if (!probe.available)
|
|
243
|
+
return report;
|
|
244
|
+
let refuted = 0;
|
|
245
|
+
const findings = report.findings.filter((f) => {
|
|
246
|
+
if (!needsFlipProbe(f))
|
|
247
|
+
return true;
|
|
248
|
+
const verdict = probe.verdicts[flipProbeKey(f.table, f.column)];
|
|
249
|
+
if (verdict === 'no-flip') {
|
|
250
|
+
refuted++;
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
return true;
|
|
254
|
+
});
|
|
255
|
+
return {
|
|
256
|
+
...report,
|
|
257
|
+
findings,
|
|
258
|
+
flipProbed: true,
|
|
259
|
+
flipRefuted: refuted,
|
|
260
|
+
notices: [...report.notices, ...probe.notices.map((n) => ({ table: '', column: '', reason: n }))],
|
|
261
|
+
};
|
|
262
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "turbine-orm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.58.0",
|
|
4
4
|
"description": "Postgres-native TypeScript ORM, runs on Neon, Vercel Postgres, Cloudflare, Supabase. Streaming cursors, typed errors, single-query nested relations. One dependency, no WASM engine",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"//exports": "Each subpath declares its types PER CONDITION. A single shared top-level \"types\" resolves to the ESM declarations for `require` too, which is TS1479 (\"is an ES module ... cannot be require()d\") for any CJS consumer on moduleResolution node16/nodenext. The require condition points at dist/cjs, which ships its own {\"type\":\"commonjs\"} package.json, so those declarations are CJS declarations. Gated in CI by publint + @arethetypeswrong/cli + a real .cts consumer typecheck (see the package-types job in ci.yml).",
|