turbine-orm 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -83,6 +83,7 @@ const index_advisor_js_1 = require("../index-advisor.js");
83
83
  const index_stats_js_1 = require("../index-stats.js");
84
84
  const introspect_js_1 = require("../introspect.js");
85
85
  const plan_divergence_js_1 = require("../plan-divergence.js");
86
+ const plan_flip_probe_js_1 = require("../plan-flip-probe.js");
86
87
  const schema_js_1 = require("../schema.js");
87
88
  const schema_sql_js_1 = require("../schema-sql.js");
88
89
  const config_js_1 = require("./config.js");
@@ -2383,9 +2384,18 @@ async function cmdDoctor(args, config) {
2383
2384
  // trustworthy stats_reset age because they normalize write counters by it;
2384
2385
  // this check reads no counter, only pg_stats, whose freshness is ANALYZE. A
2385
2386
  // cluster with a NULL stats_reset (the default) must still get the check.
2386
- const divergence = divergenceOn && snapshot.available
2387
+ const scored = divergenceOn && snapshot.available
2387
2388
  ? (0, plan_divergence_js_1.findPlanDivergence)(schema, snapshot)
2388
2389
  : { findings: [], notices: [], candidatesConsidered: 0, consideredIndexed: 0, consideredUnindexed: 0 };
2390
+ // Statistics can say how bad a flip WOULD be; only the planner can say whether
2391
+ // it is reachable. The `unindexed-filter` branch shipped in 0.57 without that
2392
+ // question answered and was right 6 times in 13 on a real schema, so every one
2393
+ // of its findings is now put to a plan-only EXPLAIN. Nothing is executed, and a
2394
+ // probe that fails keeps its finding rather than dropping it.
2395
+ const flipProbe = divergenceOn && scored.findings.some(plan_flip_probe_js_1.needsFlipProbe)
2396
+ ? await (0, plan_flip_probe_js_1.probePlanFlips)({ connectionString: url, schema: config.schema, findings: scored.findings })
2397
+ : (0, plan_flip_probe_js_1.emptyFlipProbeResult)();
2398
+ const divergence = (0, plan_flip_probe_js_1.applyFlipVerdicts)(scored, flipProbe);
2389
2399
  if (jsonMode) {
2390
2400
  spinner?.stop();
2391
2401
  console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, heat, subtract, divergence, args }), null, 2));
@@ -2479,6 +2489,12 @@ function buildDoctorJson(ctx) {
2479
2489
  considered: ctx.divergence.candidatesConsidered,
2480
2490
  indexed: ctx.divergence.consideredIndexed,
2481
2491
  unindexed: ctx.divergence.consideredUnindexed,
2492
+ // Whether the unindexed findings above were put to the planner, and how many
2493
+ // it refuted. `flipProbed: false` means they are statistics-only and carry
2494
+ // 0.57's precision, so a consumer can tell a verified list from an unverified
2495
+ // one instead of inferring it from the count.
2496
+ flipProbed: ctx.divergence.flipProbed === true,
2497
+ flipRefuted: ctx.divergence.flipRefuted ?? 0,
2482
2498
  };
2483
2499
  return out;
2484
2500
  }
@@ -2801,6 +2817,19 @@ function renderDivergenceGates(divergence) {
2801
2817
  console.log(` ${(0, ui_js_1.dim)(`${divergence.candidatesConsidered} column(s) were scored (${divergence.consideredIndexed} indexed, ${divergence.consideredUnindexed} unindexed). That population is relation-probe`)}`);
2802
2818
  console.log(` ${(0, ui_js_1.dim)('and leading-index columns only: a filter column that is neither is not covered. Skip this')}`);
2803
2819
  console.log(` ${(0, ui_js_1.dim)('section with --no-plan-divergence.')}`);
2820
+ // Say whether the unindexed findings were verified, and what verification
2821
+ // removed. Statistics can only say how bad a flip would be; a plan-only EXPLAIN
2822
+ // says whether the planner can reach it at all.
2823
+ if (divergence.flipProbed === true) {
2824
+ const refuted = divergence.flipRefuted ?? 0;
2825
+ const wereRefuted = refuted === 1 ? '1 was refuted' : `${refuted} were refuted`;
2826
+ console.log(` ${(0, ui_js_1.dim)(`Every unindexed finding was put to the planner (EXPLAIN, nothing executed); ${wereRefuted}`)}`);
2827
+ console.log(` ${(0, ui_js_1.dim)('because the generic plan keeps the same sequential scan, so no flip is reachable.')}`);
2828
+ }
2829
+ else if (divergence.consideredUnindexed > 0) {
2830
+ console.log(` ${(0, ui_js_1.dim)('Unindexed findings are UNVERIFIED here: the planner probe did not run, so some may name a')}`);
2831
+ console.log(` ${(0, ui_js_1.dim)('divergence the planner would never choose.')}`);
2832
+ }
2804
2833
  }
2805
2834
  /** Round to a whole number and group it, for the divergence report's estimates. */
2806
2835
  function divInt(n) {
@@ -424,6 +424,21 @@ export interface PlanDivergenceReport {
424
424
  */
425
425
  consideredIndexed: number;
426
426
  consideredUnindexed: number;
427
+ /**
428
+ * True when every `unindexed-filter` finding was put to the planner via
429
+ * `plan-flip-probe.ts` rather than reported on statistics alone.
430
+ *
431
+ * Absent or false means the findings below are UNVERIFIED: the probe was
432
+ * skipped, the engine is not Postgres, or the connection refused it.
433
+ */
434
+ flipProbed?: boolean;
435
+ /**
436
+ * How many candidate findings the planner refuted, i.e. columns where the
437
+ * generic plan keeps the same sequential scan the good plan uses so there is
438
+ * no divergence to reach. Reported because a check that silently discards two
439
+ * thirds of what it found should say so.
440
+ */
441
+ flipRefuted?: number;
427
442
  }
428
443
  /** A (table, column) pair whose distribution statistics the collector should read. */
429
444
  export interface DivergenceCandidate {
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Plan-flip probe: does the generic plan actually differ from the good one?
3
+ *
4
+ * ## Why this exists
5
+ *
6
+ * `plan-divergence.ts` scores a column from statistics alone and answers "IF the
7
+ * cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
8
+ * without an answer to the prior question, "CAN it flip at all", and that turned
9
+ * out to be the majority case: on a real 118-model schema the branch produced 39
10
+ * findings of which a measured sample was right 6 times in 13.
11
+ *
12
+ * Every false positive had one signature: **the generic plan was not the ordered
13
+ * index walk the finding claims.** There was no flip to be had, so the
14
+ * amplification the finding printed described a plan the planner would never
15
+ * pick. See {@link verdictFromPlanJson} for the two ways a plan fails to be that
16
+ * walk; 0.58.0 shipped only one of them and 0.59.0 added the other.
17
+ *
18
+ * ## Why this is a probe and not another rule
19
+ *
20
+ * The obvious gate is arithmetic: require the generic row estimate
21
+ * (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
22
+ * Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
23
+ * estimate at or below the limit earns no discount at all.
24
+ *
25
+ * That rule is wrong, and it was measured wrong before it was written down here.
26
+ * The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
27
+ * which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
28
+ * that, and the plan is then chosen by comparing that fraction of the full index
29
+ * scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
30
+ * between estimates 3 and 4, not at the limit of 20:
31
+ *
32
+ * ```txt
33
+ * generic estimate 2 3 4 10 20 500
34
+ * generic plan seq seq index index index index
35
+ * buffers 250 250 20,074 20,074 19,071 765
36
+ * ```
37
+ *
38
+ * Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
39
+ * would discard. The boundary is also not a fixed number: it is a cost
40
+ * comparison, so it moves with the table. Reproducing the same estimate on a
41
+ * 1976-page table and on an 89-page narrow table put the flip in a different
42
+ * place each time.
43
+ *
44
+ * So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
45
+ * WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
46
+ * planner the exact question the rule was trying to predict. This module runs it.
47
+ *
48
+ * ## What it asks, and why only half the pair
49
+ *
50
+ * It plans ONE statement, under `force_generic_plan` only:
51
+ *
52
+ * ```sql
53
+ * PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
54
+ * EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
55
+ * ```
56
+ *
57
+ * The custom plan is not needed. The finding's whole claim is that a promoted
58
+ * generic plan performs an ordered index walk, so a generic plan that is NOT that
59
+ * walk refutes it no matter what the custom plan does. Asking one question
60
+ * instead of two halves the work and removes the need for a representative rare
61
+ * value, which statistics do not carry.
62
+ *
63
+ * `NULL` is a safe argument precisely because the plan is generic: a generic plan
64
+ * is built without looking at the value, which is the property the whole check is
65
+ * about. The LIMIT is bound as `$2` rather than inlined because that is the shape
66
+ * Turbine emits, and an inlined limit takes a different code path in the planner.
67
+ *
68
+ * ## Failure is never a silent drop
69
+ *
70
+ * A probe that errors, times out, or returns an unparseable plan yields
71
+ * `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
72
+ * findings when the database is uncooperative would be worse than one that
73
+ * over-reports, because the failure would be invisible in exactly the
74
+ * environments (restricted roles, non-Postgres engines) where a human is least
75
+ * able to check.
76
+ *
77
+ * @module
78
+ */
79
+ import type { PlanDivergenceFinding, PlanDivergenceReport } from './plan-divergence.js';
80
+ /**
81
+ * The planner's answer for one finding.
82
+ *
83
+ * - `'flip-reachable'`, the generic plan IS an ordered index walk on the target
84
+ * table, so the divergence the finding describes is one the planner can
85
+ * actually choose.
86
+ * - `'no-flip'`, the generic plan is not that walk (a `Sort` bounds it, or the
87
+ * access is a plain seq scan). Nothing to diverge to.
88
+ * - `'unknown'`, the probe did not produce an answer. The finding is kept.
89
+ */
90
+ export type FlipVerdict = 'flip-reachable' | 'no-flip' | 'unknown';
91
+ /** Outcome of a probe pass, keyed by {@link flipProbeKey}. */
92
+ export interface FlipProbeResult {
93
+ /** True when the probe pass ran at all (Postgres, connection succeeded). */
94
+ available: boolean;
95
+ verdicts: Record<string, FlipVerdict>;
96
+ notices: string[];
97
+ }
98
+ /** An empty result, which keeps every finding. Used when probing is off. */
99
+ export declare function emptyFlipProbeResult(): FlipProbeResult;
100
+ /**
101
+ * Map key for a (table, column) pair.
102
+ *
103
+ * `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
104
+ * literal NUL in a source file makes `grep` treat the whole file as binary, which
105
+ * is how four of them survived a release in `cli/index.ts`.
106
+ */
107
+ export declare function flipProbeKey(table: string, column: string): string;
108
+ /**
109
+ * Which findings are worth probing.
110
+ *
111
+ * `unindexed-filter` only. The `sparse-value` branch already requires an index
112
+ * that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
113
+ * that later produced 6 of 13 here, so there is no measured precision problem to
114
+ * spend a round trip on.
115
+ */
116
+ export declare function needsFlipProbe(finding: PlanDivergenceFinding): boolean;
117
+ /**
118
+ * The SQL for one probe. Pure, so the exact text is unit-testable without a
119
+ * database.
120
+ *
121
+ * Every identifier goes through {@link quoteIdent}; the only values in the
122
+ * statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
123
+ * caller as `tpf_<index>` and is never caller-controlled text.
124
+ */
125
+ export declare function buildFlipProbeSql(finding: PlanDivergenceFinding, name: string, searchSchema?: string): {
126
+ prepare: string;
127
+ explain: string;
128
+ deallocate: string;
129
+ };
130
+ /**
131
+ * Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
132
+ *
133
+ * Exported for unit tests: the plan shapes this has to classify are exactly the
134
+ * ones that are tedious to produce live.
135
+ *
136
+ * ## The question, stated exactly
137
+ *
138
+ * The finding claims the generic plan performs an ORDERED INDEX WALK along the
139
+ * `ORDER BY` column and fetches nearly every tuple before the `LIMIT` fills. So
140
+ * the refutation is not "the plan is a seq scan", it is **"the plan is not that
141
+ * ordered walk"**, and there are two ways for it not to be:
142
+ *
143
+ * 1. A `Sort` lies above the target table's scan. A sort materializes the whole
144
+ * matched set and orders it, so the cost is bounded by how many rows match,
145
+ * not by how far into the heap the ordered walk has to travel. Whatever feeds
146
+ * it (seq scan, bitmap heap scan) the catastrophic shape is absent.
147
+ * 2. The target's own scan node is a `Seq Scan`. Kept as an independent ground
148
+ * rather than folded into the first, so a hypothetical ordered seq scan with
149
+ * no sort still refutes.
150
+ *
151
+ * 0.58.0 shipped only the second ground and therefore missed every LOW-estimate
152
+ * column that has ANY usable index, because those plan as `Limit > Sort > Bitmap
153
+ * Heap Scan`. The case that surfaced it was a column carrying
154
+ * `btree (col) WHERE col IS NOT NULL`: an equality predicate implies not-null, so
155
+ * that partial index is fully usable and the plan never reaches a seq scan.
156
+ * Reproduced, and the fixture is the pair below at the same estimate:
157
+ *
158
+ * ```txt
159
+ * partial index, est 1.9 Limit > Sort > Bitmap Heap Scan <- 0.58 kept this
160
+ * no index, est 1.9 Limit > Sort > Seq Scan <- 0.58 refuted this
161
+ * either, est 500 Limit > Index Scan (no Sort) <- both keep, correctly
162
+ * ```
163
+ *
164
+ * ## Why this is not "exclude partial-index columns"
165
+ *
166
+ * Because a partial index whose predicate is NOT implied by the equality does not
167
+ * serve the query at all, and such a column produces a genuine finding: one was
168
+ * measured at 19,961x. The property that matters is whether the planner COULD
169
+ * use it, which is a proof obligation over predicates, and the plan already
170
+ * carries the answer. Reading the plan is cheaper and cannot drift from the
171
+ * planner's own implication rules.
172
+ *
173
+ * ## The safe direction is KEEP
174
+ *
175
+ * Over-refuting deletes real findings, which is invisible in the report;
176
+ * over-keeping only costs noise. So `Incremental Sort` does NOT refute: it means
177
+ * the index supplies a PREFIX of the ordering and the walk is still partly
178
+ * ordered, which is closer to the catastrophic shape than to the bounded one.
179
+ */
180
+ export declare function verdictFromPlanJson(payload: unknown, table: string): FlipVerdict;
181
+ export interface ProbePlanFlipsOptions {
182
+ connectionString: string;
183
+ schema?: string;
184
+ findings: PlanDivergenceFinding[];
185
+ statementTimeoutMs?: number;
186
+ }
187
+ /**
188
+ * Ask the planner, once per candidate finding, whether the flip is reachable.
189
+ *
190
+ * Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
191
+ * executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
192
+ * INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
193
+ * unprobeable column degrades that column's verdict to `'unknown'` and never the
194
+ * pass.
195
+ */
196
+ export declare function probePlanFlips(options: ProbePlanFlipsOptions): Promise<FlipProbeResult>;
197
+ /**
198
+ * Drop the findings the planner refuted, and record how many.
199
+ *
200
+ * Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
201
+ * contract in the module header.
202
+ */
203
+ export declare function applyFlipVerdicts(report: PlanDivergenceReport, probe: FlipProbeResult): PlanDivergenceReport;
@@ -0,0 +1,348 @@
1
+ "use strict";
2
+ /**
3
+ * Plan-flip probe: does the generic plan actually differ from the good one?
4
+ *
5
+ * ## Why this exists
6
+ *
7
+ * `plan-divergence.ts` scores a column from statistics alone and answers "IF the
8
+ * cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
9
+ * without an answer to the prior question, "CAN it flip at all", and that turned
10
+ * out to be the majority case: on a real 118-model schema the branch produced 39
11
+ * findings of which a measured sample was right 6 times in 13.
12
+ *
13
+ * Every false positive had one signature: **the generic plan was not the ordered
14
+ * index walk the finding claims.** There was no flip to be had, so the
15
+ * amplification the finding printed described a plan the planner would never
16
+ * pick. See {@link verdictFromPlanJson} for the two ways a plan fails to be that
17
+ * walk; 0.58.0 shipped only one of them and 0.59.0 added the other.
18
+ *
19
+ * ## Why this is a probe and not another rule
20
+ *
21
+ * The obvious gate is arithmetic: require the generic row estimate
22
+ * (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
23
+ * Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
24
+ * estimate at or below the limit earns no discount at all.
25
+ *
26
+ * That rule is wrong, and it was measured wrong before it was written down here.
27
+ * The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
28
+ * which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
29
+ * that, and the plan is then chosen by comparing that fraction of the full index
30
+ * scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
31
+ * between estimates 3 and 4, not at the limit of 20:
32
+ *
33
+ * ```txt
34
+ * generic estimate 2 3 4 10 20 500
35
+ * generic plan seq seq index index index index
36
+ * buffers 250 250 20,074 20,074 19,071 765
37
+ * ```
38
+ *
39
+ * Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
40
+ * would discard. The boundary is also not a fixed number: it is a cost
41
+ * comparison, so it moves with the table. Reproducing the same estimate on a
42
+ * 1976-page table and on an 89-page narrow table put the flip in a different
43
+ * place each time.
44
+ *
45
+ * So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
46
+ * WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
47
+ * planner the exact question the rule was trying to predict. This module runs it.
48
+ *
49
+ * ## What it asks, and why only half the pair
50
+ *
51
+ * It plans ONE statement, under `force_generic_plan` only:
52
+ *
53
+ * ```sql
54
+ * PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
55
+ * EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
56
+ * ```
57
+ *
58
+ * The custom plan is not needed. The finding's whole claim is that a promoted
59
+ * generic plan performs an ordered index walk, so a generic plan that is NOT that
60
+ * walk refutes it no matter what the custom plan does. Asking one question
61
+ * instead of two halves the work and removes the need for a representative rare
62
+ * value, which statistics do not carry.
63
+ *
64
+ * `NULL` is a safe argument precisely because the plan is generic: a generic plan
65
+ * is built without looking at the value, which is the property the whole check is
66
+ * about. The LIMIT is bound as `$2` rather than inlined because that is the shape
67
+ * Turbine emits, and an inlined limit takes a different code path in the planner.
68
+ *
69
+ * ## Failure is never a silent drop
70
+ *
71
+ * A probe that errors, times out, or returns an unparseable plan yields
72
+ * `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
73
+ * findings when the database is uncooperative would be worse than one that
74
+ * over-reports, because the failure would be invisible in exactly the
75
+ * environments (restricted roles, non-Postgres engines) where a human is least
76
+ * able to check.
77
+ *
78
+ * @module
79
+ */
80
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
81
+ if (k2 === undefined) k2 = k;
82
+ var desc = Object.getOwnPropertyDescriptor(m, k);
83
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
84
+ desc = { enumerable: true, get: function() { return m[k]; } };
85
+ }
86
+ Object.defineProperty(o, k2, desc);
87
+ }) : (function(o, m, k, k2) {
88
+ if (k2 === undefined) k2 = k;
89
+ o[k2] = m[k];
90
+ }));
91
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
92
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
93
+ }) : function(o, v) {
94
+ o["default"] = v;
95
+ });
96
+ var __importStar = (this && this.__importStar) || (function () {
97
+ var ownKeys = function(o) {
98
+ ownKeys = Object.getOwnPropertyNames || function (o) {
99
+ var ar = [];
100
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
101
+ return ar;
102
+ };
103
+ return ownKeys(o);
104
+ };
105
+ return function (mod) {
106
+ if (mod && mod.__esModule) return mod;
107
+ var result = {};
108
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
109
+ __setModuleDefault(result, mod);
110
+ return result;
111
+ };
112
+ })();
113
+ Object.defineProperty(exports, "__esModule", { value: true });
114
+ exports.emptyFlipProbeResult = emptyFlipProbeResult;
115
+ exports.flipProbeKey = flipProbeKey;
116
+ exports.needsFlipProbe = needsFlipProbe;
117
+ exports.buildFlipProbeSql = buildFlipProbeSql;
118
+ exports.verdictFromPlanJson = verdictFromPlanJson;
119
+ exports.probePlanFlips = probePlanFlips;
120
+ exports.applyFlipVerdicts = applyFlipVerdicts;
121
+ const utils_js_1 = require("./query/utils.js");
122
+ /** An empty result, which keeps every finding. Used when probing is off. */
123
+ function emptyFlipProbeResult() {
124
+ return { available: false, verdicts: {}, notices: [] };
125
+ }
126
+ /**
127
+ * Map key for a (table, column) pair.
128
+ *
129
+ * `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
130
+ * literal NUL in a source file makes `grep` treat the whole file as binary, which
131
+ * is how four of them survived a release in `cli/index.ts`.
132
+ */
133
+ function flipProbeKey(table, column) {
134
+ return `${table}\u0000${column}`;
135
+ }
136
+ /**
137
+ * Which findings are worth probing.
138
+ *
139
+ * `unindexed-filter` only. The `sparse-value` branch already requires an index
140
+ * that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
141
+ * that later produced 6 of 13 here, so there is no measured precision problem to
142
+ * spend a round trip on.
143
+ */
144
+ function needsFlipProbe(finding) {
145
+ return finding.branch === 'unindexed-filter';
146
+ }
147
+ /**
148
+ * The SQL for one probe. Pure, so the exact text is unit-testable without a
149
+ * database.
150
+ *
151
+ * Every identifier goes through {@link quoteIdent}; the only values in the
152
+ * statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
153
+ * caller as `tpf_<index>` and is never caller-controlled text.
154
+ */
155
+ function buildFlipProbeSql(finding, name, searchSchema) {
156
+ const rel = searchSchema ? `${(0, utils_js_1.quoteIdent)(searchSchema)}.${(0, utils_js_1.quoteIdent)(finding.table)}` : (0, utils_js_1.quoteIdent)(finding.table);
157
+ // No declared parameter types: Postgres infers both from context, which avoids
158
+ // maintaining a second pg-type mapping that could disagree with the column's
159
+ // real type and turn a diagnostic into an error.
160
+ const prepare = `PREPARE ${name} AS SELECT * FROM ${rel} ` +
161
+ `WHERE ${(0, utils_js_1.quoteIdent)(finding.column)} = $1 ` +
162
+ `ORDER BY ${(0, utils_js_1.quoteIdent)(finding.orderColumn)} LIMIT $2`;
163
+ return {
164
+ prepare,
165
+ explain: `EXPLAIN (FORMAT JSON) EXECUTE ${name}(NULL, ${Number(finding.assumedLimit)})`,
166
+ deallocate: `DEALLOCATE ${name}`,
167
+ };
168
+ }
169
+ /**
170
+ * Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
171
+ *
172
+ * Exported for unit tests: the plan shapes this has to classify are exactly the
173
+ * ones that are tedious to produce live.
174
+ *
175
+ * ## The question, stated exactly
176
+ *
177
+ * The finding claims the generic plan performs an ORDERED INDEX WALK along the
178
+ * `ORDER BY` column and fetches nearly every tuple before the `LIMIT` fills. So
179
+ * the refutation is not "the plan is a seq scan", it is **"the plan is not that
180
+ * ordered walk"**, and there are two ways for it not to be:
181
+ *
182
+ * 1. A `Sort` lies above the target table's scan. A sort materializes the whole
183
+ * matched set and orders it, so the cost is bounded by how many rows match,
184
+ * not by how far into the heap the ordered walk has to travel. Whatever feeds
185
+ * it (seq scan, bitmap heap scan) the catastrophic shape is absent.
186
+ * 2. The target's own scan node is a `Seq Scan`. Kept as an independent ground
187
+ * rather than folded into the first, so a hypothetical ordered seq scan with
188
+ * no sort still refutes.
189
+ *
190
+ * 0.58.0 shipped only the second ground and therefore missed every LOW-estimate
191
+ * column that has ANY usable index, because those plan as `Limit > Sort > Bitmap
192
+ * Heap Scan`. The case that surfaced it was a column carrying
193
+ * `btree (col) WHERE col IS NOT NULL`: an equality predicate implies not-null, so
194
+ * that partial index is fully usable and the plan never reaches a seq scan.
195
+ * Reproduced, and the fixture is the pair below at the same estimate:
196
+ *
197
+ * ```txt
198
+ * partial index, est 1.9 Limit > Sort > Bitmap Heap Scan <- 0.58 kept this
199
+ * no index, est 1.9 Limit > Sort > Seq Scan <- 0.58 refuted this
200
+ * either, est 500 Limit > Index Scan (no Sort) <- both keep, correctly
201
+ * ```
202
+ *
203
+ * ## Why this is not "exclude partial-index columns"
204
+ *
205
+ * Because a partial index whose predicate is NOT implied by the equality does not
206
+ * serve the query at all, and such a column produces a genuine finding: one was
207
+ * measured at 19,961x. The property that matters is whether the planner COULD
208
+ * use it, which is a proof obligation over predicates, and the plan already
209
+ * carries the answer. Reading the plan is cheaper and cannot drift from the
210
+ * planner's own implication rules.
211
+ *
212
+ * ## The safe direction is KEEP
213
+ *
214
+ * Over-refuting deletes real findings, which is invisible in the report;
215
+ * over-keeping only costs noise. So `Incremental Sort` does NOT refute: it means
216
+ * the index supplies a PREFIX of the ordering and the walk is still partly
217
+ * ordered, which is closer to the catastrophic shape than to the bounded one.
218
+ */
219
+ function verdictFromPlanJson(payload, table) {
220
+ const root = Array.isArray(payload) ? payload[0] : undefined;
221
+ const plan = root?.Plan;
222
+ if (!plan)
223
+ return 'unknown';
224
+ // Walk root-downward, tracking whether a full Sort sits ABOVE the target's
225
+ // scan. Depth matters: a Sort somewhere else in a larger plan says nothing
226
+ // about how this table is reached.
227
+ const search = (node, sortedAbove) => {
228
+ const type = node['Node Type'];
229
+ if (node['Relation Name'] === table && type !== undefined) {
230
+ if (sortedAbove)
231
+ return 'no-flip';
232
+ return type === 'Seq Scan' ? 'no-flip' : 'flip-reachable';
233
+ }
234
+ // 'Incremental Sort' is deliberately excluded, see the header.
235
+ const nowSorted = sortedAbove || type === 'Sort';
236
+ for (const child of node.Plans ?? []) {
237
+ const found = search(child, nowSorted);
238
+ if (found !== null)
239
+ return found;
240
+ }
241
+ return null;
242
+ };
243
+ // The target table not appearing at all should not happen for a statement that
244
+ // selects from it. Treated as unknown rather than as a refutation.
245
+ return search(plan, false) ?? 'unknown';
246
+ }
247
+ /**
248
+ * Ask the planner, once per candidate finding, whether the flip is reachable.
249
+ *
250
+ * Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
251
+ * executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
252
+ * INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
253
+ * unprobeable column degrades that column's verdict to `'unknown'` and never the
254
+ * pass.
255
+ */
256
+ async function probePlanFlips(options) {
257
+ const targets = options.findings.filter(needsFlipProbe);
258
+ const result = { available: false, verdicts: {}, notices: [] };
259
+ if (targets.length === 0) {
260
+ result.available = true;
261
+ return result;
262
+ }
263
+ const { Client } = (await Promise.resolve().then(() => __importStar(require('pg')))).default;
264
+ const client = new Client({ connectionString: options.connectionString });
265
+ try {
266
+ await client.connect();
267
+ await client.query(`SET statement_timeout = ${Number(options.statementTimeoutMs ?? 5000)}`);
268
+ // READ ONLY is belt-and-braces: EXPLAIN without ANALYZE cannot write, and the
269
+ // transaction is rolled back regardless. It costs nothing and makes the
270
+ // read-only intent checkable from a server-side log.
271
+ await client.query('BEGIN READ ONLY');
272
+ result.available = true;
273
+ for (let i = 0; i < targets.length; i++) {
274
+ const finding = targets[i];
275
+ const key = flipProbeKey(finding.table, finding.column);
276
+ const name = `tpf_${i}`;
277
+ const sql = buildFlipProbeSql(finding, name, options.schema);
278
+ try {
279
+ // A failed probe must not poison the surrounding transaction for the
280
+ // probes after it, so each one gets its own savepoint.
281
+ await client.query(`SAVEPOINT ${name}`);
282
+ await client.query(sql.prepare);
283
+ await client.query('SET LOCAL plan_cache_mode = force_generic_plan');
284
+ const res = await client.query(sql.explain);
285
+ const row = res.rows[0];
286
+ const payload = row ? Object.values(row)[0] : undefined;
287
+ result.verdicts[key] =
288
+ typeof payload === 'string'
289
+ ? verdictFromPlanJson(JSON.parse(payload), finding.table)
290
+ : verdictFromPlanJson(payload, finding.table);
291
+ await client.query(sql.deallocate);
292
+ await client.query(`RELEASE SAVEPOINT ${name}`);
293
+ }
294
+ catch (err) {
295
+ result.verdicts[key] = 'unknown';
296
+ result.notices.push(`flip probe on ${finding.table}.${finding.column} was inconclusive (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); the finding is kept`);
297
+ try {
298
+ await client.query(`ROLLBACK TO SAVEPOINT ${name}`);
299
+ }
300
+ catch {
301
+ // The transaction itself is gone; the remaining probes will each record
302
+ // their own notice and the pass still returns what it has.
303
+ }
304
+ }
305
+ }
306
+ }
307
+ catch (err) {
308
+ result.notices.push(`plan-flip probing unavailable (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); findings are reported unverified`);
309
+ }
310
+ finally {
311
+ try {
312
+ await client.query('ROLLBACK');
313
+ }
314
+ catch {
315
+ // Nothing to roll back.
316
+ }
317
+ await client.end().catch(() => { });
318
+ }
319
+ return result;
320
+ }
321
+ /**
322
+ * Drop the findings the planner refuted, and record how many.
323
+ *
324
+ * Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
325
+ * contract in the module header.
326
+ */
327
+ function applyFlipVerdicts(report, probe) {
328
+ if (!probe.available)
329
+ return report;
330
+ let refuted = 0;
331
+ const findings = report.findings.filter((f) => {
332
+ if (!needsFlipProbe(f))
333
+ return true;
334
+ const verdict = probe.verdicts[flipProbeKey(f.table, f.column)];
335
+ if (verdict === 'no-flip') {
336
+ refuted++;
337
+ return false;
338
+ }
339
+ return true;
340
+ });
341
+ return {
342
+ ...report,
343
+ findings,
344
+ flipProbed: true,
345
+ flipRefuted: refuted,
346
+ notices: [...report.notices, ...probe.notices.map((n) => ({ table: '', column: '', reason: n }))],
347
+ };
348
+ }
package/dist/cli/index.js CHANGED
@@ -33,6 +33,7 @@ import { buildCreateIndexSql, buildDropIndexSql, collectDoctorProbeIndexNames, c
33
33
  import { auditDoctorIndexes, collectStatsSnapshot, collectTableHeat, findInvalidIndexes, findRedundantIndexes, findUnusedIndexes, formatBytes, isSnapshotUsable, STATS_THRESHOLDS, scoreMissingIndex, } from '../index-stats.js';
34
34
  import { introspect } from '../introspect.js';
35
35
  import { collectDivergenceCandidateColumns, collectDivergenceOrderColumns, findPlanDivergence, PLAN_DIVERGENCE_THRESHOLDS, } from '../plan-divergence.js';
36
+ import { applyFlipVerdicts, emptyFlipProbeResult, needsFlipProbe, probePlanFlips } from '../plan-flip-probe.js';
36
37
  import { snakeToCamel } from '../schema.js';
37
38
  import { DestructivePushRefusal, schemaDiff, schemaPush } from '../schema-sql.js';
38
39
  import { configTemplate, DEFAULT_INIT_SEED_FILE, findConfigFile, loadConfigResult, looksLikeSchemaFilePath, resolveConfig, resolveSeedFile, unwrapModuleDefault, } from './config.js';
@@ -2333,9 +2334,18 @@ async function cmdDoctor(args, config) {
2333
2334
  // trustworthy stats_reset age because they normalize write counters by it;
2334
2335
  // this check reads no counter, only pg_stats, whose freshness is ANALYZE. A
2335
2336
  // cluster with a NULL stats_reset (the default) must still get the check.
2336
- const divergence = divergenceOn && snapshot.available
2337
+ const scored = divergenceOn && snapshot.available
2337
2338
  ? findPlanDivergence(schema, snapshot)
2338
2339
  : { findings: [], notices: [], candidatesConsidered: 0, consideredIndexed: 0, consideredUnindexed: 0 };
2340
+ // Statistics can say how bad a flip WOULD be; only the planner can say whether
2341
+ // it is reachable. The `unindexed-filter` branch shipped in 0.57 without that
2342
+ // question answered and was right 6 times in 13 on a real schema, so every one
2343
+ // of its findings is now put to a plan-only EXPLAIN. Nothing is executed, and a
2344
+ // probe that fails keeps its finding rather than dropping it.
2345
+ const flipProbe = divergenceOn && scored.findings.some(needsFlipProbe)
2346
+ ? await probePlanFlips({ connectionString: url, schema: config.schema, findings: scored.findings })
2347
+ : emptyFlipProbeResult();
2348
+ const divergence = applyFlipVerdicts(scored, flipProbe);
2339
2349
  if (jsonMode) {
2340
2350
  spinner?.stop();
2341
2351
  console.log(JSON.stringify(buildDoctorJson({ schema, findings, invalid, snapshot, usable, heat, subtract, divergence, args }), null, 2));
@@ -2429,6 +2439,12 @@ function buildDoctorJson(ctx) {
2429
2439
  considered: ctx.divergence.candidatesConsidered,
2430
2440
  indexed: ctx.divergence.consideredIndexed,
2431
2441
  unindexed: ctx.divergence.consideredUnindexed,
2442
+ // Whether the unindexed findings above were put to the planner, and how many
2443
+ // it refuted. `flipProbed: false` means they are statistics-only and carry
2444
+ // 0.57's precision, so a consumer can tell a verified list from an unverified
2445
+ // one instead of inferring it from the count.
2446
+ flipProbed: ctx.divergence.flipProbed === true,
2447
+ flipRefuted: ctx.divergence.flipRefuted ?? 0,
2432
2448
  };
2433
2449
  return out;
2434
2450
  }
@@ -2751,6 +2767,19 @@ function renderDivergenceGates(divergence) {
2751
2767
  console.log(` ${dim(`${divergence.candidatesConsidered} column(s) were scored (${divergence.consideredIndexed} indexed, ${divergence.consideredUnindexed} unindexed). That population is relation-probe`)}`);
2752
2768
  console.log(` ${dim('and leading-index columns only: a filter column that is neither is not covered. Skip this')}`);
2753
2769
  console.log(` ${dim('section with --no-plan-divergence.')}`);
2770
+ // Say whether the unindexed findings were verified, and what verification
2771
+ // removed. Statistics can only say how bad a flip would be; a plan-only EXPLAIN
2772
+ // says whether the planner can reach it at all.
2773
+ if (divergence.flipProbed === true) {
2774
+ const refuted = divergence.flipRefuted ?? 0;
2775
+ const wereRefuted = refuted === 1 ? '1 was refuted' : `${refuted} were refuted`;
2776
+ console.log(` ${dim(`Every unindexed finding was put to the planner (EXPLAIN, nothing executed); ${wereRefuted}`)}`);
2777
+ console.log(` ${dim('because the generic plan keeps the same sequential scan, so no flip is reachable.')}`);
2778
+ }
2779
+ else if (divergence.consideredUnindexed > 0) {
2780
+ console.log(` ${dim('Unindexed findings are UNVERIFIED here: the planner probe did not run, so some may name a')}`);
2781
+ console.log(` ${dim('divergence the planner would never choose.')}`);
2782
+ }
2754
2783
  }
2755
2784
  /** Round to a whole number and group it, for the divergence report's estimates. */
2756
2785
  function divInt(n) {
@@ -424,6 +424,21 @@ export interface PlanDivergenceReport {
424
424
  */
425
425
  consideredIndexed: number;
426
426
  consideredUnindexed: number;
427
+ /**
428
+ * True when every `unindexed-filter` finding was put to the planner via
429
+ * `plan-flip-probe.ts` rather than reported on statistics alone.
430
+ *
431
+ * Absent or false means the findings below are UNVERIFIED: the probe was
432
+ * skipped, the engine is not Postgres, or the connection refused it.
433
+ */
434
+ flipProbed?: boolean;
435
+ /**
436
+ * How many candidate findings the planner refuted, i.e. columns where the
437
+ * generic plan keeps the same sequential scan the good plan uses so there is
438
+ * no divergence to reach. Reported because a check that silently discards two
439
+ * thirds of what it found should say so.
440
+ */
441
+ flipRefuted?: number;
427
442
  }
428
443
  /** A (table, column) pair whose distribution statistics the collector should read. */
429
444
  export interface DivergenceCandidate {
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Plan-flip probe: does the generic plan actually differ from the good one?
3
+ *
4
+ * ## Why this exists
5
+ *
6
+ * `plan-divergence.ts` scores a column from statistics alone and answers "IF the
7
+ * cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
8
+ * without an answer to the prior question, "CAN it flip at all", and that turned
9
+ * out to be the majority case: on a real 118-model schema the branch produced 39
10
+ * findings of which a measured sample was right 6 times in 13.
11
+ *
12
+ * Every false positive had one signature: **the generic plan was not the ordered
13
+ * index walk the finding claims.** There was no flip to be had, so the
14
+ * amplification the finding printed described a plan the planner would never
15
+ * pick. See {@link verdictFromPlanJson} for the two ways a plan fails to be that
16
+ * walk; 0.58.0 shipped only one of them and 0.59.0 added the other.
17
+ *
18
+ * ## Why this is a probe and not another rule
19
+ *
20
+ * The obvious gate is arithmetic: require the generic row estimate
21
+ * (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
22
+ * Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
23
+ * estimate at or below the limit earns no discount at all.
24
+ *
25
+ * That rule is wrong, and it was measured wrong before it was written down here.
26
+ * The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
27
+ * which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
28
+ * that, and the plan is then chosen by comparing that fraction of the full index
29
+ * scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
30
+ * between estimates 3 and 4, not at the limit of 20:
31
+ *
32
+ * ```txt
33
+ * generic estimate 2 3 4 10 20 500
34
+ * generic plan seq seq index index index index
35
+ * buffers 250 250 20,074 20,074 19,071 765
36
+ * ```
37
+ *
38
+ * Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
39
+ * would discard. The boundary is also not a fixed number: it is a cost
40
+ * comparison, so it moves with the table. Reproducing the same estimate on a
41
+ * 1976-page table and on an 89-page narrow table put the flip in a different
42
+ * place each time.
43
+ *
44
+ * So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
45
+ * WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
46
+ * planner the exact question the rule was trying to predict. This module runs it.
47
+ *
48
+ * ## What it asks, and why only half the pair
49
+ *
50
+ * It plans ONE statement, under `force_generic_plan` only:
51
+ *
52
+ * ```sql
53
+ * PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
54
+ * EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
55
+ * ```
56
+ *
57
+ * The custom plan is not needed. The finding's whole claim is that a promoted
58
+ * generic plan performs an ordered index walk, so a generic plan that is NOT that
59
+ * walk refutes it no matter what the custom plan does. Asking one question
60
+ * instead of two halves the work and removes the need for a representative rare
61
+ * value, which statistics do not carry.
62
+ *
63
+ * `NULL` is a safe argument precisely because the plan is generic: a generic plan
64
+ * is built without looking at the value, which is the property the whole check is
65
+ * about. The LIMIT is bound as `$2` rather than inlined because that is the shape
66
+ * Turbine emits, and an inlined limit takes a different code path in the planner.
67
+ *
68
+ * ## Failure is never a silent drop
69
+ *
70
+ * A probe that errors, times out, or returns an unparseable plan yields
71
+ * `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
72
+ * findings when the database is uncooperative would be worse than one that
73
+ * over-reports, because the failure would be invisible in exactly the
74
+ * environments (restricted roles, non-Postgres engines) where a human is least
75
+ * able to check.
76
+ *
77
+ * @module
78
+ */
79
+ import type { PlanDivergenceFinding, PlanDivergenceReport } from './plan-divergence.js';
80
+ /**
81
+ * The planner's answer for one finding.
82
+ *
83
+ * - `'flip-reachable'`, the generic plan IS an ordered index walk on the target
84
+ * table, so the divergence the finding describes is one the planner can
85
+ * actually choose.
86
+ * - `'no-flip'`, the generic plan is not that walk (a `Sort` bounds it, or the
87
+ * access is a plain seq scan). Nothing to diverge to.
88
+ * - `'unknown'`, the probe did not produce an answer. The finding is kept.
89
+ */
90
+ export type FlipVerdict = 'flip-reachable' | 'no-flip' | 'unknown';
91
+ /** Outcome of a probe pass, keyed by {@link flipProbeKey}. */
92
+ export interface FlipProbeResult {
93
+ /** True when the probe pass ran at all (Postgres, connection succeeded). */
94
+ available: boolean;
95
+ verdicts: Record<string, FlipVerdict>;
96
+ notices: string[];
97
+ }
98
+ /** An empty result, which keeps every finding. Used when probing is off. */
99
+ export declare function emptyFlipProbeResult(): FlipProbeResult;
100
+ /**
101
+ * Map key for a (table, column) pair.
102
+ *
103
+ * `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
104
+ * literal NUL in a source file makes `grep` treat the whole file as binary, which
105
+ * is how four of them survived a release in `cli/index.ts`.
106
+ */
107
+ export declare function flipProbeKey(table: string, column: string): string;
108
+ /**
109
+ * Which findings are worth probing.
110
+ *
111
+ * `unindexed-filter` only. The `sparse-value` branch already requires an index
112
+ * that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
113
+ * that later produced 6 of 13 here, so there is no measured precision problem to
114
+ * spend a round trip on.
115
+ */
116
+ export declare function needsFlipProbe(finding: PlanDivergenceFinding): boolean;
117
+ /**
118
+ * The SQL for one probe. Pure, so the exact text is unit-testable without a
119
+ * database.
120
+ *
121
+ * Every identifier goes through {@link quoteIdent}; the only values in the
122
+ * statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
123
+ * caller as `tpf_<index>` and is never caller-controlled text.
124
+ */
125
+ export declare function buildFlipProbeSql(finding: PlanDivergenceFinding, name: string, searchSchema?: string): {
126
+ prepare: string;
127
+ explain: string;
128
+ deallocate: string;
129
+ };
130
+ /**
131
+ * Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
132
+ *
133
+ * Exported for unit tests: the plan shapes this has to classify are exactly the
134
+ * ones that are tedious to produce live.
135
+ *
136
+ * ## The question, stated exactly
137
+ *
138
+ * The finding claims the generic plan performs an ORDERED INDEX WALK along the
139
+ * `ORDER BY` column and fetches nearly every tuple before the `LIMIT` fills. So
140
+ * the refutation is not "the plan is a seq scan", it is **"the plan is not that
141
+ * ordered walk"**, and there are two ways for it not to be:
142
+ *
143
+ * 1. A `Sort` lies above the target table's scan. A sort materializes the whole
144
+ * matched set and orders it, so the cost is bounded by how many rows match,
145
+ * not by how far into the heap the ordered walk has to travel. Whatever feeds
146
+ * it (seq scan, bitmap heap scan) the catastrophic shape is absent.
147
+ * 2. The target's own scan node is a `Seq Scan`. Kept as an independent ground
148
+ * rather than folded into the first, so a hypothetical ordered seq scan with
149
+ * no sort still refutes.
150
+ *
151
+ * 0.58.0 shipped only the second ground and therefore missed every LOW-estimate
152
+ * column that has ANY usable index, because those plan as `Limit > Sort > Bitmap
153
+ * Heap Scan`. The case that surfaced it was a column carrying
154
+ * `btree (col) WHERE col IS NOT NULL`: an equality predicate implies not-null, so
155
+ * that partial index is fully usable and the plan never reaches a seq scan.
156
+ * Reproduced, and the fixture is the pair below at the same estimate:
157
+ *
158
+ * ```txt
159
+ * partial index, est 1.9 Limit > Sort > Bitmap Heap Scan <- 0.58 kept this
160
+ * no index, est 1.9 Limit > Sort > Seq Scan <- 0.58 refuted this
161
+ * either, est 500 Limit > Index Scan (no Sort) <- both keep, correctly
162
+ * ```
163
+ *
164
+ * ## Why this is not "exclude partial-index columns"
165
+ *
166
+ * Because a partial index whose predicate is NOT implied by the equality does not
167
+ * serve the query at all, and such a column produces a genuine finding: one was
168
+ * measured at 19,961x. The property that matters is whether the planner COULD
169
+ * use it, which is a proof obligation over predicates, and the plan already
170
+ * carries the answer. Reading the plan is cheaper and cannot drift from the
171
+ * planner's own implication rules.
172
+ *
173
+ * ## The safe direction is KEEP
174
+ *
175
+ * Over-refuting deletes real findings, which is invisible in the report;
176
+ * over-keeping only costs noise. So `Incremental Sort` does NOT refute: it means
177
+ * the index supplies a PREFIX of the ordering and the walk is still partly
178
+ * ordered, which is closer to the catastrophic shape than to the bounded one.
179
+ */
180
+ export declare function verdictFromPlanJson(payload: unknown, table: string): FlipVerdict;
181
+ export interface ProbePlanFlipsOptions {
182
+ connectionString: string;
183
+ schema?: string;
184
+ findings: PlanDivergenceFinding[];
185
+ statementTimeoutMs?: number;
186
+ }
187
+ /**
188
+ * Ask the planner, once per candidate finding, whether the flip is reachable.
189
+ *
190
+ * Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
191
+ * executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
192
+ * INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
193
+ * unprobeable column degrades that column's verdict to `'unknown'` and never the
194
+ * pass.
195
+ */
196
+ export declare function probePlanFlips(options: ProbePlanFlipsOptions): Promise<FlipProbeResult>;
197
+ /**
198
+ * Drop the findings the planner refuted, and record how many.
199
+ *
200
+ * Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
201
+ * contract in the module header.
202
+ */
203
+ export declare function applyFlipVerdicts(report: PlanDivergenceReport, probe: FlipProbeResult): PlanDivergenceReport;
@@ -0,0 +1,306 @@
1
+ /**
2
+ * Plan-flip probe: does the generic plan actually differ from the good one?
3
+ *
4
+ * ## Why this exists
5
+ *
6
+ * `plan-divergence.ts` scores a column from statistics alone and answers "IF the
7
+ * cached plan flips, how bad is it". Its `unindexed-filter` branch (0.57) shipped
8
+ * without an answer to the prior question, "CAN it flip at all", and that turned
9
+ * out to be the majority case: on a real 118-model schema the branch produced 39
10
+ * findings of which a measured sample was right 6 times in 13.
11
+ *
12
+ * Every false positive had one signature: **the generic plan was not the ordered
13
+ * index walk the finding claims.** There was no flip to be had, so the
14
+ * amplification the finding printed described a plan the planner would never
15
+ * pick. See {@link verdictFromPlanJson} for the two ways a plan fails to be that
16
+ * walk; 0.58.0 shipped only one of them and 0.59.0 added the other.
17
+ *
18
+ * ## Why this is a probe and not another rule
19
+ *
20
+ * The obvious gate is arithmetic: require the generic row estimate
21
+ * (`rows / n_distinct`) to exceed the assumed LIMIT, on the reasoning that
22
+ * Postgres discounts an ordered index walk by `min(1, limit / estimate)` and an
23
+ * estimate at or below the limit earns no discount at all.
24
+ *
25
+ * That rule is wrong, and it was measured wrong before it was written down here.
26
+ * The real limit fraction for a BOUND limit is `ceil(0.1 x estimate) / estimate`,
27
+ * which pins to 0.1 for estimates of 10 or more but RISES to `1/estimate` below
28
+ * that, and the plan is then chosen by comparing that fraction of the full index
29
+ * scan against a seq scan plus sort. On a 247-page fixture the flip boundary sits
30
+ * between estimates 3 and 4, not at the limit of 20:
31
+ *
32
+ * ```txt
33
+ * generic estimate 2 3 4 10 20 500
34
+ * generic plan seq seq index index index index
35
+ * buffers 250 250 20,074 20,074 19,071 765
36
+ * ```
37
+ *
38
+ * Estimates 4 through 20 are full-table walks that an estimate-versus-limit gate
39
+ * would discard. The boundary is also not a fixed number: it is a cost
40
+ * comparison, so it moves with the table. Reproducing the same estimate on a
41
+ * 1976-page table and on an 89-page narrow table put the flip in a different
42
+ * place each time.
43
+ *
44
+ * So the honest gate is not a better formula, it is a measurement. `EXPLAIN`
45
+ * WITHOUT `ANALYZE` executes nothing, returns in microseconds, and asks the
46
+ * planner the exact question the rule was trying to predict. This module runs it.
47
+ *
48
+ * ## What it asks, and why only half the pair
49
+ *
50
+ * It plans ONE statement, under `force_generic_plan` only:
51
+ *
52
+ * ```sql
53
+ * PREPARE p AS SELECT * FROM t WHERE col = $1 ORDER BY ord LIMIT $2;
54
+ * EXPLAIN (FORMAT JSON) EXECUTE p(NULL, 20);
55
+ * ```
56
+ *
57
+ * The custom plan is not needed. The finding's whole claim is that a promoted
58
+ * generic plan performs an ordered index walk, so a generic plan that is NOT that
59
+ * walk refutes it no matter what the custom plan does. Asking one question
60
+ * instead of two halves the work and removes the need for a representative rare
61
+ * value, which statistics do not carry.
62
+ *
63
+ * `NULL` is a safe argument precisely because the plan is generic: a generic plan
64
+ * is built without looking at the value, which is the property the whole check is
65
+ * about. The LIMIT is bound as `$2` rather than inlined because that is the shape
66
+ * Turbine emits, and an inlined limit takes a different code path in the planner.
67
+ *
68
+ * ## Failure is never a silent drop
69
+ *
70
+ * A probe that errors, times out, or returns an unparseable plan yields
71
+ * `'unknown'` and the finding SURVIVES with a note. A diagnostic that deletes
72
+ * findings when the database is uncooperative would be worse than one that
73
+ * over-reports, because the failure would be invisible in exactly the
74
+ * environments (restricted roles, non-Postgres engines) where a human is least
75
+ * able to check.
76
+ *
77
+ * @module
78
+ */
79
+ import { quoteIdent } from './query/utils.js';
80
+ /** An empty result, which keeps every finding. Used when probing is off. */
81
+ export function emptyFlipProbeResult() {
82
+ return { available: false, verdicts: {}, notices: [] };
83
+ }
84
+ /**
85
+ * Map key for a (table, column) pair.
86
+ *
87
+ * `\u0000` as the separator, written as the ESCAPE and never as a raw byte: a
88
+ * literal NUL in a source file makes `grep` treat the whole file as binary, which
89
+ * is how four of them survived a release in `cli/index.ts`.
90
+ */
91
+ export function flipProbeKey(table, column) {
92
+ return `${table}\u0000${column}`;
93
+ }
94
+ /**
95
+ * Which findings are worth probing.
96
+ *
97
+ * `unindexed-filter` only. The `sparse-value` branch already requires an index
98
+ * that serves the equality, and its 0.56 calibration was 6 of 6 on the schema
99
+ * that later produced 6 of 13 here, so there is no measured precision problem to
100
+ * spend a round trip on.
101
+ */
102
+ export function needsFlipProbe(finding) {
103
+ return finding.branch === 'unindexed-filter';
104
+ }
105
+ /**
106
+ * The SQL for one probe. Pure, so the exact text is unit-testable without a
107
+ * database.
108
+ *
109
+ * Every identifier goes through {@link quoteIdent}; the only values in the
110
+ * statement are `$1` and `$2`, bound at EXECUTE. `name` is generated by the
111
+ * caller as `tpf_<index>` and is never caller-controlled text.
112
+ */
113
+ export function buildFlipProbeSql(finding, name, searchSchema) {
114
+ const rel = searchSchema ? `${quoteIdent(searchSchema)}.${quoteIdent(finding.table)}` : quoteIdent(finding.table);
115
+ // No declared parameter types: Postgres infers both from context, which avoids
116
+ // maintaining a second pg-type mapping that could disagree with the column's
117
+ // real type and turn a diagnostic into an error.
118
+ const prepare = `PREPARE ${name} AS SELECT * FROM ${rel} ` +
119
+ `WHERE ${quoteIdent(finding.column)} = $1 ` +
120
+ `ORDER BY ${quoteIdent(finding.orderColumn)} LIMIT $2`;
121
+ return {
122
+ prepare,
123
+ explain: `EXPLAIN (FORMAT JSON) EXECUTE ${name}(NULL, ${Number(finding.assumedLimit)})`,
124
+ deallocate: `DEALLOCATE ${name}`,
125
+ };
126
+ }
127
+ /**
128
+ * Read a verdict out of one `EXPLAIN (FORMAT JSON)` payload.
129
+ *
130
+ * Exported for unit tests: the plan shapes this has to classify are exactly the
131
+ * ones that are tedious to produce live.
132
+ *
133
+ * ## The question, stated exactly
134
+ *
135
+ * The finding claims the generic plan performs an ORDERED INDEX WALK along the
136
+ * `ORDER BY` column and fetches nearly every tuple before the `LIMIT` fills. So
137
+ * the refutation is not "the plan is a seq scan", it is **"the plan is not that
138
+ * ordered walk"**, and there are two ways for it not to be:
139
+ *
140
+ * 1. A `Sort` lies above the target table's scan. A sort materializes the whole
141
+ * matched set and orders it, so the cost is bounded by how many rows match,
142
+ * not by how far into the heap the ordered walk has to travel. Whatever feeds
143
+ * it (seq scan, bitmap heap scan) the catastrophic shape is absent.
144
+ * 2. The target's own scan node is a `Seq Scan`. Kept as an independent ground
145
+ * rather than folded into the first, so a hypothetical ordered seq scan with
146
+ * no sort still refutes.
147
+ *
148
+ * 0.58.0 shipped only the second ground and therefore missed every LOW-estimate
149
+ * column that has ANY usable index, because those plan as `Limit > Sort > Bitmap
150
+ * Heap Scan`. The case that surfaced it was a column carrying
151
+ * `btree (col) WHERE col IS NOT NULL`: an equality predicate implies not-null, so
152
+ * that partial index is fully usable and the plan never reaches a seq scan.
153
+ * Reproduced, and the fixture is the pair below at the same estimate:
154
+ *
155
+ * ```txt
156
+ * partial index, est 1.9 Limit > Sort > Bitmap Heap Scan <- 0.58 kept this
157
+ * no index, est 1.9 Limit > Sort > Seq Scan <- 0.58 refuted this
158
+ * either, est 500 Limit > Index Scan (no Sort) <- both keep, correctly
159
+ * ```
160
+ *
161
+ * ## Why this is not "exclude partial-index columns"
162
+ *
163
+ * Because a partial index whose predicate is NOT implied by the equality does not
164
+ * serve the query at all, and such a column produces a genuine finding: one was
165
+ * measured at 19,961x. The property that matters is whether the planner COULD
166
+ * use it, which is a proof obligation over predicates, and the plan already
167
+ * carries the answer. Reading the plan is cheaper and cannot drift from the
168
+ * planner's own implication rules.
169
+ *
170
+ * ## The safe direction is KEEP
171
+ *
172
+ * Over-refuting deletes real findings, which is invisible in the report;
173
+ * over-keeping only costs noise. So `Incremental Sort` does NOT refute: it means
174
+ * the index supplies a PREFIX of the ordering and the walk is still partly
175
+ * ordered, which is closer to the catastrophic shape than to the bounded one.
176
+ */
177
+ export function verdictFromPlanJson(payload, table) {
178
+ const root = Array.isArray(payload) ? payload[0] : undefined;
179
+ const plan = root?.Plan;
180
+ if (!plan)
181
+ return 'unknown';
182
+ // Walk root-downward, tracking whether a full Sort sits ABOVE the target's
183
+ // scan. Depth matters: a Sort somewhere else in a larger plan says nothing
184
+ // about how this table is reached.
185
+ const search = (node, sortedAbove) => {
186
+ const type = node['Node Type'];
187
+ if (node['Relation Name'] === table && type !== undefined) {
188
+ if (sortedAbove)
189
+ return 'no-flip';
190
+ return type === 'Seq Scan' ? 'no-flip' : 'flip-reachable';
191
+ }
192
+ // 'Incremental Sort' is deliberately excluded, see the header.
193
+ const nowSorted = sortedAbove || type === 'Sort';
194
+ for (const child of node.Plans ?? []) {
195
+ const found = search(child, nowSorted);
196
+ if (found !== null)
197
+ return found;
198
+ }
199
+ return null;
200
+ };
201
+ // The target table not appearing at all should not happen for a statement that
202
+ // selects from it. Treated as unknown rather than as a refutation.
203
+ return search(plan, false) ?? 'unknown';
204
+ }
205
+ /**
206
+ * Ask the planner, once per candidate finding, whether the flip is reachable.
207
+ *
208
+ * Runs inside a single `BEGIN READ ONLY` that is always rolled back. Nothing is
209
+ * executed: `EXPLAIN` without `ANALYZE` plans and discards. Each probe is
210
+ * INDIVIDUALLY optional, the same contract `collectStatsSnapshot` uses, so one
211
+ * unprobeable column degrades that column's verdict to `'unknown'` and never the
212
+ * pass.
213
+ */
214
+ export async function probePlanFlips(options) {
215
+ const targets = options.findings.filter(needsFlipProbe);
216
+ const result = { available: false, verdicts: {}, notices: [] };
217
+ if (targets.length === 0) {
218
+ result.available = true;
219
+ return result;
220
+ }
221
+ const { Client } = (await import('pg')).default;
222
+ const client = new Client({ connectionString: options.connectionString });
223
+ try {
224
+ await client.connect();
225
+ await client.query(`SET statement_timeout = ${Number(options.statementTimeoutMs ?? 5000)}`);
226
+ // READ ONLY is belt-and-braces: EXPLAIN without ANALYZE cannot write, and the
227
+ // transaction is rolled back regardless. It costs nothing and makes the
228
+ // read-only intent checkable from a server-side log.
229
+ await client.query('BEGIN READ ONLY');
230
+ result.available = true;
231
+ for (let i = 0; i < targets.length; i++) {
232
+ const finding = targets[i];
233
+ const key = flipProbeKey(finding.table, finding.column);
234
+ const name = `tpf_${i}`;
235
+ const sql = buildFlipProbeSql(finding, name, options.schema);
236
+ try {
237
+ // A failed probe must not poison the surrounding transaction for the
238
+ // probes after it, so each one gets its own savepoint.
239
+ await client.query(`SAVEPOINT ${name}`);
240
+ await client.query(sql.prepare);
241
+ await client.query('SET LOCAL plan_cache_mode = force_generic_plan');
242
+ const res = await client.query(sql.explain);
243
+ const row = res.rows[0];
244
+ const payload = row ? Object.values(row)[0] : undefined;
245
+ result.verdicts[key] =
246
+ typeof payload === 'string'
247
+ ? verdictFromPlanJson(JSON.parse(payload), finding.table)
248
+ : verdictFromPlanJson(payload, finding.table);
249
+ await client.query(sql.deallocate);
250
+ await client.query(`RELEASE SAVEPOINT ${name}`);
251
+ }
252
+ catch (err) {
253
+ result.verdicts[key] = 'unknown';
254
+ result.notices.push(`flip probe on ${finding.table}.${finding.column} was inconclusive (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); the finding is kept`);
255
+ try {
256
+ await client.query(`ROLLBACK TO SAVEPOINT ${name}`);
257
+ }
258
+ catch {
259
+ // The transaction itself is gone; the remaining probes will each record
260
+ // their own notice and the pass still returns what it has.
261
+ }
262
+ }
263
+ }
264
+ }
265
+ catch (err) {
266
+ result.notices.push(`plan-flip probing unavailable (${err instanceof Error ? err.message.split('\n')[0] : String(err)}); findings are reported unverified`);
267
+ }
268
+ finally {
269
+ try {
270
+ await client.query('ROLLBACK');
271
+ }
272
+ catch {
273
+ // Nothing to roll back.
274
+ }
275
+ await client.end().catch(() => { });
276
+ }
277
+ return result;
278
+ }
279
+ /**
280
+ * Drop the findings the planner refuted, and record how many.
281
+ *
282
+ * Pure. `'unknown'` and a missing verdict both KEEP the finding: see the failure
283
+ * contract in the module header.
284
+ */
285
+ export function applyFlipVerdicts(report, probe) {
286
+ if (!probe.available)
287
+ return report;
288
+ let refuted = 0;
289
+ const findings = report.findings.filter((f) => {
290
+ if (!needsFlipProbe(f))
291
+ return true;
292
+ const verdict = probe.verdicts[flipProbeKey(f.table, f.column)];
293
+ if (verdict === 'no-flip') {
294
+ refuted++;
295
+ return false;
296
+ }
297
+ return true;
298
+ });
299
+ return {
300
+ ...report,
301
+ findings,
302
+ flipProbed: true,
303
+ flipRefuted: refuted,
304
+ notices: [...report.notices, ...probe.notices.map((n) => ({ table: '', column: '', reason: n }))],
305
+ };
306
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "turbine-orm",
3
- "version": "0.57.0",
3
+ "version": "0.59.0",
4
4
  "description": "Postgres-native TypeScript ORM, runs on Neon, Vercel Postgres, Cloudflare, Supabase. Streaming cursors, typed errors, single-query nested relations. One dependency, no WASM engine",
5
5
  "type": "module",
6
6
  "//exports": "Each subpath declares its types PER CONDITION. A single shared top-level \"types\" resolves to the ESM declarations for `require` too, which is TS1479 (\"is an ES module ... cannot be require()d\") for any CJS consumer on moduleResolution node16/nodenext. The require condition points at dist/cjs, which ships its own {\"type\":\"commonjs\"} package.json, so those declarations are CJS declarations. Gated in CI by publint + @arethetypeswrong/cli + a real .cts consumer typecheck (see the package-types job in ci.yml).",
@@ -126,8 +126,9 @@
126
126
  "lint:fix": "biome check --write src/",
127
127
  "format": "biome format --write src/",
128
128
  "check:error-codes": "tsx scripts/check-error-codes.ts",
129
+ "check:changelog": "node scripts/check-changelog-headings.mjs",
129
130
  "check:package": "publint --strict && attw --pack . --profile node16",
130
- "prepublishOnly": "npm run build && npm run typecheck && npm run lint && npm run test:unit && npm run check:error-codes && npm run size",
131
+ "prepublishOnly": "npm run build && npm run typecheck && npm run lint && npm run test:unit && npm run check:error-codes && npm run check:changelog && npm run size",
131
132
  "prepack": "node scripts/strip-prepare.mjs",
132
133
  "postpack": "node scripts/restore-prepare.mjs",
133
134
  "size": "size-limit",