turbine-orm 0.56.0 → 0.57.0

package/dist/plan-divergence.js

@@ -4,8 +4,31 @@
  * `index-advisor.ts` answers "which relation probes have no index" from pure
  * topology. `index-stats.ts` answers "is adding that index worth it" from live
  * statistics. This module answers the third question on the SAME columns: "this
- * column IS indexed, and its value distribution makes a NAMED prepared
- * statement's generic plan unsafe".
+ * column's value distribution makes a NAMED prepared statement's generic plan
+ * unsafe".
+ *
+ * TWO BRANCHES, one skeleton. They differ in what the CUSTOM plan does, which is
+ * what decides both the boundary and the units of the damage:
+ *
+ *   - `sparse-value`: the filter column IS served by a btree, so the custom plan
+ *     takes a bitmap scan over the rare value's own rows and the boundary is a
+ *     page comparison. This is the original rule and is unchanged.
+ *   - `unindexed-filter`: NO index can serve `col = $1`, so the custom plan's
+ *     only alternative is a seq scan + top-N sort. Added after an earlier
+ *     revision dropped every unindexed column BEFORE counting it as considered,
+ *     which put a whole class of real divergence outside the scored population
+ *     and outside the notices too. See the branch's own header below.
+ *
+ * "Served by an index" here means a VALID, non-partial, non-expression index
+ * whose leading key column is the filter column and whose access method is btree
+ * or HASH. Hash belongs in the first branch, not the second: a hash index gives
+ * the planner the same bitmap-over-this-value's-rows path a btree does, which is
+ * the plan the crossover is derived against. Measured on the fixture below with
+ * a hash index on the filter column, the custom plan is a Bitmap Heap Scan
+ * reading 7 buffers, not the 247-page seq scan the second branch would have
+ * described. Any OTHER access method leading with the column (brin, gin, gist,
+ * spgist) is not scored at all and is reported as a notice: those paths are
+ * lossy or shaped differently, and neither branch's model describes them.
  *
  * THE MECHANISM. Postgres may promote a named prepared statement to a GENERIC
  * plan from its sixth execution onward, and ONLY when the generic plan's
@@ -15,13 +38,37 @@
  *   - an unknown equality `col = $1` is estimated as reltuples / n_distinct;
  *   - an unknown `LIMIT $n` is estimated as 10% of the CHILD node's row estimate.
  *
- * THE ONE SHAPE THIS MODELS, stated narrowly on purpose. A read shaped
- * `WHERE col = $1 ORDER BY <other indexed column> LIMIT $n`, where the generic
- * estimate for `col = $1` sits ABOVE the plan boundary (so the generic plan
- * keeps the ordered index scan and filters) while some real values sit far
- * BELOW it (so for those values the ordered scan has to walk a large fraction
- * of the table before it accumulates one page of matches, where the custom
- * plan takes a bitmap scan over the value's own rows and sorts them).
+ * THE SHAPE BOTH BRANCHES MODEL, stated narrowly on purpose. A read shaped
+ * `WHERE col = $1 ORDER BY <other indexed column> LIMIT $n`, where the promoted
+ * generic plan keeps the ordered index scan and filters, while for the table's
+ * rarest values the custom planner picks something else entirely.
+ *
+ *   - `sparse-value` (the filter column is indexed): the generic estimate for
+ *     `col = $1` sits ABOVE the plan boundary while some real values sit far
+ *     BELOW it, so for those values the ordered scan walks a large fraction of
+ *     the table before it accumulates one page of matches, where the custom plan
+ *     takes a bitmap scan over the value's own rows and sorts them.
+ *   - `unindexed-filter` (no equality path on the filter column): the custom plan
+ *     takes a seq scan + top-N sort, which is bounded by the table's PAGES. The
+ *     generic plan still takes the ordered index walk and, because it cannot see
+ *     the value is rare, it walks nearly every TUPLE before it fills the LIMIT.
+ *     Measured on the fixture below: 250 buffers against 20,074, and Postgres
+ *     promoted it after five executions of the rare value.
+ *
+ * THE FIXTURE every `unindexed-filter` number in this file was measured on,
+ * printed once here so each number is checkable rather than asserted. PostgreSQL
+ * 16, warm cache, `synchronize_seqscans`, `max_parallel_workers_per_gather` and
+ * `jit` all off:
+ *
+ *   CREATE TABLE t (id int PRIMARY KEY, organization_id int NOT NULL, payload text NOT NULL);
+ *   INSERT INTO t SELECT g, <bucket(g)>, repeat('p', 60)
+ *     FROM generate_series(1, N) g ORDER BY (g * 2654435761::bigint) % 1000003;
+ *
+ * which is 81 rows per page (20,000 rows in 247 pages) and, because of the
+ * hash-ordered INSERT, a primary key UNCORRELATED with physical position. The
+ * read is `WHERE organization_id = $1 ORDER BY id LIMIT $2` at limit 20, on the
+ * rarest bucket. The same DDL is what `plan-divergence-unindexed.integration.test.ts`
+ * builds, so the ladders below can be re-run from the repo.
  *
  * WHAT THIS DELIBERATELY DOES NOT MODEL, because it was tried and it did not
  * work. An earlier revision carried a second rule for the opposite direction (a
@@ -42,6 +89,26 @@
  * shapes where a generic plan is the better one. A clean report is not evidence
  * of immunity.
  *
+ * A SECOND, STRUCTURAL blind spot bounds both branches: the candidate set is
+ * relation probes plus leading index columns. A plain filter column that is
+ * neither an FK nor indexed (a `status`, a `deleted_at`, a tenant column with no
+ * FK constraint) is invisible to this check in EITHER branch. Feeding the index
+ * advisor's recommendations in does not fix it, because the advisor derives from
+ * the same relation topology; closing it would need a workload source
+ * (pg_stat_statements or _turbine_metrics). The report says so where it prints
+ * how many columns were scored, rather than letting that count imply coverage.
+ *
+ * A THIRD blind spot, specific to `unindexed-filter` and disclosed on the
+ * finding rather than hidden: the branch cannot tell a heap that is in near-exact
+ * {@link PlanDivergenceFinding.orderColumn} order from one that is not, and the
+ * two read 1.2x and 80x on otherwise identical fixtures. The ordering column's
+ * pg_stats.correlation separates them, but only at the fifth decimal place of a
+ * sampled statistic, so it is reported and used to QUALIFY the finding
+ * ({@link PlanDivergenceFinding.heapNearlyOrdered}), never to suppress one. Its
+ * recall is bounded on the other side by a constant: gate 1 admits only a rarest
+ * bucket below the assumed LIMIT, which declines a measured 81x flip at bucket
+ * 60 (see the gate).
+ *
  * EXPOSURE IS NOT AN INCIDENT. A finding says the DISTRIBUTION admits a
  * damaging flip. It does NOT say the backend is choosing the bad plan today:
  * `auto` promotes only when the generic plan's ESTIMATED cost is not worse than
@@ -51,14 +118,22 @@
  * compares the two plans: the counter is the only thing that establishes real
  * exposure.
  *
- * TWO UNITS, and mixing them up is what made the removed rule wrong. The
- * PLANNER costs the boundary in PAGES (an ordered index scan reads about
- * limit / matching of the table's pages before it fills the limit), which is why
- * {@link crossoverRows} is derived from relpages and why measured flip points
- * track it. The DAMAGE a user feels is also read in pages here, deliberately
- * conservatively; when the ordering index is not correlated with heap order,
- * each row examined is a separate block access and the true buffer count is
- * larger, sometimes by an order of magnitude.
+ * TWO UNITS, and mixing them up is what made the removed rule wrong. In the
+ * `sparse-value` branch the PLANNER costs the boundary in PAGES (an ordered
+ * index scan reads about limit / matching of the table's pages before it fills
+ * the limit), which is why {@link crossoverRows} is derived from relpages and
+ * why measured flip points track it. The DAMAGE is also read in pages there,
+ * deliberately conservatively; when the ordering index is not correlated with
+ * heap order, each row examined is a separate block access and the true buffer
+ * count is larger, sometimes by an order of magnitude.
+ *
+ * The `unindexed-filter` branch reads damage in TUPLES on the generic side and
+ * PAGES on the custom side, because that is what the two plans actually are: an
+ * index-order heap fetch per row against one sequential pass. Carrying
+ * {@link PlanDivergenceFinding.walkPages} into it would under-report the generic
+ * side by exactly the rows-per-page factor, which is the entire finding. That is
+ * why the page-shaped fields are left UNSET on that branch instead of being
+ * filled with numbers from a model that does not apply.
  *
  * FRESHNESS GATE. The cost-tier half of doctor gates on `stats_reset` age,
  * because it normalizes WRITE COUNTERS by it. Nothing here reads a counter: every
@@ -106,17 +181,71 @@ export const PLAN_DIVERGENCE_THRESHOLDS = {
      * This replaced a `relpages >= 1000` table-size floor, which was wrong and
      * measurably so: a wrong plan on a SMALL table is not cheap, because an ordered
      * index scan's cost is driven by how much of the table it walks, not by how big
-     * the table is. On a 285-page / 18,500-row fixture the generic plan read 336
-     * buffers against the custom plan's 7 (48x), and the old floor dropped that
-     * column without even counting it as considered.
+     * the table is. Fixture, stated in full so the number is checkable:
+     *
+     *   CREATE TABLE t (id int PRIMARY KEY, organization_id int NOT NULL, payload text NOT NULL);
+     *   -- 18,500 rows, repeat('p', 85) payload => 285 relpages, buckets
+     *   -- 9,000 / 5,000 / 4,498 / 2, CREATE INDEX ON t (organization_id).
+     *   -- read: WHERE organization_id = $1 ORDER BY id LIMIT $2, rarest value, limit 20.
+     *
+     * Inserted in `id` order the generic plan read 337 buffers against the custom
+     * plan's 6 (56x); inserted in hash order, the same shape read 18,574 against 7.
+     * The old floor dropped that column without even counting it as considered.
      */
     minWalkPages: 50,
     /**
      * ...and it must walk at least this FRACTION of the table. The two gates are
      * different questions (absolute cost, and how badly the plan is mismatched to
      * the value), and a finding needs both.
+     *
+     * `sparse-value` only. In the `unindexed-filter` branch the walk is the whole
+     * table by construction (gate 1 there puts the rarest bucket below the LIMIT,
+     * so the ordered scan can never fill it), which makes this gate vacuous.
+     * Reusing it there would read as a gate that did work.
      */
     minWalkFraction: 0.1,
+    /**
+     * `unindexed-filter` only: how many TUPLES the promoted plan must walk before
+     * this is worth a user's attention.
+     *
+     * Amplification in that branch is rows-per-page and nothing else, so it is
+     * size independent and only an absolute floor separates a real finding from
+     * noise. Measured on the fixture in the module header, warm cache, rarest
+     * bucket 2, limit 20 (rows / relpages / custom buffers / custom ms / generic
+     * buffers / generic ms):
+     *
+     *   1,500 /    19 /    22 / 0.14 /   1,506 /  0.28
+     *   5,000 /    62 /    65 / 0.17 /   5,020 /  0.64
+     *  10,000 /   124 /   127 / 0.29 /  10,038 /  2.20
+     *  20,000 /   247 /   250 / 0.83 /  20,074 /  5.84
+     * 200,000 / 2,470 / 2,473 / 6.06 / 200,587 / 63.07
+     *
+     * (An earlier revision of this table carried a page column about 8% high, from
+     * a wider row than any fixture in this repo. The numbers above come from the
+     * DDL printed in the module header and nothing else.)
+     *
+     * 10,000 tuples is where the warm-cache delta reaches the "a millisecond or
+     * two" line {@link minWalkPages} is calibrated to, and where the cold-cache
+     * exposure (10,000 candidate random page accesses) stops being trivial. Not
+     * gated on pages instead: at 10,000 rows any realistic row width already
+     * clears 50 pages, and a narrow table with 10,000 rows in 40 pages is still a
+     * 250x flip.
+     */
+    minGenericTupleWalk: 10_000,
+    /**
+     * `unindexed-filter` only, and it QUALIFIES a finding rather than gating it:
+     * at or above this correlation between the heap and {@link
+     * PlanDivergenceFinding.orderColumn}, the flip is real but reads ~1x, so the
+     * finding is printed with that stated instead of with a rows-per-page ratio
+     * it would not reach.
+     *
+     * Fitted to the measured ladder on {@link
+     * PlanDivergenceFinding.orderColumnCorrelation}: 0.99998 (one page of local
+     * disorder) reads 3.1x and 0.99993 (two pages) reads 41x, so the boundary is
+     * between them. That is the fifth decimal place of a SAMPLED statistic, which
+     * is exactly why nothing is suppressed on it.
+     */
+    nearExactOrderCorrelation: 0.99995,
 };
 // ---------------------------------------------------------------------------
 // Candidate enumeration (schema-only)
@@ -136,6 +265,14 @@ function isUniqueOnColumn(meta, column) {
  * Purely schema-derived, so the collector knows what to read BEFORE any stats
  * exist. Whether the column is really served by a btree is decided later,
  * against the live index list.
+ *
+ * NOTE, because it is the obvious next idea and it is a no-op: feeding
+ * `findMissingRelationIndexes`' recommended columns in here adds nothing. Both
+ * derive from the same relation topology, this walker already enumerates every
+ * single-column relation probe whether or not it is indexed, and the advisor's
+ * only extra members are COMPOSITE probes, which this check deliberately does
+ * not model. The unindexed population was never missing from the candidate set;
+ * it was dropped later, by a shape gate. That gate is where it was fixed.
  */
 export function collectDivergenceCandidateColumns(schema) {
     const seen = new Set();
@@ -183,53 +320,126 @@ export function collectDivergenceCandidateColumns(schema) {
     }
     return out.sort((a, b) => a.table.localeCompare(b.table) || a.column.localeCompare(b.column));
 }
+/**
+ * The columns that could end up as a finding's {@link
+ * PlanDivergenceFinding.orderColumn}, so the collector reads their correlation
+ * too.
+ *
+ * A superset, deliberately: which one the generic plan would order by is decided
+ * later against the live index list, and a pg_stats read is cheap next to
+ * getting the wrong column's statistic. It is a SEPARATE list from the candidate
+ * columns because the two are filtered differently: a primary key is excluded
+ * from the candidates (a unique equality matches one row) and is the single most
+ * likely ordering column.
+ */
+export function collectDivergenceOrderColumns(schema) {
+    const seen = new Set();
+    const out = [];
+    const add = (table, column) => {
+        const meta = schema.tables[table];
+        if (!meta?.allColumns.includes(column))
+            return;
+        const key = `${table}.${column}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        out.push({ table, column });
+    };
+    const candidateTables = new Set(collectDivergenceCandidateColumns(schema).map((c) => c.table));
+    for (const table of candidateTables) {
+        const meta = schema.tables[table];
+        if (!meta)
+            continue;
+        if (meta.primaryKey[0] !== undefined)
+            add(table, meta.primaryKey[0]);
+        for (const idx of meta.indexes) {
+            if (idx.docPath)
+                continue;
+            const lead = idx.columns[0];
+            if (lead !== undefined && lead !== EXPRESSION_COLUMN)
+                add(table, lead);
+        }
+    }
+    return out.sort((a, b) => a.table.localeCompare(b.table) || a.column.localeCompare(b.column));
+}
 // ---------------------------------------------------------------------------
 // Live index shape
 // ---------------------------------------------------------------------------
-/** A plain, valid btree with no expression column and no partial predicate. */
-function isPlainBtree(idx) {
+/** Access methods that give the planner an exact bitmap over one value's rows. */
+const EQUALITY_ACCESS_METHODS = new Set(['btree', 'hash']);
+/** Valid, no expression column, no partial predicate: usable for the bare predicate. */
+function isPlainIndex(idx) {
     if (!idx.isValid)
         return false;
-    if (idx.accessMethod !== undefined && idx.accessMethod !== 'btree')
-        return false;
     if (idx.hasExpressions === true)
         return false;
     if (idx.columns.includes(EXPRESSION_COLUMN))
         return false;
     return idx.predicate == null;
 }
+/** A plain, valid btree. The only shape that can also provide ORDERING. */
+function isPlainBtree(idx) {
+    if (!isPlainIndex(idx))
+        return false;
+    return idx.accessMethod === undefined || idx.accessMethod === 'btree';
+}
 /**
- * Whether a plain btree LEADS with `column` (so the planner has an index path
- * for `column = $1`), and whether the table also has a DIFFERENT ordering index
- * the generic plan can run away with (in practice the primary key).
+ * True when the index gives the planner a path for `column = $1` whose custom
+ * plan is a bitmap over that value's own rows: a plain btree or a plain HASH
+ * index leading with the column.
  *
- * Both halves matter. With NO index on the filter column the two plans are
- * identical and equally bad, which is the EXISTING missing-index finding, not
- * this one. With no alternative ordering index there is no other plan to flip to.
+ * Hash is included on measurement, not on principle. On the module-header
+ * fixture with a hash index on the filter column, the custom plan is a Bitmap
+ * Heap Scan reading 7 buffers; calling that column "unindexed" would print a
+ * 247-page seq scan as the good plan, which is the thing being fixed.
+ */
+function servesEquality(idx, column) {
+    if (!isPlainIndex(idx))
+        return false;
+    if (idx.columns[0] !== column)
+        return false;
+    return idx.accessMethod === undefined || EQUALITY_ACCESS_METHODS.has(idx.accessMethod);
+}
+/**
+ * Whether an index serves `column = $1` (so the custom plan has a bitmap path),
+ * whether the table also has a DIFFERENT ordering index the generic plan can run
+ * away with (in practice the primary key), and whether the only thing leading
+ * with the column is an access method neither branch models.
+ *
+ * All three matter. With NO path at all the custom plan's alternative is a seq
+ * scan, which is the `unindexed-filter` branch. With no alternative ordering
+ * index there is no other plan to flip to. And with only a brin/gin/gist path
+ * the column is scored by neither rule: those scans are lossy or shaped
+ * differently, so both models would misdescribe the custom plan.
  */
 function indexShapeFor(snapshot, meta, column) {
     let hasIdx = false;
     let orderColumn = null;
+    let unmodelledAccessMethod = null;
     const pk = meta.primaryKey.length > 0 ? meta.primaryKey[0] : undefined;
     for (const idx of snapshot.indexes) {
         if (idx.table !== meta.name)
             continue;
-        if (!isPlainBtree(idx))
-            continue;
         const lead = idx.columns[0];
         if (lead === undefined)
             continue;
         if (lead === column) {
-            hasIdx = true;
+            if (servesEquality(idx, column))
+                hasIdx = true;
+            else if (idx.isValid && idx.predicate == null && idx.hasExpressions !== true && idx.accessMethod !== undefined) {
+                unmodelledAccessMethod = idx.accessMethod;
+            }
             continue;
         }
+        if (!isPlainBtree(idx))
+            continue;
         // Prefer the primary key as the stated ordering column: it is the one a
         // paginated read almost always orders by, and it is the plan the generic
         // estimate actually chose in every measured case.
         if (orderColumn === null || lead === pk)
             orderColumn = lead;
     }
-    return { hasIdx, orderColumn };
+    return { hasIdx, orderColumn, unmodelledAccessMethod: hasIdx ? null : unmodelledAccessMethod };
 }
 // ---------------------------------------------------------------------------
 // Detection
@@ -277,10 +487,17 @@ function decodeDistinct(nDistinct, rows) {
  *   3. Put the session back, so the paste does not leave `plan_cache_mode`
  *      pinned for everything the user does next.
  */
-function buildDiagnosticSql(table, column, orderColumn, columnType) {
+function buildDiagnosticSql(table, column, orderColumn, columnType, branch) {
     const t = quoteIdent(table);
     const c = quoteIdent(column);
     const o = quoteIdent(orderColumn);
+    // The `unindexed-filter` branch needs no different SQL: this PREPARE / six
+    // executions / generic_plans / force_custom / force_generic sequence is exactly
+    // what settled it on every fixture. Only the closing advice differs, because
+    // its remedy is the missing index, not a plan-cache setting.
+    const remedy = branch === 'unindexed-filter'
+        ? ['-- 3. the fix for THIS finding is the missing index above, not a plan-cache setting.']
+        : [];
     return [
         'SET synchronize_seqscans = off;',
         'SET max_parallel_workers_per_gather = 0;',
@@ -295,22 +512,60 @@ function buildDiagnosticSql(table, column, orderColumn, columnType) {
         'EXPLAIN (ANALYZE, BUFFERS) EXECUTE turbine_divergence(<your value>, 20);',
         'SET plan_cache_mode = force_generic_plan;',
         'EXPLAIN (ANALYZE, BUFFERS) EXECUTE turbine_divergence(<your value>, 20);',
-        '-- 3. put the session back:',
+        ...remedy,
+        '-- put the session back:',
         'RESET plan_cache_mode; RESET synchronize_seqscans; RESET max_parallel_workers_per_gather;',
         'DEALLOCATE turbine_divergence;',
     ].join('\n');
 }
+/**
+ * The estimated EXTRA buffer accesses the wrong plan performs, and the ONE key
+ * both branches are sorted on.
+ *
+ *   - `sparse-value`:      walkPages - min(rarestBucket, pages)
+ *   - `unindexed-filter`:  tuplesWalked - pages
+ *
+ * Both are in buffer-access units, which `approxAmplification` (the previous
+ * key) is not: branch B does not produce it, and a ratio ranks a 76x flip on a
+ * 19-page table above a 81x flip on a 2,470-page one. This DOES reorder existing
+ * `sparse-value` findings relative to 0.56.0.
+ *
+ * Shared units are NOT the same as equal conservatism, and the ordering should
+ * be read as triage rather than as a ranking of true damage. Branch A's
+ * `walkPages` is deliberately conservative (its own field doc says the true
+ * buffer count can be an order of magnitude larger), while branch B counts
+ * exactly that per-row access. Since `rows >> pages` on any table clearing
+ * branch B's tuple floor, every branch-B finding outranks every branch-A one by
+ * roughly the rows-per-page factor whether or not it is worse. In the human
+ * report most branch-B findings are attached to a missing-index finding and
+ * leave this list; in `--json` the ordering is visible, so it is stated here. That is a deliberate behavior change: the old key is documented as "a
+ * ROUGH scale only", and ordering findings by a ratio put the cheapest ones on
+ * top.
+ */
+function extraBufferAccesses(f) {
+    if (f.branch === 'unindexed-filter')
+        return (f.tuplesWalked ?? 0) - f.pages;
+    return (f.walkPages ?? 0) - Math.min(f.rarestBucket, f.pages);
+}
 /**
  * Score every candidate column against the snapshot and return the ones whose
  * distribution admits a damaging generic-plan flip.
  *
- * ONE rule, for the direction that could be calibrated: the generic estimate
- * sits ABOVE the plan boundary (so a promoted plan keeps the ordered index
- * scan) while the table's rarest values sit below it, and the pages that
- * ordered scan must walk for such a value are worth a user's attention both in
- * absolute terms and as a fraction of the table.
+ * TWO rules, split on whether a plain btree serves the filter column, because
+ * that is what decides what the CUSTOM plan does and therefore where the
+ * boundary is and what units the damage is in.
  *
- * `correlation` is reported but no longer gates anything. It used to route
+ * `sparse-value` (indexed): the generic estimate sits ABOVE the plan boundary
+ * (so a promoted plan keeps the ordered index scan) while the table's rarest
+ * values sit below it, and the pages that ordered scan must walk for such a
+ * value are worth a user's attention both in absolute terms and as a fraction of
+ * the table.
+ *
+ * `unindexed-filter` (no btree): see the block above its gates. Its boundary is
+ * linear in the LIMIT rather than sqrt in the pages, and it carries NO
+ * generic-side gate at all, for a measured reason recorded there.
+ *
+ * `correlation` is reported but gates nothing in EITHER branch. It used to route
  * clustered columns to a second rule; that rule is gone (see the module header),
  * and routing on it also made the advisor STRUCTURALLY unable to report a real
  * sparse-direction flip on any column that happened to be clustered, which was
@@ -321,6 +576,8 @@ export function findPlanDivergence(schema, snapshot) {
     const findings = [];
     const notices = [];
     let candidatesConsidered = 0;
+    let consideredIndexed = 0;
+    let consideredUnindexed = 0;
     for (const candidate of collectDivergenceCandidateColumns(schema)) {
         const meta = schema.tables[candidate.table];
         if (!meta)
@@ -334,21 +591,40 @@ export function findPlanDivergence(schema, snapshot) {
         // pg_class: either being absent is an unknown, never a zero.
         if (rows <= 0 || pages === undefined || pages <= 0)
             continue;
-        const { hasIdx, orderColumn } = indexShapeFor(snapshot, meta, candidate.column);
-        // No btree leading with the column: both plans are equally bad and the
-        // existing missing-index finding already owns this case.
-        if (!hasIdx)
+        const { hasIdx, orderColumn, unmodelledAccessMethod } = indexShapeFor(snapshot, meta, candidate.column);
+        // A brin/gin/gist path serves the equality in a way neither branch's custom
+        // plan describes. Scored by neither rule, but SAID so: silently dropping it
+        // is how the unindexed population went missing in the first place.
+        if (unmodelledAccessMethod !== null) {
+            notices.push({
+                table: candidate.table,
+                column: candidate.column,
+                reason: `served only by a ${unmodelledAccessMethod} index: this check models btree and hash equality paths, so neither rule describes the plan here`,
+            });
             continue;
+        }
         // No other ordering index: no alternative plan for the generic estimate to
-        // run away with.
+        // run away with. The ONLY shape gate left, and it applies to both branches.
+        //
+        // `hasIdx` used to be a second one, dropped here BEFORE the considered
+        // counter. That put every unindexed column outside the scored population and
+        // outside the notices too, silently, and one of the mechanisms that produces
+        // real divergence lives exactly there: with no btree on the filter column
+        // the custom plan's alternative is a seq scan, which the generic plan will
+        // not choose. It now selects the branch instead of ending the candidate.
         if (orderColumn === null)
             continue;
+        const branch = hasIdx ? 'sparse-value' : 'unindexed-filter';
         // Counted HERE, before any scored gate. Everything above is a SHAPE
         // question (is there an index to flip between at all?); everything below is
         // a verdict on live statistics. A user must be able to tell "considered and
         // clean" from "never looked", and an earlier revision incremented this after
         // a silent table-size floor, so the two were indistinguishable.
         candidatesConsidered++;
+        if (hasIdx)
+            consideredIndexed++;
+        else
+            consideredUnindexed++;
         const dist = snapshot.columnStats?.[`${candidate.table}.${candidate.column}`];
         if (!dist) {
             notices.push({
@@ -386,8 +662,93 @@ export function findPlanDivergence(schema, snapshot) {
         // below the default statistics target), the rarest MCV IS the rarest value.
         const rarestBucket = nmcv < distinctValues ? (rows * Math.max(0, 1 - freqSum)) / (distinctValues - nmcv) : Math.min(...freqs) * rows;
         const genericEstimate = rows / distinctValues;
-        const crossover = crossoverRows(pages, t.assumedLimit);
         const correlation = dist.correlation ?? 0;
+        // The ORDER column's correlation, which is the one that decides how much an
+        // `unindexed-filter` flip costs. Read opportunistically: a caller that did
+        // not ask the collector for it leaves the finding's field null rather than
+        // asserting a heap shape from a statistic it does not have.
+        const orderDist = snapshot.columnStats?.[`${candidate.table}.${orderColumn}`];
+        const orderColumnCorrelation = orderDist?.correlation ?? null;
+        // Everything common to both branches. The branch-shaped fields are filled in
+        // below and are left UNSET, never zero-filled, on the branch they do not
+        // describe.
+        const columnType = meta.pgTypes[candidate.column] ?? 'text';
+        const common = {
+            branch,
+            table: candidate.table,
+            column: candidate.column,
+            rows,
+            pages,
+            distinctValues,
+            genericEstimate,
+            rarestBucket,
+            densestBucket,
+            correlation,
+            orderColumnCorrelation,
+            assumedLimit: t.assumedLimit,
+            orderColumn,
+            columnField: meta.reverseColumnMap[candidate.column] ?? candidate.column,
+            orderColumnField: meta.reverseColumnMap[orderColumn] ?? orderColumn,
+            lastAnalyze: stats.lastAnalyze ?? null,
+            diagnosticSql: buildDiagnosticSql(candidate.table, candidate.column, orderColumn, columnType, branch),
+            thresholds: t,
+        };
+        if (branch === 'unindexed-filter') {
+            // BRANCH B. Two gates, and deliberately no third.
+            //
+            // GATE 1, rarestBucket < assumedLimit. The measured boundary at which the
+            // CUSTOM planner abandons the seq scan for the ordered walk is k x limit,
+            // with k = cost(full ordered index scan) / cost(seq scan + sort), measured
+            // 3.32 on a 267-page table and 3.245 on a 2,667-page one: the SAME
+            // boundary at 10x the size, so it is size independent, and it is linear in
+            // the limit. Gating at 1 x limit instead of the measured 3.3 x keeps this
+            // strictly inside the boundary with better than 2x margin, and it is also
+            // the regime of maximum damage: below the limit the ordered walk can never
+            // fill the LIMIT, so it walks the entire table.
+            //
+            // The recall this gives up is real and it GROWS with the table, in the
+            // same buffer-access units this module sorts on. Measured on the
+            // module-header fixture at 200,000 rows / 2,470 pages with a rarest bucket
+            // of 60: custom Seq Scan 2,473 buffers / 8.1 ms, generic PK Index Scan
+            // 200,547 buffers / 99.0 ms. That is an 81x flip and 198,074 extra buffer
+            // accesses, declined here because 60 >= 20, and it is an order of
+            // magnitude more absolute damage than the largest finding this branch DOES
+            // report on the 20,000-row fixture (19,824 extra accesses). The honest way
+            // to take it back is to derive k from pages, rows and the cost constants,
+            // not to widen the constant; until that lands it is a stated recall limit,
+            // not a safe one.
+            //
+            // GATE 2, an absolute floor on tuples walked. See minGenericTupleWalk.
+            //
+            // NO GENERIC-SIDE GATE, and that is the measured part rather than an
+            // omission. Turbine always parameterizes LIMIT on Postgres, and an unknown
+            // `LIMIT $n` is estimated as 10% of the child node's row estimate, so the
+            // generic plan's ordered index scan is discounted 10x WHATEVER the child
+            // estimate is. It therefore takes the ordered walk whenever
+            // cost(full ordered scan) < 10 x cost(seq scan + sort), i.e. whenever
+            // k < 10, and k measured 3.2 to 3.3 on every fixture, independent of
+            // n_distinct. Demonstrated directly: a 20,000-row / 267-page table with
+            // n_distinct 5,001 (generic estimate 4 rows, far below any crossover)
+            // still gave custom = Seq Scan 267 buffers and generic = PK Index Scan
+            // 19,994 buffers, a real 74x divergence that a `genericEstimate >=
+            // crossover` gate silently rejects. The generic plan's choice here is a
+            // property of the SHAPE, not of the distribution.
+            if (rarestBucket >= t.assumedLimit)
+                continue;
+            // Always exactly `rows` under gate 1; see the field's own note.
+            const tuplesWalked = rarestBucket > 0 ? Math.min(rows, (rows * t.assumedLimit) / rarestBucket) : rows;
+            if (tuplesWalked < t.minGenericTupleWalk)
+                continue;
+            findings.push({
+                ...common,
+                tuplesWalked,
+                worstCaseAmplification: tuplesWalked / pages,
+                heapNearlyOrdered: orderColumnCorrelation !== null && orderColumnCorrelation >= t.nearExactOrderCorrelation,
+            });
+            continue;
+        }
+        // BRANCH A, unchanged from 0.56.0.
+        const crossover = crossoverRows(pages, t.assumedLimit);
         // Pages the ordered index scan the generic plan keeps must walk before it
         // accumulates `assumedLimit` matches of the rarest value: limit / matching
         // of the table, and the whole table once the value has fewer rows than the
@@ -405,34 +766,18 @@ export function findPlanDivergence(schema, snapshot) {
         if (nmcv < distinctValues && rarestBucket < crossover) {
             valuesBelowCrossover += Math.round(distinctValues - nmcv);
         }
-        const columnType = meta.pgTypes[candidate.column] ?? 'text';
         findings.push({
-            table: candidate.table,
-            column: candidate.column,
-            rows,
-            pages,
-            distinctValues,
-            genericEstimate,
-            rarestBucket,
-            densestBucket,
-            correlation,
+            ...common,
             crossoverRows: crossover,
             crossoverRowsWide: crossoverRows(pages, t.wideLimit),
-            assumedLimit: t.assumedLimit,
             valuesBelowCrossover,
             walkPages,
             walkFraction,
             approxAmplification,
-            orderColumn,
-            columnField: meta.reverseColumnMap[candidate.column] ?? candidate.column,
-            orderColumnField: meta.reverseColumnMap[orderColumn] ?? orderColumn,
-            lastAnalyze: stats.lastAnalyze ?? null,
-            diagnosticSql: buildDiagnosticSql(candidate.table, candidate.column, orderColumn, columnType),
-            thresholds: t,
         });
     }
-    findings.sort((a, b) => b.approxAmplification - a.approxAmplification ||
+    findings.sort((a, b) => extraBufferAccesses(b) - extraBufferAccesses(a) ||
         a.table.localeCompare(b.table) ||
         a.column.localeCompare(b.column));
-    return { findings, notices, candidatesConsidered };
+    return { findings, notices, candidatesConsidered, consideredIndexed, consideredUnindexed };
 }