shadowjoin 0.0.0-stage → 0.1.0

package/dist/format.js

@@ -0,0 +1,52 @@
+const pad = (s, n) => s.length >= n ? s.slice(0, n) : s + " ".repeat(n - s.length);
+const lpad = (s, n) => s.length >= n ? s : " ".repeat(n - s.length) + s;
+/** Render a report as plain text for a terminal. */
+export function formatReport(report) {
+    const out = [];
+    const n = (v) => v.toLocaleString("en-US");
+    out.push("");
+    out.push(`  left rows ${n(report.leftRows)}   right rows ${n(report.rightRows)}`);
+    out.push("");
+    out.push(`  exact join matches        ${lpad(n(report.exactMatches), 9)}   ${report.exactMatchRate}%`);
+    out.push(`  recovered by shadow join  ${lpad(n(report.recovered), 9)}   ${report.lowerBoundLossPct}%`);
+    out.push(`  still unexplained         ${lpad(n(report.unexplained), 9)}`);
+    out.push(`  reachable match rate      ${lpad(report.reachableMatchRate + "%", 9)}`);
+    out.push("");
+    out.push(`  LOWER BOUND ON LOST MATCHES: ${report.lowerBoundLossPct}% of left rows`);
+    if (report.precision !== null) {
+        out.push(`  verified against ground truth: ${n(report.verified)} correct, ${n(report.contradicted)} wrong, ${n(report.ambiguous)} ambiguous  (precision ${report.precision}%)`);
+    }
+    out.push("");
+    if (report.byRule.length) {
+        out.push(`  ${pad("rule", 54)}${lpad("alone", 8)}${lpad("ok", 7)}${lpad("wrong", 7)}`);
+        out.push("  " + "-".repeat(76));
+        for (const r of report.byRule) {
+            out.push(`  ${pad(r.label, 54)}${lpad(n(r.recovered), 8)}${lpad(n(r.verified), 7)}${lpad(n(r.contradicted), 7)}`);
+        }
+        out.push("");
+    }
+    if (report.stack.length) {
+        out.push("  rules worth adopting, in order (greedy by marginal recovery):");
+        out.push(`  ${pad("", 4)}${pad("rule", 54)}${lpad("+rows", 9)}${lpad("running", 9)}`);
+        out.push("  " + "-".repeat(76));
+        report.stack.forEach((s2, i) => {
+            out.push(`  ${pad(String(i + 1) + ".", 4)}${pad(s2.label, 54)}${lpad("+" + n(s2.marginal), 9)}${lpad(n(s2.cumulative), 9)}`);
+        });
+        out.push("");
+    }
+    if (report.samplePairs.length) {
+        out.push("  sample recoveries (verify these by hand before acting):");
+        const mark = { verified: "ok ", contradicted: "BAD", ambiguous: "?? ", unverifiable: "   " };
+        for (const s of report.samplePairs.slice(0, 12)) {
+            const multi = s.candidates > 1 ? `  [${s.candidates} candidates]` : "";
+            out.push(`    ${mark[s.verification]} ${s.leftKey}`);
+            out.push(`         -> ${s.rightKey}   (${s.rule})${multi}`);
+        }
+        out.push("");
+    }
+    for (const w of report.warnings)
+        out.push(`  ! ${w}`);
+    if (report.warnings.length)
+        out.push("");
+    return out.join("\n");
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { scan } from "./scan.js";
+export type { ScanOptions } from "./scan.js";
+export { BASE_RULES, OPTIONAL_RULES, ALL_RULE_IDS, normalize, rulesByIds, toSql } from "./normalize.js";
+export { formatReport } from "./format.js";
+export { parseCsv, rowsFromCsv, rowsFromCsvText } from "./adapters/csv.js";
+export type { CsvOptions } from "./adapters/csv.js";
+export type { Report, Row, Rule, RuleContribution, SamplePair, Verification } from "./types.js";

package/dist/index.js

@@ -0,0 +1,4 @@
+export { scan } from "./scan.js";
+export { BASE_RULES, OPTIONAL_RULES, ALL_RULE_IDS, normalize, rulesByIds, toSql } from "./normalize.js";
+export { formatReport } from "./format.js";
+export { parseCsv, rowsFromCsv, rowsFromCsvText } from "./adapters/csv.js";

package/dist/mcp.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/mcp.js

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+/**
+ * shadowjoin MCP server.
+ *
+ * Two deliberate constraints, because this points at databases:
+ *   1. There is NO tool that executes arbitrary SQL. An agent can name a
+ *      configured source plus a table and column, and nothing else.
+ *   2. No tool accepts a connection string. Credentials live in a local config
+ *      file and never enter a tool call, because tool calls land in transcripts.
+ */
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { scan } from "./scan.js";
+import { ALL_RULE_IDS, rulesByIds, toSql } from "./normalize.js";
+import { rowsFromCsv } from "./adapters/csv.js";
+import { rowsFromJson } from "./adapters/json.js";
+const CONFIG_PATH = resolve(process.env.SHADOWJOIN_CONFIG ?? "shadowjoin.config.json");
+async function loadConfig() {
+    try {
+        const parsed = JSON.parse(await readFile(CONFIG_PATH, "utf8"));
+        if (!parsed.sources || typeof parsed.sources !== "object")
+            throw new Error("missing `sources`");
+        return parsed;
+    }
+    catch (err) {
+        throw new Error(`Could not read ${CONFIG_PATH}. Create it with your named sources, e.g.\n` +
+            `{"sources":{"prod":{"type":"postgres","connectionString":"postgres://readonly@host/db"}}}\n` +
+            `(${err instanceof Error ? err.message : String(err)})`);
+    }
+}
+async function loadSide(side, config, limit) {
+    const def = config.sources[side.source];
+    if (!def) {
+        throw new Error(`Unknown source "${side.source}". Configured: ${Object.keys(config.sources).join(", ") || "(none)"}`);
+    }
+    const opts = { keyColumn: side.column, idColumn: side.idColumn, truthColumn: side.truthColumn };
+    if (def.type === "postgres") {
+        if (!side.table)
+            throw new Error(`Source "${side.source}" is a database; "table" is required.`);
+        const { rowsFromPostgres } = await import("./adapters/postgres.js");
+        return rowsFromPostgres({ connectionString: def.connectionString, table: side.table, limit, ...opts });
+    }
+    return def.type === "json" ? rowsFromJson(def.path, opts) : rowsFromCsv(def.path, opts);
+}
+const runs = new Map();
+function summarize(runId, report) {
+    return {
+        runId,
+        leftRows: report.leftRows,
+        rightRows: report.rightRows,
+        exactMatches: report.exactMatches,
+        exactMatchRate: report.exactMatchRate,
+        recovered: report.recovered,
+        lowerBoundLossPct: report.lowerBoundLossPct,
+        reachableMatchRate: report.reachableMatchRate,
+        unexplained: report.unexplained,
+        precision: report.precision,
+        verified: report.verified,
+        contradicted: report.contradicted,
+        ambiguous: report.ambiguous,
+        topRules: report.stack.slice(0, 5).map((s) => ({ rule: s.rule, marginal: s.marginal })),
+        warnings: report.warnings,
+        note: "lowerBoundLossPct is a FLOOR, not an estimate: the shadow matcher's own recall is unknown, so the true loss is at least this large. Call explain_misses and show the user real pairs before acting on it.",
+    };
+}
+const SIDE_SCHEMA = {
+    type: "object",
+    properties: {
+        source: { type: "string", description: "Name of a source from shadowjoin.config.json. Not a connection string." },
+        table: { type: "string", description: "Table name (database sources only). Plain identifier; arbitrary SQL is rejected." },
+        column: { type: "string", description: "Column holding the join key." },
+        idColumn: { type: "string", description: "Column holding a stable row id." },
+        truthColumn: { type: "string", description: "Column holding ground truth. Supply on BOTH sides to get verified precision." },
+    },
+    required: ["source", "column"],
+};
+const TOOLS = [
+    {
+        name: "scan_join",
+        description: "Measure how many matches an exact join is silently dropping. Returns a compact summary including a FLOOR on lost matches and the rules worth adopting. Read-only. Call this before changing matching logic, and again after, to prove the change helped.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                left: SIDE_SCHEMA,
+                right: SIDE_SCHEMA,
+                rules: { type: "array", items: { type: "string" }, description: `Rule ids to enable. Default all: ${ALL_RULE_IDS.join(", ")}` },
+                limit: { type: "number", description: "Max rows per side. Default 100000." },
+            },
+            required: ["left", "right"],
+        },
+    },
+    {
+        name: "explain_misses",
+        description: "Return concrete example pairs from a previous scan, with the real key values and the rule that recovered each. Use this to show a human actual evidence instead of a percentage. May return key values, so prefer redact:true when the data is sensitive.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                runId: { type: "string" },
+                rule: { type: "string", description: "Only pairs attributed to this rule." },
+                limit: { type: "number", description: "Default 20." },
+            },
+            required: ["runId"],
+        },
+    },
+    {
+        name: "suggest_normalization",
+        description: "Turn a scan into a concrete fix: the rules worth adopting in order, each as a Postgres expression and a JavaScript function ready to paste into the matching pipeline.",
+        inputSchema: {
+            type: "object",
+            properties: { runId: { type: "string" }, column: { type: "string", description: "Column name to build the SQL around. Default 'key'." } },
+            required: ["runId"],
+        },
+    },
+    {
+        name: "compare_runs",
+        description: "Compare two scans to prove whether a change to matching logic actually helped. Reports the change in recovered matches, reachable match rate and precision. Use after editing matching logic — never claim an improvement without this.",
+        inputSchema: {
+            type: "object",
+            properties: { baselineRunId: { type: "string" }, candidateRunId: { type: "string" } },
+            required: ["baselineRunId", "candidateRunId"],
+        },
+    },
+];
+const server = new Server({ name: "shadowjoin", version: "0.1.0" }, { capabilities: { tools: {} } });
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name } = request.params;
+    const args = (request.params.arguments ?? {});
+    const ok = (payload) => ({ content: [{ type: "text", text: JSON.stringify(payload, null, 2) }] });
+    try {
+        if (name === "scan_join") {
+            const config = await loadConfig();
+            const limit = typeof args.limit === "number" ? args.limit : 100_000;
+            const [left, right] = await Promise.all([
+                loadSide(args.left, config, limit),
+                loadSide(args.right, config, limit),
+            ]);
+            const report = scan(left, right, {
+                rules: Array.isArray(args.rules) && args.rules.length ? args.rules : ALL_RULE_IDS,
+                maxSamplePairs: 50,
+            });
+            const runId = randomUUID().slice(0, 8);
+            runs.set(runId, { report, left: args.left, right: args.right });
+            return ok(summarize(runId, report));
+        }
+        if (name === "explain_misses") {
+            const run = runs.get(String(args.runId));
+            if (!run)
+                throw new Error(`Unknown runId "${args.runId}". Call scan_join first.`);
+            const limit = typeof args.limit === "number" ? args.limit : 20;
+            const pairs = run.report.samplePairs
+                .filter((p) => !args.rule || p.rule === args.rule)
+                .slice(0, limit);
+            return ok({
+                runId: args.runId,
+                pairs,
+                legend: {
+                    verified: "ground truth confirms this pair",
+                    contradicted: "ground truth says this pair is WRONG — the matcher over-merged",
+                    ambiguous: "several right-hand rows collapsed onto one key; needs a tie-breaker, not a rule",
+                    unverifiable: "no ground truth available for this pair",
+                },
+            });
+        }
+        if (name === "suggest_normalization") {
+            const run = runs.get(String(args.runId));
+            if (!run)
+                throw new Error(`Unknown runId "${args.runId}". Call scan_join first.`);
+            const column = typeof args.column === "string" ? args.column : "key";
+            const chosen = run.report.stack.map((s) => s.rule);
+            const steps = run.report.stack.map((step, i) => {
+                const upto = rulesByIds(chosen.slice(0, i + 1));
+                return { ...step, sql: toSql(column, upto) };
+            });
+            return ok({
+                runId: args.runId,
+                recommended: chosen,
+                steps,
+                fullSql: toSql(column, rulesByIds(chosen)),
+                javascript: "import { normalize, rulesByIds } from 'shadowjoin';\n" +
+                    `const rules = rulesByIds(${JSON.stringify(chosen)});\n` +
+                    "const key = (value) => normalize(value, rules);",
+                caution: "Adopting a rule widens matching and can merge records that should stay separate. Re-run scan_join afterwards and compare_runs to confirm precision did not fall.",
+            });
+        }
+        if (name === "compare_runs") {
+            const a = runs.get(String(args.baselineRunId));
+            const b = runs.get(String(args.candidateRunId));
+            if (!a || !b)
+                throw new Error("Both runIds must come from scan_join calls in this session.");
+            const delta = (x, y) => Number((y - x).toFixed(2));
+            return ok({
+                baseline: summarize(String(args.baselineRunId), a.report),
+                candidate: summarize(String(args.candidateRunId), b.report),
+                delta: {
+                    exactMatches: delta(a.report.exactMatches, b.report.exactMatches),
+                    recovered: delta(a.report.recovered, b.report.recovered),
+                    reachableMatchRate: delta(a.report.reachableMatchRate, b.report.reachableMatchRate),
+                    lowerBoundLossPct: delta(a.report.lowerBoundLossPct, b.report.lowerBoundLossPct),
+                    precision: a.report.precision !== null && b.report.precision !== null ? delta(a.report.precision, b.report.precision) : null,
+                    contradicted: delta(a.report.contradicted, b.report.contradicted),
+                },
+                verdict: b.report.exactMatchRate > a.report.exactMatchRate
+                    ? "The exact join matches more rows than before."
+                    : "The exact join does not match more rows than before. If you changed the matching logic, it did not take effect.",
+            });
+        }
+        throw new Error(`Unknown tool "${name}"`);
+    }
+    catch (err) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: err instanceof Error ? err.message : String(err) }],
+        };
+    }
+});
+await server.connect(new StdioServerTransport());

package/dist/normalize.d.ts

@@ -0,0 +1,19 @@
+import type { Rule } from "./types.js";
+/**
+ * Applied to every candidate key before any optional rule. These are the
+ * transforms that are safe on essentially any text: they never merge values
+ * that a human would read as different entities.
+ */
+export declare const BASE_RULES: Rule[];
+/**
+ * Each of these is measured INDEPENDENTLY on top of BASE_RULES, so the number
+ * next to a rule is what that rule alone buys you — not an artifact of the
+ * order rules happen to run in.
+ */
+export declare const OPTIONAL_RULES: Rule[];
+export declare const ALL_RULE_IDS: string[];
+export declare function rulesByIds(ids: string[]): Rule[];
+/** Apply BASE_RULES, then the supplied rules, in order. */
+export declare function normalize(value: string, rules?: Rule[]): string;
+/** Compose the enabled rules into one Postgres expression over a column. */
+export declare function toSql(column: string, rules?: Rule[]): string;

package/dist/normalize.js

@@ -0,0 +1,102 @@
+const strip = (re) => (v) => v.replace(re, " ");
+/**
+ * Applied to every candidate key before any optional rule. These are the
+ * transforms that are safe on essentially any text: they never merge values
+ * that a human would read as different entities.
+ */
+export const BASE_RULES = [
+    { id: "casefold", label: "lowercase", apply: (v) => v.toLowerCase(), sql: (e) => `lower(${e})` },
+    {
+        id: "deaccent",
+        label: "unicode NFKD + strip diacritics",
+        apply: (v) => v.normalize("NFKD").replace(/\p{M}+/gu, ""),
+        sql: (e) => `unaccent(${e})`,
+    },
+    { id: "trim", label: "trim and collapse whitespace", apply: (v) => v.replace(/\s+/g, " ").trim(), sql: (e) => `btrim(regexp_replace(${e}, '\\s+', ' ', 'g'))` },
+];
+/**
+ * Each of these is measured INDEPENDENTLY on top of BASE_RULES, so the number
+ * next to a rule is what that rule alone buys you — not an artifact of the
+ * order rules happen to run in.
+ */
+export const OPTIONAL_RULES = [
+    {
+        id: "punctuation",
+        label: "strip punctuation",
+        apply: strip(/[^\p{L}\p{N}\s]+/gu),
+        sql: (e) => `regexp_replace(${e}, '[^[:alnum:][:space:]]+', ' ', 'g')`,
+    },
+    {
+        id: "ampersand",
+        label: "& -> and",
+        apply: (v) => v.replace(/&/g, " and "),
+        sql: (e) => `replace(${e}, '&', ' and ')`,
+    },
+    {
+        id: "leading-zeros",
+        label: "strip leading zeros from numeric tokens",
+        apply: (v) => v.replace(/\b0+(\d)/g, "$1"),
+        sql: (e) => `regexp_replace(${e}, '\\m0+(\\d)', '\\1', 'g')`,
+    },
+    {
+        id: "article",
+        label: "strip a leading article",
+        apply: (v) => v.replace(/^\s*(the|a|an)\s+/i, " "),
+        sql: (e) => `regexp_replace(${e}, '^\\s*(the|a|an)\\s+', ' ', 'i')`,
+    },
+    {
+        id: "corp-suffix",
+        label: "strip corporate suffixes (inc, corp, ltd, plc, gmbh ...)",
+        apply: strip(/\b(incorporated|inc|corporation|corp|company|co|limited|ltd|llc|llp|lp|plc|holdings?|group|gmbh|ag|sa|nv|bv|oyj|ab|as|pte|pty)\b\.?/gi),
+        sql: (e) => `regexp_replace(${e}, '\\m(incorporated|inc|corporation|corp|company|co|limited|ltd|llc|llp|lp|plc|holdings?|group|gmbh|ag|sa|nv|bv|oyj|ab|as|pte|pty)\\M\\.?', ' ', 'gi')`,
+    },
+    {
+        id: "security-boilerplate",
+        label: "strip security/listing boilerplate (common stock, ADS, class A ...)",
+        apply: strip(/\b(common stock|common shares?|ordinary shares?|american depositary (shares?|receipts?)|depositary shares?|class [a-z]\b|series [a-z]\b|when issued|new)\b/gi),
+        sql: (e) => `regexp_replace(${e}, '\\m(common stock|common shares?|ordinary shares?|american depositary (shares?|receipts?)|depositary shares?|class [a-z]\\M|series [a-z]\\M|when issued|new)\\M', ' ', 'gi')`,
+    },
+    {
+        id: "sort-tokens",
+        label: "sort tokens (order-insensitive match)",
+        apply: (v) => v.split(/\s+/).filter(Boolean).sort().join(" "),
+        sql: (e) => `(SELECT string_agg(t, ' ' ORDER BY t) FROM unnest(string_to_array(${e}, ' ')) AS t WHERE t <> '')`,
+    },
+    {
+        id: "despace",
+        label: "remove all whitespace",
+        apply: (v) => v.replace(/\s+/g, ""),
+        sql: (e) => `regexp_replace(${e}, '\\s+', '', 'g')`,
+    },
+];
+export const ALL_RULE_IDS = OPTIONAL_RULES.map((r) => r.id);
+export function rulesByIds(ids) {
+    const index = new Map(OPTIONAL_RULES.map((r) => [r.id, r]));
+    return ids.map((id) => {
+        const rule = index.get(id);
+        if (!rule) {
+            throw new Error(`Unknown rule "${id}". Known rules: ${ALL_RULE_IDS.join(", ")}`);
+        }
+        return rule;
+    });
+}
+/** Apply BASE_RULES, then the supplied rules, in order. */
+export function normalize(value, rules = []) {
+    let out = value;
+    for (const rule of BASE_RULES)
+        out = rule.apply(out);
+    for (const rule of rules)
+        out = rule.apply(out);
+    // Rules insert spaces where they remove things; tidy up so "3m  company" and
+    // "3m company" do not end up as different keys.
+    return out.replace(/\s+/g, " ").trim();
+}
+/** Compose the enabled rules into one Postgres expression over a column. */
+export function toSql(column, rules = []) {
+    let expr = column;
+    for (const rule of BASE_RULES)
+        expr = rule.sql(expr);
+    for (const rule of rules)
+        expr = rule.sql(expr);
+    return expr;
+}

package/dist/scan.d.ts

@@ -0,0 +1,17 @@
+import type { Report, Row } from "./types.js";
+export interface ScanOptions {
+    /** Rule ids to enable. Defaults to every rule. */
+    rules?: string[];
+    /** How many example recoveries to include for human review. Default 20. */
+    maxSamplePairs?: number;
+    /** Replace key values in samples with a short hash. Counts are unaffected. */
+    redactSamples?: boolean;
+}
+/**
+ * Compare an exact join against a shadow matcher and report what the exact
+ * join is losing.
+ *
+ * The figure this produces is a FLOOR. The shadow matcher has unknown recall of
+ * its own, so the true loss is at least this large and may be larger.
+ */
+export declare function scan(left: Row[], right: Row[], options?: ScanOptions): Report;

package/dist/scan.js

@@ -0,0 +1,218 @@
+import { ALL_RULE_IDS, normalize, rulesByIds } from "./normalize.js";
+/** Tickers, SKUs and ids differ cosmetically across sources; compare them loosely. */
+function canonicalTruth(value) {
+    return value.replace(/[^\p{L}\p{N}]+/gu, "").toUpperCase();
+}
+function redact(value) {
+    let h = 2166136261;
+    for (let i = 0; i < value.length; i++) {
+        h ^= value.charCodeAt(i);
+        h = Math.imul(h, 16777619);
+    }
+    return `#${(h >>> 0).toString(16).padStart(8, "0")}`;
+}
+function buildIndex(rows, rules) {
+    const index = new Map();
+    for (const row of rows) {
+        const key = rules === null ? row.key : normalize(row.key, rules);
+        if (!key)
+            continue;
+        const bucket = index.get(key);
+        if (bucket)
+            bucket.push(row);
+        else
+            index.set(key, [row]);
+    }
+    return index;
+}
+function verify(left, candidates) {
+    if (left.truth === undefined)
+        return "unverifiable";
+    const known = candidates.filter((c) => c.truth !== undefined);
+    if (known.length === 0)
+        return "unverifiable";
+    const target = canonicalTruth(left.truth);
+    const agreeing = known.filter((c) => canonicalTruth(c.truth) === target);
+    if (agreeing.length === 0)
+        return "contradicted";
+    if (agreeing.length === known.length)
+        return "verified";
+    return "ambiguous";
+}
+/**
+ * Compare an exact join against a shadow matcher and report what the exact
+ * join is losing.
+ *
+ * The figure this produces is a FLOOR. The shadow matcher has unknown recall of
+ * its own, so the true loss is at least this large and may be larger.
+ */
+export function scan(left, right, options = {}) {
+    const ruleIds = options.rules ?? ALL_RULE_IDS;
+    const rules = rulesByIds(ruleIds);
+    const maxSamples = options.maxSamplePairs ?? 20;
+    const warnings = [];
+    if (left.length === 0 || right.length === 0) {
+        warnings.push("One side of the join is empty; there is nothing to measure.");
+    }
+    // 1. The baseline: the join exactly as it runs today.
+    const exactIndex = buildIndex(right, null);
+    const missed = [];
+    let exactMatches = 0;
+    for (const row of left) {
+        if (exactIndex.has(row.key))
+            exactMatches++;
+        else
+            missed.push(row);
+    }
+    // 2. Each rule measured independently against that baseline, so the number
+    //    beside a rule is what the rule alone buys and not a product of ordering.
+    const attribution = new Map();
+    const byRule = rules.map((rule) => {
+        const index = buildIndex(right, [rule]);
+        let recovered = 0, verified = 0, contradicted = 0, ambiguous = 0;
+        for (const row of missed) {
+            const candidates = index.get(normalize(row.key, [rule]));
+            if (!candidates || candidates.length === 0)
+                continue;
+            recovered++;
+            const verdict = verify(row, candidates);
+            if (verdict === "verified")
+                verified++;
+            else if (verdict === "contradicted")
+                contradicted++;
+            else if (verdict === "ambiguous")
+                ambiguous++;
+            if (!attribution.has(row.id))
+                attribution.set(row.id, { rule: rule.id, candidates });
+        }
+        return { rule: rule.id, label: rule.label, recovered, verified, contradicted, ambiguous };
+    });
+    // 3. Greedy forward selection: which rules, in which order, actually pay.
+    //    Most recoveries need a combination, so per-rule figures alone are not
+    //    enough to act on.
+    const stack = [];
+    const covered = new Set();
+    const chosen = [];
+    let pool = [...rules];
+    while (pool.length > 0) {
+        let best = null;
+        let bestGain = 0;
+        let bestMatched = null;
+        for (const candidate of pool) {
+            const trial = [...chosen, candidate];
+            const index = buildIndex(right, trial);
+            const matched = new Set();
+            for (const row of missed) {
+                if (index.has(normalize(row.key, trial)))
+                    matched.add(row.id);
+            }
+            let gain = 0;
+            for (const id of matched)
+                if (!covered.has(id))
+                    gain++;
+            if (gain > bestGain) {
+                bestGain = gain;
+                best = candidate;
+                bestMatched = matched;
+            }
+        }
+        if (best === null || bestGain === 0)
+            break;
+        chosen.push(best);
+        for (const id of bestMatched)
+            covered.add(id);
+        stack.push({ rule: best.id, label: best.label, marginal: bestGain, cumulative: covered.size });
+        pool = pool.filter((r) => r !== best);
+    }
+    // 4. Attribute what only a COMBINATION explains. The greedy stack is tried
+    //    first, because applying every rule can recover fewer rows than the best
+    //    subset does — over-normalization merges keys that should stay apart.
+    const greedyIndex = buildIndex(right, chosen);
+    for (const row of missed) {
+        if (attribution.has(row.id))
+            continue;
+        const candidates = greedyIndex.get(normalize(row.key, chosen));
+        if (candidates && candidates.length > 0) {
+            attribution.set(row.id, { rule: "combined", candidates });
+        }
+    }
+    const fullIndex = buildIndex(right, rules);
+    let fullStackRecovered = 0;
+    for (const row of missed) {
+        const candidates = fullIndex.get(normalize(row.key, rules));
+        if (candidates && candidates.length > 0) {
+            fullStackRecovered++;
+            if (!attribution.has(row.id))
+                attribution.set(row.id, { rule: "combined", candidates });
+        }
+    }
+    // 5. Tally and collect examples.
+    let verified = 0, contradicted = 0, ambiguous = 0, unverifiable = 0;
+    const samplePairs = [];
+    const missedById = new Map(missed.map((r) => [r.id, r]));
+    for (const [leftId, { rule, candidates }] of attribution) {
+        const row = missedById.get(leftId);
+        const verdict = verify(row, candidates);
+        if (verdict === "verified")
+            verified++;
+        else if (verdict === "contradicted")
+            contradicted++;
+        else if (verdict === "ambiguous")
+            ambiguous++;
+        else
+            unverifiable++;
+        if (samplePairs.length < maxSamples) {
+            const best = candidates[0];
+            samplePairs.push({
+                rule,
+                leftId: row.id,
+                leftKey: options.redactSamples ? redact(row.key) : row.key,
+                rightId: best.id,
+                rightKey: options.redactSamples ? redact(best.key) : best.key,
+                candidates: candidates.length,
+                verification: verdict,
+            });
+        }
+    }
+    const recovered = attribution.size;
+    const unexplained = missed.length - recovered;
+    const pct = (n) => (left.length === 0 ? 0 : Number(((n / left.length) * 100).toFixed(2)));
+    if (rules.length > 1) {
+        warnings.push("Per-rule counts are measured independently against the exact baseline, so they overlap and will not sum to the total.");
+    }
+    if (chosen.length > 0 && chosen.length < rules.length && fullStackRecovered < covered.size) {
+        warnings.push(`Applying every rule recovers ${covered.size - fullStackRecovered} fewer match(es) than the best subset. Over-normalization is costing you matches — rerun with --rules ${chosen.map((r) => r.id).join(",")}`);
+    }
+    if (contradicted > 0) {
+        warnings.push(`${contradicted} recovered match(es) are contradicted by ground truth — the shadow matcher is over-merging. Review samplePairs before acting.`);
+    }
+    if (ambiguous > 0) {
+        warnings.push(`${ambiguous} recovered match(es) collapsed onto a key shared by several right-hand rows (e.g. share classes or product variants). These need a tie-breaker, not a normalization rule.`);
+    }
+    if (unverifiable > 0 && verified + contradicted === 0) {
+        warnings.push("No ground truth was supplied, so recoveries are unverified candidates. Supply a `truth` field on both sides to turn this figure into a verified floor.");
+    }
+    return {
+        generatedAt: new Date().toISOString(),
+        leftRows: left.length,
+        rightRows: right.length,
+        exactMatches,
+        exactMatchRate: pct(exactMatches),
+        recovered,
+        unexplained,
+        reachableMatchRate: pct(exactMatches + recovered),
+        lowerBoundLossPct: pct(recovered),
+        byRule: byRule.sort((a, b) => b.recovered - a.recovered),
+        stack,
+        verified,
+        contradicted,
+        ambiguous,
+        unverifiable,
+        precision: verified + contradicted === 0
+            ? null
+            : Number(((verified / (verified + contradicted)) * 100).toFixed(2)),
+        samplePairs,
+        warnings,
+        rulesApplied: ruleIds,
+    };
+}