shadowjoin 0.0.0-stage → 0.3.0

package/dist/adapters/postgres.js

@@ -0,0 +1,72 @@
+const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/;
+function quoteIdent(name) {
+    const parts = name.split(".");
+    for (const part of parts) {
+        if (!IDENT.test(part)) {
+            throw new Error(`Refusing identifier "${name}". Use a plain table or column name; shadowjoin does not accept arbitrary SQL.`);
+        }
+    }
+    return parts.map((p) => `"${p}"`).join(".");
+}
+/**
+ * Read join keys out of Postgres. Opens a READ ONLY transaction with a
+ * statement and lock timeout, so it cannot write, cannot block a writer, and
+ * cannot run away with your CPU.
+ *
+ * Requires the optional peer dependency `pg`.
+ */
+export async function rowsFromPostgres(source) {
+    let pg;
+    try {
+        pg = await import("pg");
+    }
+    catch {
+        throw new Error("The Postgres adapter needs the optional peer dependency `pg`. Install it with: npm i pg");
+    }
+    const table = quoteIdent(source.table);
+    const key = quoteIdent(source.keyColumn);
+    const id = source.idColumn ? quoteIdent(source.idColumn) : null;
+    const truth = source.truthColumn ? quoteIdent(source.truthColumn) : null;
+    const limit = Math.max(1, Math.min(source.limit ?? 100_000, 5_000_000));
+    const timeout = source.statementTimeoutMs ?? 30_000;
+    const client = new pg.Client({ connectionString: source.connectionString });
+    await client.connect();
+    try {
+        if (!source.allowWritableRole) {
+            const { rows } = await client.query(`SELECT (has_table_privilege($1,'INSERT')
+              OR has_table_privilege($1,'UPDATE')
+              OR has_table_privilege($1,'DELETE')) AS writable`, [source.table]);
+            if (rows[0]?.writable) {
+                throw new Error(`The connected role can write to ${source.table}. shadowjoin only reads — point it at a read-only role (see the README) or pass allowWritableRole: true to override.`);
+            }
+        }
+        await client.query("BEGIN TRANSACTION READ ONLY");
+        await client.query(`SET LOCAL statement_timeout = ${Number(timeout)}`);
+        await client.query("SET LOCAL lock_timeout = 1000");
+        await client.query("SET LOCAL idle_in_transaction_session_timeout = 60000");
+        const select = [
+            `${key} AS key`,
+            id ? `${id}::text AS id` : `row_number() OVER ()::text AS id`,
+            truth ? `${truth}::text AS truth` : `NULL::text AS truth`,
+        ].join(", ");
+        const { rows } = await client.query(`SELECT ${select} FROM ${table} WHERE ${key} IS NOT NULL LIMIT ${limit}`);
+        await client.query("COMMIT");
+        return rows
+            .filter((r) => (r.key ?? "").trim() !== "")
+            .map((r) => ({
+            id: r.id,
+            key: String(r.key).trim(),
+            ...(r.truth ? { truth: r.truth.trim() } : {}),
+        }));
+    }
+    catch (err) {
+        try {
+            await client.query("ROLLBACK");
+        }
+        catch { /* connection may already be gone */ }
+        throw err;
+    }
+    finally {
+        await client.end();
+    }
+}

package/dist/adapters/sqlite.d.ts

@@ -0,0 +1,18 @@
+import type { Row } from "../types.js";
+export interface SqliteSource {
+    /** Path to the database file. */
+    file: string;
+    table: string;
+    keyColumn: string;
+    idColumn?: string;
+    truthColumn?: string;
+    limit?: number;
+}
+/**
+ * Read join keys out of a SQLite file. Opens the database READ ONLY.
+ *
+ * Uses Node's built-in `node:sqlite`, so there is nothing to install — but that
+ * module landed in Node 22.5, so older runtimes get a clear error rather than a
+ * confusing one.
+ */
+export declare function rowsFromSqlite(source: SqliteSource): Promise<Row[]>;

package/dist/adapters/sqlite.js

@@ -0,0 +1,49 @@
+const IDENT = /^[A-Za-z_][A-Za-z0-9_$]*$/;
+function quoteIdent(name) {
+    if (!IDENT.test(name)) {
+        throw new Error(`Refusing identifier "${name}". Use a plain table or column name; shadowjoin does not accept arbitrary SQL.`);
+    }
+    return `"${name}"`;
+}
+/**
+ * Read join keys out of a SQLite file. Opens the database READ ONLY.
+ *
+ * Uses Node's built-in `node:sqlite`, so there is nothing to install — but that
+ * module landed in Node 22.5, so older runtimes get a clear error rather than a
+ * confusing one.
+ */
+export async function rowsFromSqlite(source) {
+    let DatabaseSync;
+    try {
+        ({ DatabaseSync } = await import("node:sqlite"));
+    }
+    catch {
+        throw new Error(`The SQLite adapter needs Node's built-in node:sqlite, added in Node 22.5. This process is ${process.version}. Upgrade Node, or export the table to CSV and point shadowjoin at that.`);
+    }
+    const table = quoteIdent(source.table);
+    const key = quoteIdent(source.keyColumn);
+    const id = source.idColumn ? quoteIdent(source.idColumn) : null;
+    const truth = source.truthColumn ? quoteIdent(source.truthColumn) : null;
+    const limit = Math.max(1, Math.min(source.limit ?? 100_000, 5_000_000));
+    const db = new DatabaseSync(source.file, { readOnly: true });
+    try {
+        const select = [
+            `${key} AS key`,
+            id ? `CAST(${id} AS TEXT) AS id` : `CAST(rowid AS TEXT) AS id`,
+            truth ? `CAST(${truth} AS TEXT) AS truth` : `NULL AS truth`,
+        ].join(", ");
+        const rows = db
+            .prepare(`SELECT ${select} FROM ${table} WHERE ${key} IS NOT NULL LIMIT ${limit}`)
+            .all();
+        return rows
+            .filter((r) => String(r.key ?? "").trim() !== "")
+            .map((r) => ({
+            id: String(r.id),
+            key: String(r.key).trim(),
+            ...(r.truth ? { truth: String(r.truth).trim() } : {}),
+        }));
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+import { parseArgs } from "node:util";
+import { extname } from "node:path";
+import { scan } from "./scan.js";
+import { formatReport } from "./format.js";
+import { ALL_RULE_IDS, OPTIONAL_RULES } from "./normalize.js";
+import { rowsFromCsv } from "./adapters/csv.js";
+import { rowsFromJson } from "./adapters/json.js";
+import { banner, footer, isTty, Progress } from "./ui.js";
+import { createRequire } from "node:module";
+const VERSION = createRequire(import.meta.url)("../package.json").version;
+const USAGE = `
+shadowjoin — measure the matches your exact join is silently dropping.
+
+  npx shadowjoin scan --left <src> --left-key <col> --right <src> --right-key <col>
+
+Sources are .csv, .json (array of objects), .db/.sqlite, or a postgres:// or
+mysql:// connection string. Database sources also need --left-table/--right-table.
+
+  --left, --right              file path, or postgres:// / mysql:// URL
+  --left-key, --right-key      column holding the join key          (required)
+  --left-id, --right-id        column holding a stable id
+  --left-truth, --right-truth  column holding ground truth; supply it on BOTH
+                               sides and every recovery is verified, not guessed
+  --left-table, --right-table  table name (database sources only)
+
+  --rules <a,b,c>   limit to these rules (default: all; see \`shadowjoin rules\`)
+  --samples <n>     example recoveries to print            (default 20)
+  --redact          hash key values in samples; counts unaffected
+  --limit <n>       max rows per side                      (default 100000)
+  --json            emit the raw report as JSON
+  --max-loss <pct>  exit 1 if the lower bound exceeds this (for CI)
+
+  shadowjoin rules  list the normalization rules
+
+The figure it reports is a FLOOR. The shadow matcher's own recall is unknown,
+so the true loss is at least this large and may be larger.
+`;
+async function load(src, kind, v, limit) {
+    const keyColumn = v[`${kind}-key`];
+    if (!keyColumn)
+        throw new Error(`--${kind}-key is required`);
+    const opts = { keyColumn, idColumn: v[`${kind}-id`], truthColumn: v[`${kind}-truth`] };
+    const table = v[`${kind}-table`];
+    const needsTable = () => {
+        if (!table)
+            throw new Error(`--${kind}-table is required for a database source`);
+        return table;
+    };
+    if (src.startsWith("postgres://") || src.startsWith("postgresql://")) {
+        const { rowsFromPostgres } = await import("./adapters/postgres.js");
+        return rowsFromPostgres({ connectionString: src, table: needsTable(), limit, ...opts });
+    }
+    if (src.startsWith("mysql://")) {
+        const { rowsFromMysql } = await import("./adapters/mysql.js");
+        return rowsFromMysql({ connectionString: src, table: needsTable(), limit, ...opts });
+    }
+    const ext = extname(src).toLowerCase();
+    if (ext === ".db" || ext === ".sqlite" || ext === ".sqlite3") {
+        const { rowsFromSqlite } = await import("./adapters/sqlite.js");
+        return rowsFromSqlite({ file: src, table: needsTable(), limit, ...opts });
+    }
+    if (ext === ".json")
+        return rowsFromJson(src, opts);
+    if (ext === ".csv" || ext === ".tsv" || ext === "")
+        return rowsFromCsv(src, opts);
+    throw new Error(`Unsupported source "${src}". Use .csv, .json, .db/.sqlite, or a postgres:// or mysql:// connection string.`);
+}
+async function main() {
+    const { values, positionals } = parseArgs({
+        allowPositionals: true,
+        options: {
+            left: { type: "string" }, "left-key": { type: "string" }, "left-id": { type: "string" },
+            "left-truth": { type: "string" }, "left-table": { type: "string" },
+            right: { type: "string" }, "right-key": { type: "string" }, "right-id": { type: "string" },
+            "right-truth": { type: "string" }, "right-table": { type: "string" },
+            rules: { type: "string" }, samples: { type: "string" }, limit: { type: "string" },
+            redact: { type: "boolean" }, json: { type: "boolean" },
+            "max-loss": { type: "string" }, help: { type: "boolean", short: "h" },
+        },
+    });
+    const command = positionals[0] ?? (values.left ? "scan" : "help");
+    if (values.help || command === "help") {
+        process.stdout.write(banner(VERSION));
+        console.log(USAGE);
+        process.stdout.write(footer(VERSION));
+        return 0;
+    }
+    if (command === "rules") {
+        console.log("\n  Normalization rules (each measured independently against the exact baseline):\n");
+        for (const r of OPTIONAL_RULES)
+            console.log(`    ${r.id.padEnd(24)}${r.label}`);
+        console.log("");
+        return 0;
+    }
+    if (command !== "scan") {
+        console.error(`Unknown command "${command}".`);
+        console.log(USAGE);
+        return 2;
+    }
+    if (!values.left || !values.right) {
+        console.error("Both --left and --right are required.\n");
+        console.log(USAGE);
+        return 2;
+    }
+    const v = values;
+    const limit = values.limit ? Number(values.limit) : 100_000;
+    const quiet = Boolean(values.json) || !isTty();
+    if (!quiet)
+        process.stdout.write(banner(VERSION));
+    const progress = new Progress();
+    if (!quiet)
+        progress.start(`reading ${values.left}`);
+    let left, right;
+    try {
+        left = await load(values.left, "left", v, limit);
+        if (!quiet)
+            progress.step(`reading ${values.right}`);
+        right = await load(values.right, "right", v, limit);
+    }
+    catch (err) {
+        progress.abort();
+        throw err;
+    }
+    const report = scan(left, right, {
+        onPhase: quiet ? undefined : (label) => progress.step(label),
+        rules: values.rules ? values.rules.split(",").map((s) => s.trim()).filter(Boolean) : ALL_RULE_IDS,
+        maxSamplePairs: values.samples ? Number(values.samples) : 20,
+        redactSamples: Boolean(values.redact),
+    });
+    if (!quiet)
+        progress.stop();
+    console.log(values.json ? JSON.stringify(report, null, 2) : formatReport(report));
+    if (!quiet)
+        process.stdout.write(footer(VERSION));
+    if (values["max-loss"] !== undefined) {
+        const threshold = Number(values["max-loss"]);
+        if (report.lowerBoundLossPct > threshold) {
+            console.error(`\nFAIL: lower bound on lost matches is ${report.lowerBoundLossPct}%, above the --max-loss threshold of ${threshold}%.`);
+            return 1;
+        }
+    }
+    return 0;
+}
+main().then((code) => process.exit(code), (err) => { console.error(`\nshadowjoin: ${err instanceof Error ? err.message : String(err)}\n`); process.exit(2); });

package/dist/format.d.ts

@@ -0,0 +1,3 @@
+import type { Report } from "./types.js";
+/** Render a report as plain text for a terminal. */
+export declare function formatReport(report: Report): string;

package/dist/format.js

@@ -0,0 +1,63 @@
+const pad = (s, n) => s.length >= n ? s.slice(0, n) : s + " ".repeat(n - s.length);
+const lpad = (s, n) => s.length >= n ? s : " ".repeat(n - s.length) + s;
+/** Render a report as plain text for a terminal. */
+export function formatReport(report) {
+    const out = [];
+    const n = (v) => v.toLocaleString("en-US");
+    out.push("");
+    out.push(`  left rows ${n(report.leftRows)}   right rows ${n(report.rightRows)}`);
+    out.push("");
+    out.push(`  exact join matches        ${lpad(n(report.exactMatches), 9)}   ${report.exactMatchRate}%`);
+    out.push(`  recovered by shadow join  ${lpad(n(report.recovered), 9)}   ${report.lowerBoundLossPct}%`);
+    out.push(`  still unexplained         ${lpad(n(report.unexplained), 9)}`);
+    out.push(`  reachable match rate      ${lpad(report.reachableMatchRate + "%", 9)}`);
+    out.push("");
+    out.push(`  LOWER BOUND ON LOST MATCHES: ${report.lowerBoundLossPct}% of left rows`);
+    if (report.precision !== null) {
+        out.push(`  verified against ground truth: ${n(report.verified)} correct, ${n(report.contradicted)} wrong, ${n(report.ambiguous)} ambiguous  (precision ${report.precision}%)`);
+    }
+    out.push("");
+    if (report.byRule.length) {
+        out.push(`  ${pad("rule", 54)}${lpad("alone", 8)}${lpad("ok", 7)}${lpad("wrong", 7)}`);
+        out.push("  " + "-".repeat(76));
+        for (const r of report.byRule) {
+            out.push(`  ${pad(r.label, 54)}${lpad(n(r.recovered), 8)}${lpad(n(r.verified), 7)}${lpad(n(r.contradicted), 7)}`);
+        }
+        out.push("");
+    }
+    if (report.stack.length) {
+        out.push("  rules worth adopting, in order (greedy by marginal recovery):");
+        out.push(`  ${pad("", 4)}${pad("rule", 54)}${lpad("+rows", 9)}${lpad("running", 9)}`);
+        out.push("  " + "-".repeat(76));
+        report.stack.forEach((s2, i) => {
+            out.push(`  ${pad(String(i + 1) + ".", 4)}${pad(s2.label, 54)}${lpad("+" + n(s2.marginal), 9)}${lpad(n(s2.cumulative), 9)}`);
+        });
+        out.push("");
+    }
+    if (report.samplePairs.length) {
+        out.push("  sample recoveries (verify these by hand before acting):");
+        const mark = { verified: "ok ", contradicted: "BAD", ambiguous: "?? ", unverifiable: "   " };
+        for (const s of report.samplePairs.slice(0, 12)) {
+            const multi = s.candidates > 1 ? `  [${s.candidates} candidates]` : "";
+            out.push(`    ${mark[s.verification]} ${s.leftKey}`);
+            out.push(`         -> ${s.rightKey}   (${s.rule})${multi}`);
+        }
+        out.push("");
+    }
+    if (report.nearMisses.length) {
+        out.push("  closest neighbours among rows NO RULE explains (suggestions, not counted):");
+        const mark = { verified: "ok ", contradicted: "BAD", ambiguous: "?? ", unverifiable: "   " };
+        for (const nm of report.nearMisses.slice(0, 10)) {
+            out.push(`    ${mark[nm.verification]} ${pad(nm.leftKey, 34)} ~  ${pad(nm.rightKey, 38)} ${nm.similarity}`);
+        }
+        if (report.noCandidate > 0) {
+            out.push(`    ${report.noCandidate} more have no string similarity at all — those need an alias table.`);
+        }
+        out.push("");
+    }
+    for (const w of report.warnings)
+        out.push(`  ! ${w}`);
+    if (report.warnings.length)
+        out.push("");
+    return out.join("\n");
+}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { scan } from "./scan.js";
+export type { ScanOptions } from "./scan.js";
+export { BASE_RULES, OPTIONAL_RULES, ALL_RULE_IDS, normalize, rulesByIds, toSql } from "./normalize.js";
+export { formatReport } from "./format.js";
+export { parseCsv, rowsFromCsv, rowsFromCsvText } from "./adapters/csv.js";
+export type { CsvOptions } from "./adapters/csv.js";
+export { trigrams, dice, TrigramIndex } from "./similarity.js";
+export type { NearMiss, Report, Row, Rule, RuleContribution, SamplePair, StackStep, Verification } from "./types.js";

package/dist/index.js

@@ -0,0 +1,5 @@
+export { scan } from "./scan.js";
+export { BASE_RULES, OPTIONAL_RULES, ALL_RULE_IDS, normalize, rulesByIds, toSql } from "./normalize.js";
+export { formatReport } from "./format.js";
+export { parseCsv, rowsFromCsv, rowsFromCsvText } from "./adapters/csv.js";
+export { trigrams, dice, TrigramIndex } from "./similarity.js";

package/dist/mcp.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/mcp.js

@@ -0,0 +1,229 @@
+#!/usr/bin/env node
+/**
+ * shadowjoin MCP server.
+ *
+ * Two deliberate constraints, because this points at databases:
+ *   1. There is NO tool that executes arbitrary SQL. An agent can name a
+ *      configured source plus a table and column, and nothing else.
+ *   2. No tool accepts a connection string. Credentials live in a local config
+ *      file and never enter a tool call, because tool calls land in transcripts.
+ */
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { scan } from "./scan.js";
+import { ALL_RULE_IDS, rulesByIds, toSql } from "./normalize.js";
+import { rowsFromCsv } from "./adapters/csv.js";
+import { rowsFromJson } from "./adapters/json.js";
+const CONFIG_PATH = resolve(process.env.SHADOWJOIN_CONFIG ?? "shadowjoin.config.json");
+async function loadConfig() {
+    try {
+        const parsed = JSON.parse(await readFile(CONFIG_PATH, "utf8"));
+        if (!parsed.sources || typeof parsed.sources !== "object")
+            throw new Error("missing `sources`");
+        return parsed;
+    }
+    catch (err) {
+        throw new Error(`Could not read ${CONFIG_PATH}. Create it with your named sources, e.g.\n` +
+            `{"sources":{"prod":{"type":"postgres","connectionString":"postgres://readonly@host/db"}}}\n` +
+            `(${err instanceof Error ? err.message : String(err)})`);
+    }
+}
+async function loadSide(side, config, limit) {
+    const def = config.sources[side.source];
+    if (!def) {
+        throw new Error(`Unknown source "${side.source}". Configured: ${Object.keys(config.sources).join(", ") || "(none)"}`);
+    }
+    const opts = { keyColumn: side.column, idColumn: side.idColumn, truthColumn: side.truthColumn };
+    if (def.type === "postgres" || def.type === "mysql" || def.type === "sqlite") {
+        if (!side.table)
+            throw new Error(`Source "${side.source}" is a database; "table" is required.`);
+        if (def.type === "postgres") {
+            const { rowsFromPostgres } = await import("./adapters/postgres.js");
+            return rowsFromPostgres({ connectionString: def.connectionString, table: side.table, limit, ...opts });
+        }
+        if (def.type === "mysql") {
+            const { rowsFromMysql } = await import("./adapters/mysql.js");
+            return rowsFromMysql({ connectionString: def.connectionString, table: side.table, limit, ...opts });
+        }
+        const { rowsFromSqlite } = await import("./adapters/sqlite.js");
+        return rowsFromSqlite({ file: def.path, table: side.table, limit, ...opts });
+    }
+    return def.type === "json" ? rowsFromJson(def.path, opts) : rowsFromCsv(def.path, opts);
+}
+const runs = new Map();
+function summarize(runId, report) {
+    return {
+        runId,
+        leftRows: report.leftRows,
+        rightRows: report.rightRows,
+        exactMatches: report.exactMatches,
+        exactMatchRate: report.exactMatchRate,
+        recovered: report.recovered,
+        lowerBoundLossPct: report.lowerBoundLossPct,
+        reachableMatchRate: report.reachableMatchRate,
+        unexplained: report.unexplained,
+        precision: report.precision,
+        verified: report.verified,
+        contradicted: report.contradicted,
+        ambiguous: report.ambiguous,
+        topRules: report.stack.slice(0, 5).map((s) => ({ rule: s.rule, marginal: s.marginal })),
+        warnings: report.warnings,
+        note: "lowerBoundLossPct is a FLOOR, not an estimate: the shadow matcher's own recall is unknown, so the true loss is at least this large. Call explain_misses and show the user real pairs before acting on it.",
+    };
+}
+const SIDE_SCHEMA = {
+    type: "object",
+    properties: {
+        source: { type: "string", description: "Name of a source from shadowjoin.config.json. Not a connection string." },
+        table: { type: "string", description: "Table name (database sources only). Plain identifier; arbitrary SQL is rejected." },
+        column: { type: "string", description: "Column holding the join key." },
+        idColumn: { type: "string", description: "Column holding a stable row id." },
+        truthColumn: { type: "string", description: "Column holding ground truth. Supply on BOTH sides to get verified precision." },
+    },
+    required: ["source", "column"],
+};
+const TOOLS = [
+    {
+        name: "scan_join",
+        description: "Measure how many matches an exact join is silently dropping. Returns a compact summary including a FLOOR on lost matches and the rules worth adopting. Read-only. Call this before changing matching logic, and again after, to prove the change helped.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                left: SIDE_SCHEMA,
+                right: SIDE_SCHEMA,
+                rules: { type: "array", items: { type: "string" }, description: `Rule ids to enable. Default all: ${ALL_RULE_IDS.join(", ")}` },
+                limit: { type: "number", description: "Max rows per side. Default 100000." },
+            },
+            required: ["left", "right"],
+        },
+    },
+    {
+        name: "explain_misses",
+        description: "Return concrete example pairs from a previous scan, with the real key values and the rule that recovered each. Use this to show a human actual evidence instead of a percentage. May return key values, so prefer redact:true when the data is sensitive.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                runId: { type: "string" },
+                rule: { type: "string", description: "Only pairs attributed to this rule." },
+                limit: { type: "number", description: "Default 20." },
+            },
+            required: ["runId"],
+        },
+    },
+    {
+        name: "suggest_normalization",
+        description: "Turn a scan into a concrete fix: the rules worth adopting in order, each as a Postgres expression and a JavaScript function ready to paste into the matching pipeline.",
+        inputSchema: {
+            type: "object",
+            properties: { runId: { type: "string" }, column: { type: "string", description: "Column name to build the SQL around. Default 'key'." } },
+            required: ["runId"],
+        },
+    },
+    {
+        name: "compare_runs",
+        description: "Compare two scans to prove whether a change to matching logic actually helped. Reports the change in recovered matches, reachable match rate and precision. Use after editing matching logic — never claim an improvement without this.",
+        inputSchema: {
+            type: "object",
+            properties: { baselineRunId: { type: "string" }, candidateRunId: { type: "string" } },
+            required: ["baselineRunId", "candidateRunId"],
+        },
+    },
+];
+const server = new Server({ name: "shadowjoin", version: "0.1.0" }, { capabilities: { tools: {} } });
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name } = request.params;
+    const args = (request.params.arguments ?? {});
+    const ok = (payload) => ({ content: [{ type: "text", text: JSON.stringify(payload, null, 2) }] });
+    try {
+        if (name === "scan_join") {
+            const config = await loadConfig();
+            const limit = typeof args.limit === "number" ? args.limit : 100_000;
+            const [left, right] = await Promise.all([
+                loadSide(args.left, config, limit),
+                loadSide(args.right, config, limit),
+            ]);
+            const report = scan(left, right, {
+                rules: Array.isArray(args.rules) && args.rules.length ? args.rules : ALL_RULE_IDS,
+                maxSamplePairs: 50,
+            });
+            const runId = randomUUID().slice(0, 8);
+            runs.set(runId, { report, left: args.left, right: args.right });
+            return ok(summarize(runId, report));
+        }
+        if (name === "explain_misses") {
+            const run = runs.get(String(args.runId));
+            if (!run)
+                throw new Error(`Unknown runId "${args.runId}". Call scan_join first.`);
+            const limit = typeof args.limit === "number" ? args.limit : 20;
+            const pairs = run.report.samplePairs
+                .filter((p) => !args.rule || p.rule === args.rule)
+                .slice(0, limit);
+            return ok({
+                runId: args.runId,
+                pairs,
+                legend: {
+                    verified: "ground truth confirms this pair",
+                    contradicted: "ground truth says this pair is WRONG — the matcher over-merged",
+                    ambiguous: "several right-hand rows collapsed onto one key; needs a tie-breaker, not a rule",
+                    unverifiable: "no ground truth available for this pair",
+                },
+            });
+        }
+        if (name === "suggest_normalization") {
+            const run = runs.get(String(args.runId));
+            if (!run)
+                throw new Error(`Unknown runId "${args.runId}". Call scan_join first.`);
+            const column = typeof args.column === "string" ? args.column : "key";
+            const chosen = run.report.stack.map((s) => s.rule);
+            const steps = run.report.stack.map((step, i) => {
+                const upto = rulesByIds(chosen.slice(0, i + 1));
+                return { ...step, sql: toSql(column, upto) };
+            });
+            return ok({
+                runId: args.runId,
+                recommended: chosen,
+                steps,
+                fullSql: toSql(column, rulesByIds(chosen)),
+                javascript: "import { normalize, rulesByIds } from 'shadowjoin';\n" +
+                    `const rules = rulesByIds(${JSON.stringify(chosen)});\n` +
+                    "const key = (value) => normalize(value, rules);",
+                caution: "Adopting a rule widens matching and can merge records that should stay separate. Re-run scan_join afterwards and compare_runs to confirm precision did not fall.",
+            });
+        }
+        if (name === "compare_runs") {
+            const a = runs.get(String(args.baselineRunId));
+            const b = runs.get(String(args.candidateRunId));
+            if (!a || !b)
+                throw new Error("Both runIds must come from scan_join calls in this session.");
+            const delta = (x, y) => Number((y - x).toFixed(2));
+            return ok({
+                baseline: summarize(String(args.baselineRunId), a.report),
+                candidate: summarize(String(args.candidateRunId), b.report),
+                delta: {
+                    exactMatches: delta(a.report.exactMatches, b.report.exactMatches),
+                    recovered: delta(a.report.recovered, b.report.recovered),
+                    reachableMatchRate: delta(a.report.reachableMatchRate, b.report.reachableMatchRate),
+                    lowerBoundLossPct: delta(a.report.lowerBoundLossPct, b.report.lowerBoundLossPct),
+                    precision: a.report.precision !== null && b.report.precision !== null ? delta(a.report.precision, b.report.precision) : null,
+                    contradicted: delta(a.report.contradicted, b.report.contradicted),
+                },
+                verdict: b.report.exactMatchRate > a.report.exactMatchRate
+                    ? "The exact join matches more rows than before."
+                    : "The exact join does not match more rows than before. If you changed the matching logic, it did not take effect.",
+            });
+        }
+        throw new Error(`Unknown tool "${name}"`);
+    }
+    catch (err) {
+        return {
+            isError: true,
+            content: [{ type: "text", text: err instanceof Error ? err.message : String(err) }],
+        };
+    }
+});
+await server.connect(new StdioServerTransport());

package/dist/normalize.d.ts

@@ -0,0 +1,19 @@
+import type { Rule } from "./types.js";
+/**
+ * Applied to every candidate key before any optional rule. These are the
+ * transforms that are safe on essentially any text: they never merge values
+ * that a human would read as different entities.
+ */
+export declare const BASE_RULES: Rule[];
+/**
+ * Each of these is measured INDEPENDENTLY on top of BASE_RULES, so the number
+ * next to a rule is what that rule alone buys you — not an artifact of the
+ * order rules happen to run in.
+ */
+export declare const OPTIONAL_RULES: Rule[];
+export declare const ALL_RULE_IDS: string[];
+export declare function rulesByIds(ids: string[]): Rule[];
+/** Apply BASE_RULES, then the supplied rules, in order. */
+export declare function normalize(value: string, rules?: Rule[]): string;
+/** Compose the enabled rules into one Postgres expression over a column. */
+export declare function toSql(column: string, rules?: Rule[]): string;