pi-mega-compact 0.4.5 → 0.4.6

package/dist/src/recall.integration.test.js

@@ -0,0 +1,77 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { compactSession } from "./engine.js";
+import { recallAndInline } from "./recall.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-resume-"));
+let counter = 0;
+/** Two instances, SAME disk dir — simulates a fresh process / resumed session. */
+function storeForDir(dir) {
+    return new VectorStore({ dedupSim: 0.9, stateDir: dir });
+}
+function msg(role, text, toolName) {
+    return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
+}
+const SESS = "sess_resume";
+/**
+ * Simulate the extension's `recentUserQuery`: the resume query is built from the
+ * newest user message in the (re-loaded) session. We model "newest user msg"
+ * directly rather than going through pi's session manager.
+ */
+function latestUserQuery(messages) {
+    for (let i = messages.length - 1; i >= 0; i--) {
+        if (messages[i].role === "user")
+            return messages[i].text;
+    }
+    return "";
+}
+test("resume contract: compact in one process, recall in a fresh one from disk", () => {
+    const dir = join(baseTmp, `run-${counter++}`);
+    // --- Process 1: the original session compacts ---
+    const writer = storeForDir(dir);
+    const session = [
+        msg("user", "investigated src/compact.ts and added a truncate helper"),
+        msg("assistant", "added truncate", "Edit"),
+        msg("user", "then wired it into the summary pipeline"),
+        msg("assistant", "wired it in", "Edit"),
+    ];
+    const ran = compactSession({ sessionId: SESS, messages: session, keepFrom: session.length, timestamp: 1 }, writer);
+    assert.equal(ran.skipped, false);
+    assert.ok(ran.checkpointId, "a checkpoint was persisted");
+    // --- Process 2: pi restarts, session resumes from disk ---
+    const reader = storeForDir(dir); // brand new instance, same dir
+    const resumeQuery = latestUserQuery(session); // newest user msg
+    const r = recallAndInline({ sessionId: SESS, query: resumeQuery, limit: 3, source: "resume" }, reader);
+    assert.equal(r.empty, false, "resume must re-surface the compacted context");
+    assert.equal(r.toInject.length, 1);
+    assert.equal(r.toInject[0].checkpoint.checkpointId, ran.checkpointId);
+    assert.ok(r.block.includes("Recalled context"), "block is model-visible system-prompt text");
+});
+test("resume contract: nothing to recall for a brand-new session", () => {
+    const dir = join(baseTmp, `run-${counter++}`);
+    const fresh = storeForDir(dir);
+    const r = recallAndInline({ sessionId: "sess_never_seen", query: "anything at all", limit: 3, source: "resume" }, fresh);
+    assert.equal(r.empty, true);
+    assert.equal(r.block, "");
+});
+test("resume contract: re-inject is deduped after the first recall", () => {
+    const dir = join(baseTmp, `run-${counter++}`);
+    const writer = storeForDir(dir);
+    const session = [
+        msg("user", "built the trigram embedder for the vector store"),
+        msg("assistant", "built it", "Edit"),
+    ];
+    compactSession({ sessionId: SESS, messages: session, keepFrom: 2, timestamp: 1 }, writer);
+    const reader = storeForDir(dir);
+    const q = latestUserQuery(session);
+    const first = recallAndInline({ sessionId: SESS, query: q, limit: 3, source: "resume" }, reader);
+    const second = recallAndInline({ sessionId: SESS, query: q, limit: 3, source: "resume" }, reader);
+    assert.equal(first.empty, false);
+    assert.equal(second.empty, true, "second resume does not re-inject (sentinel)");
+});
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/recall.js

@@ -0,0 +1,60 @@
+/**
+ * recall.ts — Layer 5 (RECALL / INLINE): the unified injection path.
+ *
+ * ONE vector store, THREE entry points, ONE dedup engine. Every way context
+ * gets re-injected into the window (auto-inline on resume, on-demand
+ * /recall-context, and the dedup sentinel) goes through `recallAndInline`.
+ * It always does: search -> dedupe -> inject. The only thing that differs per
+ * entry point is *what triggers it* and *what query it uses*.
+ *
+ * Injection respects PREVENT-PI-003: pi has no `system` message role, so we
+ * prepend our recall block to the system prompt via the `before_agent_start`
+ * hook's `systemPrompt` result (the extension wires that). This module is
+ * pi-agnostic: it returns an injectable text block and records injections; the
+ * extension decides where it lands.
+ */
+import { recall as searchRecall } from "./engine.js";
+/** Wrap a recall block so the model reads it as restored compacted context. */
+export function formatRecallBlock(hits) {
+    if (hits.length === 0)
+        return "";
+    const parts = hits.map((h, i) => {
+        const score = (h.score * 100).toFixed(0);
+        return (`### Recalled context [${i + 1}] (relevance ${score}%)\n` +
+            `${h.checkpoint.summary.trim()}\n` +
+            (h.checkpoint.filesModified.length
+                ? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
+                : ""));
+    });
+    return ("The following compacted context was recalled from earlier in this session " +
+        "and is relevant to the current request. Treat it as background you already know:\n\n" +
+        parts.join("\n"));
+}
+/**
+ * Run the unified recall+dudupe+prepare-inject pipeline. Does NOT touch pi;
+ * it records injections via `markInjected` so the next call dedupes. The
+ * `store` is passed by the extension (defaults to the engine's default store).
+ */
+export function recallAndInline(opts, store) {
+    const limit = opts.limit ?? 3;
+    const skip = opts.skipInjected ?? true;
+    const { hits } = searchRecall({ sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false }, store);
+    // Shared dedup: drop checkpoints already injected this session, then mark the
+    // survivors so repeated triggers are free. (Cosine near-dup collapse already
+    // happened inside store.search.)
+    const toInject = [];
+    for (const h of hits) {
+        if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId))
+            continue;
+        toInject.push(h);
+        store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+    }
+    const block = formatRecallBlock(toInject);
+    const report = toInject.map((h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`);
+    return {
+        toInject,
+        report,
+        block,
+        empty: toInject.length === 0,
+    };
+}

package/dist/src/recall.test.js

@@ -0,0 +1,50 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { compactSession } from "./engine.js";
+import { recallAndInline, formatRecallBlock } from "./recall.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
+let counter = 0;
+function store() {
+    return new VectorStore({ dedupSim: 0.9, stateDir: join(baseTmp, `run-${counter++}`) });
+}
+function msg(role, text, toolName) {
+    return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
+}
+const SESS = "sess_recall";
+test("recallAndInline injects new hits and marks them injected", () => {
+    const s = store();
+    compactSession({ sessionId: SESS, messages: [msg("user", "investigated src/vectorStore.ts embedding"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    compactSession({ sessionId: SESS, messages: [msg("user", "fixed the dedupe race in store.ts"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+    const r1 = recallAndInline({ sessionId: SESS, query: "vectorStore embedding", limit: 1, source: "command" }, s);
+    assert.equal(r1.empty, false);
+    assert.equal(r1.toInject.length, 1);
+    assert.ok(r1.block.includes("Recalled context"));
+    // Second call with the same query must NOT re-inject (shared dedup).
+    const r2 = recallAndInline({ sessionId: SESS, query: "vectorStore embedding", limit: 1, source: "command" }, s);
+    assert.equal(r2.empty, true);
+    assert.equal(r2.toInject.length, 0);
+});
+test("recallAndInline skipInjected=false re-returns hits", () => {
+    const s = store();
+    compactSession({ sessionId: SESS, messages: [msg("user", "configured the fast gate threshold"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+    const r1 = recallAndInline({ sessionId: SESS, query: "fast gate threshold", limit: 5, source: "resume" }, s);
+    const r2 = recallAndInline({ sessionId: SESS, query: "fast gate threshold", limit: 5, source: "resume", skipInjected: false }, s);
+    assert.equal(r1.toInject.length, 1);
+    assert.equal(r2.toInject.length, 1);
+});
+test("formatRecallBlock is empty for no hits", () => {
+    assert.equal(formatRecallBlock([]), "");
+});
+test("recallAndInline empty when store has nothing for query", () => {
+    const s = store();
+    const r = recallAndInline({ sessionId: SESS, query: "no such topic exists here", limit: 5, source: "command" }, s);
+    assert.equal(r.empty, true);
+    assert.equal(r.block, "");
+});
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/sprint14.test.js

@@ -0,0 +1,219 @@
+/**
+ * sprint14.test.ts — Sprint 14 full-pipeline wiring (flags, backfill, monitoring, canary).
+ * Hermetic: isolated state dirs, no network, no remote.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { defaultEmbedder } from "./embedder.js";
+import { loadDedupConfig } from "./config/dedup.js";
+import { loadMetrics, saveMetrics, recordDecision, evaluateAlerts, fpRate, p95, } from "./monitoring.js";
+import { backfillPhase, backfillRaptor } from "./store/backfill.js";
+import { listCheckpoints, closeStore } from "./store/sqlite.js";
+import { CanaryController, runCanary } from "./canary.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-s14-"));
+function cfg(over = {}) {
+    return { ...loadDedupConfig(), ...over };
+}
+function store(over = {}, eventsPath) {
+    const dir = join(baseTmp, `run-${Math.floor(performance.now() * 1000)}-${Math.random()}`);
+    return new VectorStore({ stateDir: dir, config: cfg(over), eventsPath });
+}
+// --- 1. Flag matrix: 16 combos don't crash add()/search() -------------------
+test("flag matrix: all 16 L0/L1/L2/RAPTOR enable combos are safe", () => {
+    const flags = [false, true];
+    let combos = 0;
+    for (const l0 of flags)
+        for (const l1 of flags)
+            for (const l2 of flags)
+                for (const raptor of flags) {
+                    combos++;
+                    const s = store({ L0_ENABLED: l0, L1_ENABLED: l1, L2_ENABLED: l2, RAPTOR_ENABLED: raptor });
+                    s.add({ sessionId: "s", summary: "x", regionText: `region A for combo ${combos} about the cache`, timestamp: 1 });
+                    const r2 = s.add({ sessionId: "s", summary: "x", regionText: `region B for combo ${combos} about the parser`, timestamp: 2 });
+                    // search must not throw under any combination
+                    const hits = s.search("s", "cache", 3);
+                    assert.ok(Array.isArray(hits));
+                    // With all tiers off, the second add is always "new" (no collapse).
+                    if (!l0 && !l1 && !l2)
+                        assert.equal(r2.deduped, false);
+                }
+    assert.equal(combos, 16);
+});
+// --- 2. MARK_ONLY_L1 records but doesn't collapse ---------------------------
+test("MARK_ONLY_L1: L1 match is recorded but not collapsed (new checkpoint)", () => {
+    // Disable L2 so we isolate L1 behavior (L2 cosine would otherwise catch the near-dup).
+    const s = store({ L1_ENABLED: true, MARK_ONLY_L1: true, L2_ENABLED: false });
+    const a = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loop", timestamp: 1 });
+    const b = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loops", timestamp: 2 });
+    assert.equal(a.deduped, false);
+    // MARK_ONLY → b is NOT collapsed into a; both stored as active.
+    assert.equal(b.deduped, false);
+    const all = listCheckpoints("s", s.stateDir);
+    assert.equal(all.length, 2);
+    assert.ok(all.every((c) => c.dedupStatus === "active"));
+});
+test("MARK_ONLY_L1 off: L1 match IS collapsed", () => {
+    const s = store({ L1_ENABLED: true, MARK_ONLY_L1: false });
+    s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loop", timestamp: 1 });
+    const b = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loops", timestamp: 2 });
+    assert.equal(b.deduped, true);
+    assert.equal(b.reason, "l1MinHash");
+});
+// --- 3. Backfill resumes after interrupt -------------------------------------
+test("backfill L1 resumes after simulated interrupt", () => {
+    const dir = join(baseTmp, `bf-${Math.floor(performance.now())}`);
+    // Seed 12 checkpoints, 2 per batch of 5.
+    const s = new VectorStore({ stateDir: dir, config: cfg() });
+    const texts = [
+        "The walrus drifted past the lighthouse while the baker kneaded sourdough at dawn",
+        "Quantum entanglement linked the two photons across the lab in a cryogenic chamber",
+        "A medieval scribe copied the gospel by candlelight atop a windswept cliff",
+        "The rover sampled basalt from the crater and transmitted spectra to mission control",
+        "Jazz musicians improvised a syncopated triangle rhythm beneath the streetlamp",
+        "The glacier calved a towering iceberg into the fjord with a thunderous crack",
+        "A botanist cataloged the orchid species thriving in the cloud forest canopy",
+        "The blacksmith forged a horseshoe while sparks danced across the anvil",
+        "Astronomers imaged a distant nebula glowing with newborn stellar furnaces",
+        "The ferry crossed the strait as gulls wheeled above the churning wake",
+        "A weaver threaded crimson silk through the loom in the mountain village",
+        "The surgeon sutured the incision with steady hands under the theatre lights",
+    ];
+    for (let i = 0; i < 12; i++) {
+        s.add({ sessionId: "sess_bf", summary: `n${i}`, regionText: texts[i], timestamp: i });
+    }
+    // Interrupt after batch 1 (5 rows).
+    const r1 = backfillPhase("L1", "sess_bf", dir, { batchSize: 5, interruptAfterBatches: 1 });
+    assert.equal(r1.interrupted, true);
+    assert.equal(r1.processed, 5);
+    assert.ok(r1.cursor === "chkpt_005" || r1.cursor === "chkpt_05");
+    // Resume: should continue from the cursor → process the remaining 7.
+    const r2 = backfillPhase("L1", "sess_bf", dir, { batchSize: 5 });
+    assert.equal(r2.interrupted, false);
+    assert.equal(r2.processed, 12); // total across both runs (cursor-based resume)
+    closeStore(dir);
+});
+// --- 4. Alert fires on injected FP spike -------------------------------------
+test("alert: FP spike breaches threshold → MARK_ONLY flagged + warning", () => {
+    const config = cfg();
+    const m = loadMetrics("/dev/null");
+    // Inject 100 L1 decisions, 20 false positives → 20% > FP_RATE_L1L2 (5%).
+    for (let i = 0; i < 100; i++) {
+        recordDecision(m, "L1", i < 20 ? "deduped" : "new", 5, i < 20);
+    }
+    const res = evaluateAlerts(m, config);
+    assert.ok(res.breached.includes("L1"));
+    assert.ok(res.warnings.some((w) => w.includes("DEDUP FP BREACH tier=L1")));
+    assert.ok(fpRate(m, "L1") > config.FP_RATE_L1L2);
+});
+test("alert: clean run does NOT breach", () => {
+    const config = cfg();
+    const m = loadMetrics("/dev/null");
+    for (let i = 0; i < 100; i++)
+        recordDecision(m, "L1", "new", 5, false);
+    const res = evaluateAlerts(m, config);
+    assert.equal(res.breached.length, 0);
+});
+// --- 5. Canary auto-disables a tier whose p95 exceeds budget ---------------
+test("canary: auto-disables a tier whose p95 exceeds budget", () => {
+    // Feed where L2 always has high latency (breaches P95_BUDGET_MS).
+    const feed = (_step, c) => {
+        const m = loadMetrics("/dev/null");
+        if (c.L0_ENABLED)
+            recordDecision(m, "L0", "new", 1, false);
+        if (c.L1_ENABLED)
+            recordDecision(m, "L1", "new", 1, false);
+        if (c.L2_ENABLED)
+            recordDecision(m, "L2", "new", 500, false); // > 100ms budget
+        if (c.RAPTOR_ENABLED)
+            recordDecision(m, "RAPTOR", "new", 1, false);
+        return m;
+    };
+    const { controller, disabled } = runCanary(feed, cfg({ P95_BUDGET_MS: 100 }));
+    assert.ok(disabled.includes("L2"), `expected L2 auto-disabled, got ${JSON.stringify(disabled)}`);
+    assert.equal(controller.config.L2_ENABLED, false);
+    // Lower tiers should still be enabled.
+    assert.equal(controller.config.L0_ENABLED, true);
+});
+test("canary: sequential enablement order L0→L1→L2→RAPTOR", () => {
+    const c = new CanaryController(cfg());
+    assert.deepEqual([...c.getState().enabled], ["L0"]);
+    const t1 = c.stepForward();
+    assert.equal(t1, "L1");
+    assert.equal(c.stepForward(), "L2");
+    assert.equal(c.stepForward(), "RAPTOR");
+    assert.equal(c.stepForward(), null); // all enabled
+});
+// --- 6. Monitoring: structured decision events written ----------------------
+test("monitoring: add() writes structured decision events to events.log", () => {
+    const dir = join(baseTmp, `mon-${Math.floor(performance.now())}`);
+    const eventsPath = join(dir, "events.log");
+    const s = new VectorStore({
+        stateDir: dir,
+        config: cfg(),
+        eventsPath,
+    });
+    s.add({ sessionId: "s", summary: "x", regionText: "unique region alpha one", timestamp: 1 });
+    s.add({ sessionId: "s", summary: "x", regionText: "unique region alpha one", timestamp: 2 }); // L0 content dup
+    assert.ok(existsSync(eventsPath));
+    const lines = readFileSync(eventsPath, "utf-8").trim().split("\n").filter(Boolean);
+    assert.ok(lines.length >= 2);
+    const ev = JSON.parse(lines[0]);
+    assert.ok(["L0"].includes(ev.tier));
+    assert.ok(["new", "deduped", "mark_only"].includes(ev.result));
+    closeStore(dir);
+});
+// --- 7. RAPTOR backfill builds + persists a tree ---------------------------
+test("backfill RAPTOR builds + persists a tree for a session", () => {
+    const dir = join(baseTmp, `raptorbf-${Math.floor(performance.now())}`);
+    const s = new VectorStore({ stateDir: dir, config: cfg() });
+    const texts = [
+        "The whale breached beside the research vessel near the polar ice shelf",
+        "A potter shaped the clay vessel on the spinning wheel at the riverside studio",
+        "The comet streaked across the pre dawn sky witnessed by the hilltop observatory",
+        "Lumberjacks felled the ancient cedar while the river carried the logs downstream",
+        "The chemist titrated the solution until the indicator turned faint violet",
+        "A flock of cranes migrated northward over the thawing wetland at first light",
+        "The locksmith picked the stubborn tumbler and opened the oak cabinet",
+        "Geologists hammered the schist sample from the canyon wall into the satchel",
+        "The chocolatier tempered the couverture until it snapped with a clean gloss",
+        "A fisher cast the line into the mist where the trout rose to the fly",
+        "The archivist unsealed the parchment scroll recovered from the coastal ruin",
+        "Beekeepers harvested the golden comb while the orchard blossoms drifted down",
+        "The pilot navigated the canyon winds using only the instrument panel glow",
+        "A tailor stitched the velvet cuff with silk thread by the window",
+        "The miner extracted the quartz crystal from the vein deep in the shaft",
+        "Cartographers plotted the uncharted island onto the worn leather map",
+        "The gardener pruned the rosebush and tied the canes to the cedar trellis",
+        "A violinist tuned the gut strings until the chamber rang pure and bright",
+        "The diver surfaced with the amphora lifted from the sunken galleon",
+        "Shepherds guided the flock across the high pasture toward the stone bothy",
+    ];
+    for (let i = 0; i < 20; i++) {
+        s.add({ sessionId: "sess_rb", summary: `n${i}`, regionText: texts[i], timestamp: i });
+    }
+    const res = backfillRaptor("sess_rb", dir, defaultEmbedder());
+    assert.ok(res.processed > 0);
+    // Sanity: source checkpoints remain intact (backfill is additive).
+    const nodes = listCheckpoints("sess_rb", dir).length;
+    assert.ok(nodes >= 20);
+    closeStore(dir);
+});
+// --- 8. dashboard.json metrics round-trip -----------------------------------
+test("metrics: dashboard.json persists + p95 computed", () => {
+    const path = join(baseTmp, `dash-${Math.floor(performance.now())}.json`);
+    const m = loadMetrics(path);
+    for (let i = 0; i < 10; i++)
+        recordDecision(m, "L2", "new", i * 10, false);
+    saveMetrics(path, m);
+    const reloaded = loadMetrics(path);
+    assert.equal(reloaded.decisions.L2, 10);
+    assert.equal(p95(reloaded.latency.L2), 90); // 95th pct of [0,10,..,90]
+});
+// --- cleanup ----------------------------------------------------------------
+test("Sprint 14 cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/store/backfill.js

@@ -0,0 +1,189 @@
+/**
+ * backfill.ts — resumable / idempotent hash backfill (Sprint 10).
+ *
+ * Purpose: populate `content_hash` / `content_hash2` / `content_hash_version` /
+ * `normalized_text` for any rows left with null hashes (e.g. pre-Sprint-9 data
+ * or rows that degraded to "store without dedup" under the QA #13 timeout).
+ * Structured so Sprint 11 can plug in its own MinHash/LSH phase after the
+ * content hashes land.
+ *
+ * Properties (QA #1 / QA #14):
+ *   - Resumable: progress stored in a `backfill_progress` table (last processed id).
+ *   - Idempotent: ON CONFLICT DO NOTHING + partial UNIQUE on (session_id, content_hash)
+ *     make a second run a no-op where it safely can.
+ *   - Batched: 1000 rows/commit to bound lock time; throttle between batches.
+ *
+ * SQLite is the source of truth; this touches no network (PREVENT-PI-004).
+ */
+import { openStore } from "./sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
+import { lshBands } from "../dedup/l1-lsh.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { buildRaptorTree } from "../dedup/raptor/tree.js";
+import { defaultEmbedder } from "../embedder.js";
+import { getStateDir } from "../store.js";
+const BATCH = 1000;
+const THROTTLE_MS = 0; // synchronous backfill; no cross-process yield needed
+function ensureProgressTable(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS backfill_progress (
+      name TEXT PRIMARY KEY,
+      last_session_id TEXT, -- session of the highest (session_id, id) scanned
+      last_id TEXT,         -- highest context_chunks.id scanned within that session
+      updated INTEGER,
+      duplicates_resolved INTEGER
+    );
+  `);
+}
+function progress(db) {
+    const row = db
+        .prepare("SELECT last_session_id, last_id, updated, duplicates_resolved FROM backfill_progress WHERE name='content_hashes'")
+        .get();
+    return { lastSid: row?.last_session_id ?? null, lastId: row?.last_id ?? null, updated: row?.updated ?? 0, dups: row?.duplicates_resolved ?? 0 };
+}
+/**
+ * Backfill content hashes for all rows missing them, from the last scanned
+ * (session_id, id) forward. Returns counts; fully idempotent and resumable.
+ */
+export function backfillContentHashes(stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    ensureProgressTable(db);
+    const start = progress(db);
+    // Rows needing hashing: null content_hash, ordered by (session_id, id) for a
+    // stable, resumable cursor (ids are only unique per session).
+    const pending = db
+        .prepare(`SELECT id, session_id, summary FROM context_chunks
+       WHERE content_hash IS NULL
+         AND (session_id > COALESCE(?, '')
+              OR (session_id = COALESCE(?, '') AND id > COALESCE(?, '')))
+       ORDER BY session_id ASC, id ASC LIMIT ?`)
+        .all(start.lastSid, start.lastSid, start.lastId, BATCH);
+    let updated = start.updated;
+    let duplicatesResolved = start.dups;
+    let processed = 0;
+    let lastSid = start.lastSid;
+    let lastId = start.lastId;
+    const tx = db.transaction((rows) => {
+        const lookup = db.prepare("SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1");
+        const update = db.prepare(`UPDATE context_chunks
+       SET content_hash=?, content_hash2=?, content_hash_version=?, normalized_text=?,
+           dedup_status='active'
+       WHERE id=?`);
+        for (const row of rows) {
+            const digest = computeContentDigest(row.summary ?? "");
+            // Keep the oldest row on a collision (partial UNIQUE would reject the
+            // newer insert); mark the newer one as superseded-without-store.
+            const clash = lookup.get(row.session_id, digest.contentHash, digest.contentHash2, row.id);
+            if (clash) {
+                db.prepare("UPDATE context_chunks SET dedup_status='dup-resolved' WHERE id=?").run(row.id);
+                duplicatesResolved++;
+            }
+            else {
+                update.run(digest.contentHash, digest.contentHash2, digest.contentHashVersion, digest.normalizedText, row.id);
+                updated++;
+            }
+            lastSid = row.session_id;
+            lastId = row.id;
+            processed++;
+        }
+    });
+    if (pending.length > 0) {
+        tx(pending);
+        db.prepare("INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved").run(lastSid, lastId, updated, duplicatesResolved);
+    }
+    if (THROTTLE_MS > 0) {
+        // No-op in this synchronous build; placeholder for future streaming backfill.
+    }
+    return { processed, updated, duplicatesResolved };
+}
+/** True when no rows remain pending (backfill complete for this state dir). */
+export function isBackfillComplete(stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    const row = db
+        .prepare("SELECT COUNT(*) AS c FROM context_chunks WHERE content_hash IS NULL")
+        .get();
+    return row.c === 0;
+}
+// ---- Sprint 14: L1 / L2 / RAPTOR phase backfill (resumable) ---------------
+function phaseCursor(db, phase) {
+    ensureProgressTable(db);
+    const row = db
+        .prepare("SELECT last_id, updated AS processed FROM backfill_progress WHERE name = ?")
+        .get(`phase_${phase}`);
+    return { lastId: row?.last_id ?? null, processed: row?.processed ?? 0 };
+}
+function savePhaseCursor(db, phase, lastId, processed) {
+    db.prepare(`INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved)
+     VALUES(?, NULL, ?, ?, 0)
+     ON CONFLICT(name) DO UPDATE SET last_id=excluded.last_id, updated=excluded.updated`).run(`phase_${phase}`, lastId, processed);
+}
+/**
+ * Backfill L1 (MinHash sigs + LSH buckets) or L2 (MinHash sigs only) derived
+ * data for a session, in batches, resumable from the persisted cursor. Pass
+ * `interruptAfterBatches` to simulate a crash for resume testing.
+ */
+export function backfillPhase(phase, sessionId, stateDir, opts = {}) {
+    const db = openStore(stateDir);
+    const batchSize = opts.batchSize ?? BATCH;
+    const all = listCheckpoints(sessionId, stateDir).sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
+    const { lastId } = phaseCursor(db, phase);
+    let { processed } = phaseCursor(db, phase);
+    const startIndex = lastId ? all.findIndex((c) => c.checkpointId === lastId) + 1 : 0;
+    let batches = 0;
+    let interrupted = false;
+    let cursor = lastId ?? undefined;
+    for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
+        const batch = all.slice(i, i + batchSize);
+        const tx = db.transaction(() => {
+            for (const cp of batch) {
+                const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
+                if (sig.length === NUM_HASHES) {
+                    upsertMinhashSignature(cp.checkpointId, sessionId, SIGNATURE_VERSION, sig, stateDir);
+                    if (phase === "L1") {
+                        insertLshBuckets(cp.checkpointId, sessionId, SIGNATURE_VERSION, lshBands(sig, sessionId, SIGNATURE_VERSION), stateDir);
+                    }
+                }
+                cursor = cp.checkpointId;
+                processed++;
+            }
+        });
+        tx();
+        savePhaseCursor(db, phase, cursor ?? null, processed);
+        batches++;
+        if (THROTTLE_MS > 0) {
+            const end = Date.now() + THROTTLE_MS;
+            while (Date.now() < end) { /* throttle */ }
+        }
+        if (opts.interruptAfterBatches && batches >= opts.interruptAfterBatches) {
+            interrupted = true;
+            break;
+        }
+    }
+    return { phase, processed, batches, interrupted, cursor };
+}
+/**
+ * Backfill the RAPTOR tree for a session (single pass over all leaves). Builds
+ * + persists raptor_nodes. Not batched — the builder has its own budget cap.
+ */
+export function backfillRaptor(sessionId, stateDir, embedder = defaultEmbedder()) {
+    const all = listCheckpoints(sessionId, stateDir).sort((a, b) => a.checkpointId.localeCompare(b.checkpointId));
+    const leaves = all.map((cp) => {
+        const text = cp.normalizedText ?? cp.summary ?? "";
+        return {
+            id: cp.checkpointId,
+            messages: [{ role: "user", text }],
+            sourceText: text,
+            embedding: embedder.embed(text),
+        };
+    });
+    if (leaves.length === 0) {
+        return { phase: "RAPTOR", processed: 0, batches: 0, interrupted: false, cursor: undefined };
+    }
+    const tree = buildRaptorTree(leaves, { embedder });
+    saveRaptorTree(sessionId, tree, stateDir);
+    const db = openStore(stateDir);
+    ensureProgressTable(db);
+    savePhaseCursor(db, "RAPTOR", leaves[leaves.length - 1].id, leaves.length);
+    return { phase: "RAPTOR", processed: leaves.length, batches: 1, interrupted: false, cursor: leaves[leaves.length - 1].id };
+}