pi-mega-compact 0.4.0

package/src/dedup/raptor/retrieval.ts

@@ -0,0 +1,102 @@
+/**
+ * retrieval.ts — staged RAPTOR retrieval (Sprint 13, Phase 6).
+ *
+ * Given a built tree (shadow or live), answer a query by:
+ *   1. ANN: score the top-level nodes at every level by cosine to the query.
+ *   2. expand: take the top-M nodes (across levels) and descend.
+ *   3. BFS: from those anchors, walk down to leaf nodes.
+ *   4. MMR: diversify the resulting leaf set before returning.
+ *
+ * This module is pure query logic over an in-memory RaptorTree. In Sprint 13 it
+ * is exercised only in shadow/eval; the live store (vectorStore.search) is NOT
+ * replaced until Sprint 14 promotes RAPTOR.
+ */
+
+import type { Embedder, Vector } from "../../embedder.js";
+import { cosineSimilarity } from "../../embedder.js";
+import { mmrRerank } from "../mmr.js";
+import type { RaptorTree, RaptorNode } from "./tree.js";
+
+export interface RaptorRetrieveOptions {
+  embedder: Embedder;
+  /** How many top nodes per level to expand from. */
+  topM?: number;
+  /** Final number of leaf nodes to return. */
+  k?: number;
+  /** MMR diversity weight. */
+  mmrLambda?: number;
+}
+
+/** Any child id not present in the node map is a raw leaf id. */
+function isLeafId(id: string, tree: RaptorTree): boolean {
+  return !tree.nodes.has(id);
+}
+
+/** All leaf (raw) ids reachable beneath a node via BFS. */
+function leafDescendants(node: RaptorNode, tree: RaptorTree): string[] {
+  const out: string[] = [];
+  const queue = [node];
+  while (queue.length) {
+    const cur = queue.shift()!;
+    for (const childId of cur.children) {
+      if (isLeafId(childId, tree)) out.push(childId);
+      else {
+        const child = tree.nodes.get(childId);
+        if (child) queue.push(child);
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Staged expansion retrieval over a RAPTOR tree.
+ *
+ * Returns up to `k` leaf ids, diversified by MMR. Deterministic given the
+ * tree + query. Throws nothing — returns [] on empty tree.
+ */
+export function stagedExpansion(
+  query: string,
+  tree: RaptorTree,
+  opts: RaptorRetrieveOptions,
+): string[] {
+  if (!tree.rootId) return [];
+  const embedder = opts.embedder;
+  const topM = opts.topM ?? 3;
+  const k = opts.k ?? 5;
+  const lambda = opts.mmrLambda ?? 0.5;
+
+  const qv = embedder.embed(query);
+
+  // 1. ANN: score every node at every level by cosine to the query.
+  const scored = [...tree.nodes.values()].map((n) => ({
+    node: n,
+    score: cosineSimilarity(qv, n.embedding),
+  }));
+
+  // 2. expand: top-M nodes overall (BFS anchors).
+  const anchors = scored
+    .slice()
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topM)
+    .map((s) => s.node);
+
+  // 3. BFS to leaves from those anchors.
+  const leaves = new Map<string, RaptorNode>();
+  for (const a of anchors) {
+    for (const lid of leafDescendants(a, tree)) {
+      // Represent each leaf by its nearest internal parent so we can score it.
+      // (The leaf's own centroid is stored on the level-0 node that wraps it.)
+      const parent = [...tree.nodes.values()].find((n) => n.children.includes(lid));
+      if (parent) leaves.set(lid, parent);
+    }
+  }
+
+  // 4. MMR diversify the expanded leaf set by their (parent) embeddings.
+  const items = [...leaves.entries()].map(([lid, n]) => ({
+    item: lid,
+    vector: n.embedding as Vector,
+    relevance: cosineSimilarity(qv, n.embedding),
+  }));
+  return mmrRerank(items, k, lambda);
+}

package/src/dedup/raptor/summarizer.ts

@@ -0,0 +1,91 @@
+/**
+ * summarizer.ts — per-cluster summary for the RAPTOR tree (Sprint 13, Phase 6).
+ *
+ * Default: pure extractive (deterministic, zero network, zero model) reusing
+ * extractive.ts. Optional: a LOCAL Ollama model (llama3.2:3b by default) when
+ * MEGACOMPACT_RAPTOR_MODEL is set — localhost only, same PREVENT-PI-004
+ * exception class as HttpEmbedder/the dashboard. No remote API is ever called.
+ *
+ * The summarizer returns structured text + a token estimate. Faithfulness is
+ * enforced downstream by guardrails.ts — this module only produces candidates.
+ */
+
+import type { EngineMessage } from "../../types.js";
+import { extractiveSummarize } from "../../extractive.js";
+import { estimateBlockTokens } from "../../tokens.js";
+import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+
+export interface ClusterSummary {
+  summary: string;
+  tokenEstimate: number;
+}
+
+/** The local Ollama endpoint (loopback). Read lazily so tests can avoid it. */
+function ollamaEndpoint(): { url: string; model: string } | null {
+  const model = process.env.MEGACOMPACT_RAPTOR_MODEL;
+  if (!model) return null;
+  const base = process.env.MEGACOMPACT_RAPTOR_URL ?? "http://127.0.0.1:11434";
+  // Guard: only loopback is permitted (remote Ollama would violate PREVENT-PI-004).
+  if (!/^https?:\/\/(localhost|127\.0\.0\.1)([:/]|$)/.test(base)) {
+    throw new Error(
+      `MEGACOMPACT_RAPTOR_URL must be localhost/127.0.0.1 (got ${base}). ` +
+        `Remote Ollama is not allowed (PREVENT-PI-004).`,
+    );
+  }
+  return { url: `${base}/api/generate`, model };
+}
+
+/**
+ * Extractive summarization of a cluster's source messages. Deterministic and
+ * fully local — the on-by-default path.
+ */
+export function extractiveClusterSummary(messages: EngineMessage[]): ClusterSummary {
+  const s = extractiveSummarize(messages);
+  return { summary: s.topicSummary, tokenEstimate: s.tokenEstimate };
+}
+
+/**
+ * Build a summary for one cluster of source messages.
+ *
+ * Uses local Ollama when MEGACOMPACT_RAPTOR_MODEL is set (localhost-only);
+ * otherwise falls back to deterministic extractive. The `fetch` is a localhost
+ * call inside the PREVENT-PI-004 exception — annotated accordingly.
+ */
+export function summarizeCluster(messages: EngineMessage[]): ClusterSummary {
+  const ollama = ollamaEndpoint();
+  if (!ollama) return extractiveClusterSummary(messages);
+  return ollamaSummarize(messages, ollama);
+}
+
+function ollamaSummarize(messages: EngineMessage[], ollama: { url: string; model: string }): ClusterSummary {
+  // The fetch below is localhost-only (loopback Ollama) — the PREVENT-PI-004
+  // sanctioned local-model exceptions (same class as /dashboard, HttpEmbedder).
+  const prompt = messages.map((m) => `${m.role}: ${m.text}`).join("\n");
+  // Synchronous bridge: spawnSync an inline worker so the call blocks without
+  // deadlocking fetch (mirrors HttpEmbedder — Atomics.wait on main thread would
+  // hang). A blocked main thread cannot pump the socket.
+  const WORKER = String.raw`
+    const url = process.env.R_URL, model = process.env.R_MODEL, prompt = process.env.R_PROMPT;
+    try {
+      const r = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model, prompt, stream: false }) }); // guardrails-allow PREVENT-PI-004: localhost-only user-spawned Ollama server (BYO local model, never remote)
+      const j = await r.json();
+      process.stdout.write(JSON.stringify({ ok: r.ok, text: j.response || "" }));
+    } catch (e) {
+      process.stdout.write(JSON.stringify({ ok: false, error: String(e && e.message ? e.message : e) }));
+    }
+  `;
+  const res = spawnSync(process.execPath, ["-e", WORKER], {
+    encoding: "utf8",
+    env: { ...process.env, R_URL: ollama.url, R_MODEL: ollama.model, R_PROMPT: prompt },
+  });
+  let parsed: { ok: boolean; text?: string; error?: string } = { ok: false, error: "no response" };
+  if (typeof res.stdout === "string" && res.stdout.length > 0) {
+    try { parsed = JSON.parse(res.stdout); } catch { parsed = { ok: false, error: "bad json" }; }
+  }
+  if (!parsed.ok || !parsed.text) {
+    // Ollama unavailable → deterministic extractive fallback (never fail the build).
+    return extractiveClusterSummary(messages);
+  }
+  const summary = parsed.text.trim();
+  return { summary, tokenEstimate: estimateBlockTokens(summary) };
+}

package/src/dedup/raptor/tree.ts

@@ -0,0 +1,254 @@
+/**
+ * tree.ts — RAPTOR hierarchical summary-tree builder (Sprint 13, Phase 6).
+ *
+ * Builds a multi-level tree of summary nodes over leaf chunks: leaves are the
+ * original regions; each higher level summarizes clusters of the level below
+ * until a single root remains. QA ops: a wall-clock budget guard — on exhaustion
+ * we build an extractive fallback root. <10 leaves → a single summary node.
+ *
+ * Node model (kept simple + flat): every RaptorNode stores the LIST OF LEAF IDS
+ * it ultimately covers in `children` (not a mix of node/leaf ids). So the node
+ * map holds ONLY internal summary nodes — never per-leaf wrappers — which is
+ * what makes RAPTOR consolidate (nodes.size << leaves) and makes retrieval's
+ * leaf walk trivial.
+ *
+ * PREVENT-PI-004: no network here. summarizeCluster() may call a localhost
+ * Ollama (annotated in summarizer.ts); extractive is the default.
+ */
+
+import type { Embedder, Vector } from "../../embedder.js";
+import { kmeanspp, meanVector } from "./kmeans.js";
+import { summarizeCluster } from "./summarizer.js";
+import { applyHallucinationGuardrails, sourceTokenSet, type QualityMarker } from "./guardrails.js";
+import type { EngineMessage } from "../../types.js";
+
+export interface RaptorNode {
+  id: string;
+  level: number;
+  parentId: string | null;
+  /** Leaf ids this node ultimately covers (flattened through the hierarchy). */
+  children: string[];
+  summary: string;
+  embedding: Vector; // centroid of the covered leaves
+  qualityMarker: QualityMarker;
+  tokenEstimate: number;
+}
+
+export interface RaptorTree {
+  nodes: Map<string, RaptorNode>;
+  rootId: string | null;
+  levels: number;
+  /** True when the budget forced an extractive fallback root. */
+  timedOut: boolean;
+}
+
+export interface Leaf {
+  id: string;
+  messages: EngineMessage[];
+  /** Precomputed source text (for grounding) + embedding (leaf centroid). */
+  sourceText: string;
+  embedding: Vector;
+}
+
+export interface BuildOptions {
+  embedder: Embedder;
+  /** Max wall-clock ms for the whole build (QA ops budget). */
+  budgetMs?: number;
+  /** Target clusters per level (k for k-means). */
+  clustersPerLevel?: number;
+  /** Consistency gate (guardrails.ts). */
+  consistencyThreshold?: number;
+  /** Injectable id generator (deterministic in tests). */
+  nextId?: (level: number, index: number) => string;
+  /** Injectable clock for the budget guard (ms). */
+  now?: () => number;
+}
+
+const DEFAULT_BUDGET_MS = 5000;
+const DEFAULT_CLUSTERS = 5;
+
+function defaultNextId(level: number, index: number): string {
+  return `r${level}_${index}`;
+}
+
+/** An item being clustered at any level: a leaf, or a grouping of leaves. */
+interface ClusterItem {
+  id: string;
+  embedding: Vector;
+  leafIds: string[];
+  /** Source messages for summarization (flattened). */
+  messages: EngineMessage[];
+  /** Source texts for grounding (flattened). */
+  sources: string[];
+}
+
+function summarizeInto(
+  item: ClusterItem,
+  centroid: Vector,
+  embedder: Embedder,
+  consistencyThreshold?: number,
+): { summary: string; tokenEstimate: number; qualityMarker: QualityMarker } {
+  let summary = summarizeCluster(item.messages);
+  const guard = applyHallucinationGuardrails({
+    summary: summary.summary,
+    sources: item.sources,
+    centroid,
+    embedder,
+    sourceTokens: sourceTokenSet(item.sources),
+    consistencyThreshold,
+  });
+  if (guard.marker === "extractive_fallback") {
+    summary = summarizeCluster(item.messages); // deterministic extractive text
+  }
+  return {
+    summary: summary.summary,
+    tokenEstimate: summary.tokenEstimate,
+    qualityMarker: guard.marker === "extractive_fallback" ? "low" : guard.marker,
+  };
+}
+
+/**
+ * Build a RAPTOR tree from leaf chunks. Synchronous; guarded by an elapsed-time
+ * budget. Returns a tree whose `nodes` map holds ONLY internal summary nodes.
+ */
+export function buildRaptorTree(leaves: Leaf[], opts: BuildOptions): RaptorTree {
+  const embedder = opts.embedder;
+  const budgetMs = opts.budgetMs ?? DEFAULT_BUDGET_MS;
+  const clustersPerLevel = opts.clustersPerLevel ?? DEFAULT_CLUSTERS;
+  const nextId = opts.nextId ?? defaultNextId;
+  const now = opts.now ?? (() => Date.now());
+  const start = now();
+  const within = () => now() - start <= budgetMs;
+
+  const nodes = new Map<string, RaptorNode>();
+
+  // <10 leaves → single summary root (no hierarchy needed).
+  if (leaves.length < 10) {
+    const item: ClusterItem = {
+      id: "root",
+      embedding: meanVector(leaves.map((l) => l.embedding)),
+      leafIds: leaves.map((l) => l.id),
+      messages: leaves.flatMap((l) => l.messages),
+      sources: leaves.map((l) => l.sourceText),
+    };
+    const centroid = item.embedding;
+    const { summary, tokenEstimate, qualityMarker } = summarizeInto(item, centroid, embedder, opts.consistencyThreshold);
+    const rootId = nextId(0, 0);
+    nodes.set(rootId, {
+      id: rootId,
+      level: 0,
+      parentId: null,
+      children: item.leafIds,
+      summary,
+      embedding: centroid,
+      qualityMarker,
+      tokenEstimate,
+    });
+    return { nodes, rootId, levels: 1, timedOut: false };
+  }
+
+  let currentLevel: ClusterItem[] = leaves.map((l) => ({
+    id: l.id,
+    embedding: l.embedding,
+    leafIds: [l.id],
+    messages: l.messages,
+    sources: [l.sourceText],
+  }));
+
+  let level = 0;
+  while (currentLevel.length > 1) {
+    if (!within()) return extractiveFallbackRoot(leaves, nodes, nextId);
+
+    // Once we're down to a handful of items, collapse them all into one root.
+    // (k === currentLevel.length would make every item its own singleton
+    // cluster and never shrink — an infinite loop until the budget blows.)
+    if (currentLevel.length <= clustersPerLevel) {
+      const merged: ClusterItem = {
+        id: "merge",
+        embedding: meanVector(currentLevel.map((c) => c.embedding)),
+        leafIds: currentLevel.flatMap((c) => c.leafIds),
+        messages: currentLevel.flatMap((c) => c.messages),
+        sources: currentLevel.flatMap((c) => c.sources),
+      };
+      const centroid = merged.embedding;
+      const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, centroid, embedder, opts.consistencyThreshold);
+      const rootId = nextId(level + 1, 0);
+      nodes.set(rootId, {
+        id: rootId,
+        level: level + 1,
+        parentId: null,
+        children: merged.leafIds,
+        summary,
+        embedding: centroid,
+        qualityMarker,
+        tokenEstimate,
+      });
+      return { nodes, rootId, levels: level + 2, timedOut: false };
+    }
+
+    const k = Math.max(1, Math.min(clustersPerLevel, currentLevel.length));
+    const clustered = kmeanspp(currentLevel.map((c) => c.embedding), k, { seed: 0x1234 + level });
+
+    const groups: ClusterItem[][] = Array.from({ length: clustered.k }, () => []);
+    clustered.assignments.forEach((c, i) => groups[c].push(currentLevel[i]));
+
+    const nextLevel: ClusterItem[] = [];
+    for (let g = 0; g < groups.length; g++) {
+      const group = groups[g];
+      if (group.length === 0) continue;
+      const merged: ClusterItem = {
+        id: nextId(level + 1, g),
+        embedding: clustered.centroids[g],
+        leafIds: group.flatMap((c) => c.leafIds),
+        messages: group.flatMap((c) => c.messages),
+        sources: group.flatMap((c) => c.sources),
+      };
+      const { summary, tokenEstimate, qualityMarker } = summarizeInto(merged, merged.embedding, embedder, opts.consistencyThreshold);
+      nodes.set(merged.id, {
+        id: merged.id,
+        level: level + 1,
+        parentId: null,
+        children: merged.leafIds,
+        summary,
+        embedding: merged.embedding,
+        qualityMarker,
+        tokenEstimate,
+      });
+      nextLevel.push(merged);
+    }
+    currentLevel = nextLevel;
+    level++;
+  }
+
+  const root = currentLevel[0];
+  return {
+    nodes,
+    rootId: root ? root.id : null,
+    levels: level + 1,
+    timedOut: false,
+  };
+}
+
+/**
+ * Budget-exceeded fallback: build a single deterministic extractive root over
+ * all leaves and mark it low quality. Keeps a valid (if shallow) tree.
+ */
+function extractiveFallbackRoot(
+  leaves: Leaf[],
+  nodes: Map<string, RaptorNode>,
+  nextId: (level: number, index: number) => string,
+): RaptorTree {
+  const summary = summarizeCluster(leaves.flatMap((l) => l.messages));
+  const rootId = nextId(99, 0);
+  nodes.set(rootId, {
+    id: rootId,
+    level: 99,
+    parentId: null,
+    children: leaves.map((l) => l.id),
+    summary: summary.summary,
+    embedding: meanVector(leaves.map((l) => l.embedding)),
+    qualityMarker: "low",
+    tokenEstimate: summary.tokenEstimate,
+  });
+  return { nodes, rootId, levels: 2, timedOut: true };
+}

package/src/dedup/sprint12.test.ts

@@ -0,0 +1,242 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore, L2_ENABLED } from "../vectorStore.js";
+import { mmrRerank } from "./mmr.js";
+import { topK } from "./topk.js";
+import { cosineSimilarity, defaultEmbedder } from "../embedder.js";
+import { upsertCheckpoint } from "../store/sqlite.js";
+import type { StoredCheckpoint } from "../store.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-s12-"));
+let counter = 0;
+function store(opts: Record<string, unknown> = {}) {
+  const dir = join(baseTmp, `run-${counter++}`);
+  return new VectorStore({ stateDir: dir, ...opts });
+}
+
+// --- MMR diversity ---------------------------------------------------------
+
+test("mmrRerank diversifies: a cluster yields distinct-relevance results", () => {
+  const e = defaultEmbedder();
+  // Three near-identical vectors + one distinct.
+  const v = e.embed("the compiler optimized the parser hot loop");
+  const v2 = e.embed("the compiler optimized the parser hot loops"); // near-dup of v
+  const v3 = e.embed("the compiler optimized the parser hot loop now"); // near-dup of v
+  const vDistinct = e.embed("the database added a covering index for queries");
+  const items = [
+    { item: "a", vector: v, relevance: 0.9 },
+    { item: "b", vector: v2, relevance: 0.88 },
+    { item: "c", vector: v3, relevance: 0.87 },
+    { item: "d", vector: vDistinct, relevance: 0.5 },
+  ];
+  const ranked = mmrRerank(items, 2, 0.5);
+  assert.equal(ranked.length, 2);
+  // The near-dup cluster (a) and the distinct one (d) should both survive.
+  assert.ok(ranked.includes("a"));
+  assert.ok(ranked.includes("d"));
+  assert.ok(!ranked.includes("b") || !ranked.includes("c"));
+});
+
+test("mmrRerank with lambda=1 is pure relevance ranking", () => {
+  const items = [
+    { item: "low", vector: [1, 0, 0], relevance: 0.1 },
+    { item: "high", vector: [0, 1, 0], relevance: 0.9 },
+  ];
+  const ranked = mmrRerank(items, 2, 1);
+  assert.deepEqual(ranked, ["high", "low"]);
+});
+
+// --- Heap top-k ------------------------------------------------------------
+
+test("topK matches brute-force full sort on a fixture", () => {
+  const items = Array.from({ length: 1000 }, (_, i) => ({ item: i, score: Math.sin(i) * 100 + (i % 7) }));
+  for (const k of [1, 3, 10, 50]) {
+    const heap = topK(items, k).map((s) => s.item).sort((a, b) => b - a);
+    const brute = [...items].sort((a, b) => b.score - a.score).slice(0, k).map((s) => s.item).sort((a, b) => b - a);
+    assert.deepEqual(heap, brute, `topK(${k}) should match brute force`);
+  }
+});
+
+test("topK with k >= n returns all (descending by score)", () => {
+  const items = [{ item: "x", score: 1 }, { item: "y", score: 2 }];
+  assert.deepEqual(topK(items, 5).map((s) => s.item), ["y", "x"]);
+});
+
+// --- Empty-vector guard ----------------------------------------------------
+
+test("cosineSimilarity guards empty vector → 0 (no NaN)", () => {
+  assert.equal(cosineSimilarity([], [1, 2, 3]), 0);
+  assert.equal(cosineSimilarity([0, 0, 0], [1, 2, 3]), 0);
+  assert.ok(!Number.isNaN(cosineSimilarity([], [])));
+});
+
+// --- L2_ENABLED flag -------------------------------------------------------
+
+test("L2_ENABLED defaults true; search still returns hits", () => {
+  assert.equal(L2_ENABLED, true);
+  const s = store();
+  s.add({ sessionId: "sess_l2", summary: "investigated the parser", regionText: "investigated src/parser.ts and added a tokenizer", timestamp: 1 });
+  const hits = s.search("sess_l2", "src/parser.ts tokenizer", 3);
+  assert.ok(hits.length >= 1);
+});
+
+test("L2_ENABLED=false skips semantic tier but L0/L1 still work", () => {
+  const s = store({ l2Enabled: false });
+  const r1 = s.add({ sessionId: "sess_l2off", summary: "x", regionText: "the auth module validates the session token", timestamp: 1 });
+  const r2 = s.add({ sessionId: "sess_l2off", summary: "x", regionText: "the auth module validates the session token", timestamp: 2 });
+  assert.equal(r2.deduped, true); // L0 catches exact
+  assert.equal(r1.deduped, false);
+});
+
+// --- SemDeDup --------------------------------------------------------------
+
+// Seed a legacy session directly (bypassing add-time dedup tiers) so SemDeDup
+// has a redundant pair to prune — its real job is offline cleanup of data that
+// predates / escaped the online cascade.
+function seed(dir: string, sessionId: string, rows: { id: string; text: string; tok: number }[]): void {
+  const e = defaultEmbedder();
+  for (const r of rows) {
+    const cp: StoredCheckpoint = {
+      checkpointId: r.id,
+      sessionId,
+      summary: r.text,
+      keyDecisions: [],
+      nextSteps: [],
+      filesModified: [],
+      tokenEstimate: r.tok,
+      regionHash: `r-${r.id}`,
+      embedding: e.embed(r.text),
+      timestamp: 1,
+    };
+    upsertCheckpoint(cp, dir);
+  }
+}
+
+test("semDedup marks redundant near-identical rows 'removed' and search excludes them", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ stateDir: dir });
+  seed(dir, "sess_sd", [
+    { id: "chkpt_001", text: "the cache stores parsed ast nodes for fast lookup", tok: 100 },
+    { id: "chkpt_002", text: "the cache stores parsed ast nodes for fast lookup and reuse", tok: 900 },
+  ]);
+  const removed = s.semDedup("sess_sd", 0.85);
+  assert.equal(removed, 1);
+  const st = s.list("sess_sd");
+  const dropped = st.find((c) => c.dedupStatus === "removed");
+  assert.ok(dropped);
+  assert.equal(dropped.checkpointId, "chkpt_001"); // lower tokenEstimate removed
+  // Search excludes the removed row (only one active remains).
+  const hits = s.search("sess_sd", "cache parsed ast nodes", 5);
+  assert.equal(hits.length, 1);
+});
+
+test("semDedup is idempotent (re-run removes nothing new)", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const s = new VectorStore({ stateDir: dir });
+  seed(dir, "sess_sd2", [
+    { id: "chkpt_001", text: "identical region text for the dedup job now", tok: 100 },
+    { id: "chkpt_002", text: "identical region text for the dedup job right now", tok: 200 },
+  ]);
+  const first = s.semDedup("sess_sd2", 0.85);
+  const second = s.semDedup("sess_sd2", 0.85);
+  assert.equal(first, 1);
+  assert.equal(second, 0);
+});
+
+// --- HttpEmbedder (BYO localhost backend) ----------------------------------
+// Hermetic: a self-test server returns a deterministic embedding. The server
+// runs in an INDEPENDENT child process (its own event loop) so that when
+// HttpEmbedder.embed() blocks the *parent* via spawnSync, the server can still
+// accept the connection — hosting it in-process would deadlock.
+
+import { spawn, type ChildProcess } from "node:child_process";
+
+const ECHO_SERVER = String.raw`
+import { createServer } from "node:http";
+const s = createServer((req, res) => {
+  let b = "";
+  req.on("data", (c) => (b += c));
+  req.on("end", () => {
+    const input = JSON.parse(b).input || [""];
+    const text = (input[0] || "").toLowerCase().replace(/\s+/g, " ");
+    const vec = new Array(8).fill(0);
+    for (const w of text.split(" ")) {
+      let h = 0x811c9dc5;
+      for (let i = 0; i < w.length; i++) { h ^= w.charCodeAt(i); h = Math.imul(h, 0x01000193); }
+      vec[h % 8] += 1;
+    }
+    res.setHeader("content-type", "application/json");
+    res.end(JSON.stringify({ data: [{ embedding: vec }] }));
+  });
+});
+s.listen(0, "127.0.0.1", () => process.stdout.write(String(s.address().port)));
+`;
+
+// A simpler shape-only server: always returns { data: [[0.1,0.2,0.3]] }.
+const DATA_ARR_SERVER = String.raw`
+import { createServer } from "node:http";
+const s = createServer((_req, res) => {
+  res.setHeader("content-type", "application/json");
+  res.end(JSON.stringify({ data: [[0.1, 0.2, 0.3]] }));
+});
+s.listen(0, "127.0.0.1", () => process.stdout.write(String(s.address().port)));
+`;
+
+/** Spawn an independent echo server; resolves with its url once the port is up. */
+function startEchoServer(shape?: "data-arr"): Promise<{ url: string; proc: ChildProcess }> {
+  const script = shape === "data-arr" ? DATA_ARR_SERVER : ECHO_SERVER;
+  const proc = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"] });
+  return new Promise((resolve, reject) => {
+    let buf = "";
+    proc.stdout!.on("data", (d) => {
+      buf += d.toString();
+      const m = buf.match(/(\d+)/);
+      if (m) resolve({ url: `http://127.0.0.1:${m[1]}`, proc });
+    });
+    proc.on("error", reject);
+    setTimeout(() => reject(new Error("echo server did not start")), 5000);
+  });
+}
+
+test("HttpEmbedder parses OpenAI-style response and resolves dim", async () => {
+  const { url, proc } = await startEchoServer();
+  try {
+    const { HttpEmbedder } = await import("../httpEmbedder.js");
+    const emb = new HttpEmbedder({ url });
+    assert.equal(emb.dim, 0); // unknown until first embed
+    const v = emb.embed("the parser optimized the hot loop");
+    assert.equal(v.length, 8); // echo server returns 8-dim
+    assert.equal(emb.dim, 8); // resolved after first call
+    const v2 = emb.embed("the parser optimized the hot loop");
+    assert.deepEqual(v2, v); // deterministic
+  } finally {
+    proc.kill();
+  }
+});
+
+test("HttpEmbedder tolerant parser: accepts { data: [[...]] } shape", async () => {
+  const { url, proc } = await startEchoServer("data-arr");
+  try {
+    const { HttpEmbedder } = await import("../httpEmbedder.js");
+    const emb = new HttpEmbedder({ url });
+    // embed() L2-normalizes; verify the { data: [[...]] } shape parsed (dim 3)
+    // and the returned vector is the unit-normalized form of [0.1, 0.2, 0.3].
+    const v = emb.embed("x");
+    assert.equal(v.length, 3);
+    const norm = Math.sqrt(0.1 ** 2 + 0.2 ** 2 + 0.3 ** 2);
+    assert.ok(Math.abs(v[0] - 0.1 / norm) < 1e-9);
+    assert.ok(Math.abs(v[1] - 0.2 / norm) < 1e-9);
+    assert.ok(Math.abs(v[2] - 0.3 / norm) < 1e-9);
+  } finally {
+    proc.kill();
+  }
+});
+
+// --- cleanup ---------------------------------------------------------------
+
+test("Sprint 12 cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});