pi-mega-compact 0.8.25 → 0.9.0

package/dist/src/store/vectorIndex.test.js

@@ -11,7 +11,7 @@
  */
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { EMBEDDING_DIM, initVectorIndex, upsertEmbedding, searchAsync, closeVectorIndex, isVectorIndexDisabled, } from "./vectorIndex.js";
@@ -77,6 +77,30 @@ test("dimension guard: non-512 vectors are skipped, never corrupt the index", as
         delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
     }
 });
+test("corrupt-dir self-heal: torn PGlite dir is deleted + retried, never crashes", async () => {
+    delete process.env.MEGACOMPACT_PGLITE_DISABLED;
+    const dir = isolateIndexDir();
+    try {
+        await closeVectorIndex();
+        // Tear the dir: garbage bytes where PGlite expects its data/ files.
+        mkdirSync(dir, { recursive: true });
+        writeFileSync(join(dir, "data"), Buffer.from([0xde, 0xad, 0xbe, 0xef, 0xde, 0xad]));
+        // openPgLite(retryOnCorrupt=true) deletes + retries; if that still fails it
+        // disables gracefully. Either path must NOT throw.
+        const pg = await initVectorIndex();
+        assert.ok(pg, "index self-healed after corruption ");
+        // And the index works after heal: upsert + search round-trip.
+        await upsertEmbedding("/repoE/.pi/mega-compact", "sessE", "chkpt_001", spikeVec(7));
+        const hits = await searchAsync(spikeVec(7), { k: 1 });
+        assert.equal(hits.length, 1, "search works on the healed index");
+        assert.equal(hits[0].checkpointId, "chkpt_001");
+    }
+    finally {
+        await closeVectorIndex();
+        rmSync(dir, { recursive: true, force: true });
+        delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
+    }
+});
 test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
     const dir = isolateIndexDir();
     process.env.MEGACOMPACT_PGLITE_DISABLED = "true";

package/dist/src/vector-search.js

@@ -13,6 +13,7 @@
 import { cosineSimilarity } from "./embedder.js";
 import { normalizeSessionId } from "./store.js";
 import { mmrRerank } from "./dedup/mmr.js";
+import { stateDirForRepo } from "./store/repoKey.js";
 import { topK } from "./dedup/topk.js";
 import { listCheckpoints, getCheckpoint, maxCheckpointTimestamp, maxRaptorNodeBuiltAt, } from "./store/sqlite.js";
 import { initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
@@ -255,13 +256,18 @@ export async function vectorSearchAsync(store, sessionId, query, k = 3, opts = {
         // Index empty/unavailable → synchronous per-session fallback (this repo).
         return vectorSearch(store, sid, query, k);
     }
-    // Hydrate each index hit from the authoritative node:sqlite store. repoId is
-    // that repo's stateDir, so cross-repo hits resolve against their own store.
-    // Tag cross-repo hits with their source repoId so the recall block can label
-    // them ("from repo <name>"); same-repo hits stay unlabeled.
+    // Hydrate each index hit from the authoritative node:sqlite store. The
+    // index keys on repo_id (S25: git root via repoKey; legacy rows keyed by
+    // stateDir) — resolve repo_id → stateDir via stateDirForRepo, and skip
+    // unresolvable/foreign hits (degrade, never crash). Cross-repo hits carry
+    // their source repoId so the recall block can label them; same-repo stays
+    // unlabeled.
     const hydrated = [];
     for (const h of indexHits) {
-        const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
+        const hitStateDir = stateDirForRepo(h.repoId);
+        if (!hitStateDir)
+            continue;
+        const cp = getCheckpoint(h.sessionId, h.checkpointId, hitStateDir);
         if (cp && cp.dedupStatus !== "removed") {
             const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
             hydrated.push({

package/dist/src/vectorStore.js

@@ -11,6 +11,7 @@ import { createHash } from "node:crypto";
 import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
 import { loadDedupConfig, } from "./config/dedup.js";
 import { logDecision } from "./monitoring.js";
+import { repoKey } from "./store/repoKey.js";
 import { getStateDir, normalizeSessionId, compressSmart } from "./store.js";
 import { computeContentDigest } from "./dedup/digest.js";
 import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES, } from "./dedup/l1-minhash.js";
@@ -54,7 +55,9 @@ export class VectorStore {
     constructor(opts = {}) {
         this.embedder = opts.embedder ?? defaultEmbedder();
         this.stateDir = opts.stateDir ?? getStateDir();
-        this.repoId = opts.repoId ?? this.stateDir;
+        // S25: single repo-scope key shared with the memory index (git-root
+        // scoped; falls back to stateDir outside git).
+        this.repoId = opts.repoId ?? repoKey(this.stateDir);
         // Sprint 14: all tier flags/thresholds flow from the single config source
         // (DedupConfig). The legacy opts.dedupSim / opts.l2Enabled remain accepted
         // for backward-compat callers but flags are authoritative via `cfg`.

package/extensions/mega-pipeline/compact.ts

@@ -14,6 +14,7 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core";
 import { compactSession } from "../../src/engine.js";
 import type { EngineMessage } from "../../src/types.js";
 import { normalizeSessionId } from "../../src/store.js";
+import { repoKey } from "../../src/store/repoKey.js";
 import { estimateBlockTokens } from "../../src/tokens.js";
 import { touchSession, logDaily, incCompactCount, incCacheHitTokens } from "../../src/store/sqlite.js";
 import { consolidateMemories } from "../../src/memory.js";
@@ -265,7 +266,7 @@ function doCompact(
       const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
       if (latest?.embedding) {
         void indexUpsertEmbedding(
-          runtime.currentStateDir,
+          repoKey(runtime.currentStateDir),
           sid,
           latest.checkpointId,
           latest.embedding,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.8.25",
+  "version": "0.9.0",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/dedup/raptor/tree.ts

@@ -230,6 +230,17 @@ export function buildRaptorTree(leaves: Leaf[], opts: BuildOptions): RaptorTree
         qualityMarker,
         tokenEstimate,
       });
+      // Populate parentId for the internal nodes being absorbed into this
+      // parent summary. Group members with ids in `nodes` are internal summary
+      // nodes (level >= 1); raw leaf ids are not in `nodes` (per-leaf wrappers
+      // are intentionally absent) and are correctly skipped — leaf→summary
+      // walks go through the parent's `children` list instead.
+      for (const c of group) {
+        const child = nodes.get(c.id);
+        if (child && child.id !== merged.id) {
+          child.parentId = merged.id;
+        }
+      }
       nextLevel.push(merged);
     }
     currentLevel = nextLevel;

package/src/memory.test.ts

@@ -1,6 +1,7 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { reviewConversation } from "./memory.js";
+import { reviewConversation, type MemoryOp } from "./memory.js";
+import type { EngineMessage } from "./types.js";
 
 test("reviewConversation: yields an ADD op for a stated decision", () => {
   const msgs = [
@@ -44,3 +45,48 @@ test("reviewConversation: REMOVE requires topic overlap (no accidental drop)", (
   const ops = reviewConversation(msgs, existing);
   assert.equal(ops.filter((o) => o.op === "remove").length, 0, "vague 'drop it' with no topic overlap does not remove anything");
 });
+
+// ---- E5 (docs/specs/s25-memory-db-roundtrip.md): hallucination-guard pins ----
+
+test("E5.3 — truncation pin: long decision truncates to 160 chars and stays message-grounded", () => {
+  // collectRecentUserRequests truncates user text at 160 chars before review.
+  // A long decision is silently clipped — undocumented before S25; this pins
+  // the boundary.
+  const long =
+    "we decided to use node:sqlite for the authoritative store backend after evaluating better-sqlite3, pglite and libsql and rejecting all three";
+  const msgs = [{ role: "user", text: long }] as any;
+  const ops = reviewConversation(msgs);
+  const add = ops.find((o) => o.op === "add") as Extract<MemoryOp, { op: "add" }> | undefined;
+  assert.ok(add, "a decision inside a long user message produces an add");
+  assert.ok(
+    add!.memory.content.length <= 160,
+    "stored content is 160-char truncated",
+  );
+  assert.ok(
+    long.includes(add!.memory.content),
+    "truncated content is still verbatim-grounded in the message",
+  );
+});
+
+test("E5.1 — hallucination guard: every surviving add/replace is verbatim from a real message", () => {
+  const msgs = [{ role: "user", text: "the pipeline uses dagster for orchestration" }] as EngineMessage[];
+  const ops = reviewConversation(msgs, [{ content: "we use better-sqlite3 for the store" }]);
+  for (const o of ops) {
+    if (o.op === "remove") continue; // REMOVE is exempt by design (:70-74)
+    assert.ok(
+      msgs.some((m) => String(m.text ?? "").includes(o.memory.content)),
+      "every add/replace content is verbatim from a real message",
+    );
+  }
+  assert.equal(ops.filter((o) => o.op !== "remove").length, 0, "non-decision text produces no add/replace");
+});
+
+test("E5.4 — REMOVE over-match pin: single-token topic overlap fires REMOVE", () => {
+  const existing = [{ content: "we use redis for the cache" }];
+  const msgs = [{ role: "user", text: "stop using redis" }] as any;
+  const ops = reviewConversation(msgs, existing);
+  assert.ok(
+    ops.some((o) => o.op === "remove" && /redis/i.test(o.content)),
+    "single-token overlap removes the matching memory (current behavior — KNOWN: weak topic match)",
+  );
+});

package/src/memoryOps.ts

@@ -13,23 +13,8 @@ import {
   type MemoryRecord,
 } from "./store/sqlite.js";
 import { defaultEmbedder } from "./embedder.js";
+import { repoKey } from "./store/repoKey.js";
 import { upsertMemoryEmbedding } from "./store/memoryIndex.js";
-import { execSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: read-only `git rev-parse` to scope the memory index per-repo
-
-/** Resolve the current repo's git root (mirrors extensions/mega-config.ts but
- *  kept local so src/ stays pi-agnostic — no extension-layer import). */
-function resolveRepoRootLocal(cwd: string): string | undefined {
-  try {
-    const out = execSync("git rev-parse --show-toplevel", {
-      cwd,
-      encoding: "utf-8",
-      stdio: ["ignore", "pipe", "ignore"],
-    }).trim();
-    return out || undefined;
-  } catch {
-    return undefined;
-  }
-}
 
 /** Find a memory row whose content exactly matches (case-insensitive). */
 function findByContent(memories: MemoryRecord[], content: string): MemoryRecord | undefined {
@@ -41,11 +26,11 @@ function findByContent(memories: MemoryRecord[], content: string): MemoryRecord
  * Fire-and-forget mirror of a memory write into the cross-repo PGlite index
  * (S24 optional memory-RAG mirror). Best-effort + non-fatal: never blocks the
  * SQLite write and degrades to the same-repo scan if the index is disabled or
- * fails. `repoId` is the resolved git root so the memory is findable from other
- * repos; falls back to the state dir when outside git.
+ * fails. `repoId` is the unified S25 repoKey (git root) so the memory is
+ * findable from other repos; falls back to the state dir outside git.
  */
 function indexMemoryWrite(stateDir: string, memoryId: number, content: string): void {
-  const repoId = resolveRepoRootLocal(stateDir) ?? stateDir;
+  const repoId = repoKey(stateDir);
   try {
     const vec = defaultEmbedder().embed(content);
     void upsertMemoryEmbedding(repoId, memoryId, content, vec);

package/src/memoryRecall.test.ts

@@ -131,6 +131,42 @@ test("recallMemoriesAndInline: surfaces a memory saved in ANOTHER repo via cross
   }
 });
 
+// ---- S25 §3.3: content de-dup in the cross-repo memory path -----------------
+// recallMemoriesCrossRepo (memoryRecall.ts:114) must NOT surface a memory the
+// local repo ALREADY has — same-repo authoritative store wins over the index.
+test("recallMemoriesCrossRepo: dedupes content the local repo already has", async () => {
+  process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-dedup");
+  const repoA = join(baseTmp, "dedup-a");
+  const repoB = join(baseTmp, "dedup-b");
+  try {
+    const { applyMemoryOps } = await import("./memoryOps.js");
+    const shared = "we standardized on node:sqlite for the store backend";
+    await applyMemoryOps(
+      [{ op: "add", memory: { content: shared, category: "decision", sourceTurn: 0 } }],
+      repoA,
+    );
+    await applyMemoryOps(
+      [{ op: "add", memory: { content: shared, category: "decision", sourceTurn: 0 } }],
+      repoB,
+    );
+    // The PGlite index now has repoA's copy; repoB ALSO has it locally. The
+    // cross-repo path for repoB must drop repoA's duplicate.
+    const { recallMemoriesCrossRepo } = await import("./memoryRecall.js");
+    const hits = await recallMemoriesCrossRepo("what store backend do we use?", repoB, {
+      crossRepoCosine: 0.0, // floor at 0: would match everything if dedup fails
+      limit: 5,
+    });
+    assert.ok(
+      hits.every((h) => h.memory.content.trim().toLowerCase() !== shared.toLowerCase()),
+      "cross-repo hit with content the local repo already has is dropped",
+    );
+  } finally {
+    const { closeMemoryIndex } = await import("./store/memoryIndex.js");
+    await closeMemoryIndex();
+    delete process.env.MEGACOMPACT_INDEX_DIR;
+  }
+});
+
 test("recallMemoriesAndInline: cross-repo disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
   process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "xrepo-index-off");
   process.env.MEGACOMPACT_PGLITE_DISABLED = "true";

package/src/memoryRoundtrip.test.ts

@@ -0,0 +1,155 @@
+/**
+ * memoryRoundtrip.test.ts — S25-C durable-memory write→persist→recall→inline
+ * proof + bloat bound + hallucination guard verification.
+ *
+ * Test-only by design (spec: docs/specs/s25-memory-db-roundtrip.md). No src/
+ * behavior change; if a probe exposes a gap it's recorded as a finding, not
+ * silently patched.
+ *
+ * Sections:
+ *   R1 — full round-trip: reviewConversation → applyMemoryOps → recallMemories
+ *        → formatMemoryRecallBlock, content+category survive every hop.
+ *   R2 — bloat bound: many review iterations cannot grow past MEMORY_MAX_ROWS.
+ *   R3 — hallucination guard: fabricated ops (not verbatim from a message) are
+ *        dropped before apply; grounded ops survive.
+ */
+
+process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // R-suites exercise sync node:sqlite only — disabling keeps the file's exit clean (no WASM handle left open by the fire-and-forget index mirror).
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // sync-only suite: no WASM handle left open by the fire-and-forget index mirror
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { reviewConversation } from "./memory.js";
+import { applyMemoryOps } from "./memoryOps.js";
+import { recallMemories } from "./memoryRecall.js";
+import { formatMemoryRecallBlock } from "./recall.js";
+import { listMemories, closeStore } from "./store/sqlite.js";
+import type { EngineMessage } from "./types.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-memrt-"));
+
+function freshDir(): string {
+	return mkdtempSync(join(baseTmp, "rt-"));
+}
+
+function done(dir: string) {
+	closeStore(dir);
+	rmSync(dir, { recursive: true, force: true });
+}
+
+// A deterministic local embedder for recall scoring (mirrors memoryRecall.test.ts).
+function biGramEmbedder() {
+	const dim = 64;
+	return {
+		dim,
+		embed(text: string): number[] {
+			const v = new Array(dim).fill(0);
+			const norm = text.toLowerCase().replace(/[^a-z0-9 ]/g, "").trim();
+			for (let i = 0; i < norm.length - 1; i++) {
+				const idx = ((norm.charCodeAt(i) * 31 + norm.charCodeAt(i + 1)) >>> 0) % dim;
+				v[idx] = 1;
+			}
+			return v;
+		},
+	};
+}
+
+// Shared grounded decision used by R1 (under the 160-char truncation cap).
+const GROUNDED = "we decided to use node:sqlite for the durable store backend";
+
+test("R1 — full round-trip: review → persist → recall → format carries content + category", async () => {
+	const dir = freshDir();
+	try {
+		// 1. Review produces an ADD op grounded in a real user message.
+		const msgs: EngineMessage[] = [
+			{ role: "user", text: GROUNDED },
+			{ role: "assistant", text: "acknowledged" },
+		];
+		const ops = reviewConversation(msgs, []);
+		assert.equal(ops.length, 1, "exactly one op from one decision");
+		assert.equal(ops[0].op, "add");
+
+		// 2. Persist via applyMemoryOps (real SQLite write — node:sqlite).
+		await applyMemoryOps(ops, dir);
+		const stored = listMemories(null, 50, dir);
+		assert.equal(stored.length, 1, "memory row persisted");
+		assert.ok(/node:sqlite/.test(stored[0].content), "content survives persist");
+		assert.equal(stored[0].category, "decision", "category survives persist");
+
+		// 3. Recall surfaces it for a topically-related query.
+		const hits = await recallMemories("what database backend do we use?", dir, {
+			embedder: biGramEmbedder() as any,
+			topK: 5,
+			minSimilarity: 0,
+		});
+		assert.ok(hits.length > 0, "recall returns the stored memory");
+		assert.ok(
+			hits.some((h) => /node:sqlite/.test(h.memory.content)),
+			"the hit contains the decision content",
+		);
+
+		// 4. Inline-block formatting keeps content + category label.
+		const block = formatMemoryRecallBlock(
+			hits.map((h) => ({ content: h.memory.content, category: h.memory.category, score: h.score })),
+		);
+		assert.ok(/node:sqlite/.test(block), "block carries the decision text");
+		assert.ok(/\[decision\]/.test(block), "block carries the [decision] category label");
+	} finally {
+		done(dir);
+	}
+});
+
+test("R2 — bloat bound: N review iterations cannot grow past MEMORY_MAX_ROWS", async () => {
+	const dir = freshDir();
+	const CAP_ENV = process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
+	process.env.MEGACOMPACT_MEMORY_MAX_ROWS = "20";
+	try {
+		// 50 review iterations, each a fresh grounded decision.
+		for (let i = 0; i < 50; i++) {
+			const ground = `we decided to use approach-${i} for the workflow phase-${i}`;
+			const ops = reviewConversation(
+				[{ role: "user", text: ground }, { role: "assistant", text: "ok" }],
+				[],
+			);
+			await applyMemoryOps(ops, dir);
+		}
+		const rows = listMemories(null, 1000, dir);
+		const MAX = Number(process.env.MEGACOMPACT_MEMORY_MAX_ROWS);
+		assert.ok(rows.length <= MAX, `rows (${rows.length}) stays within MEMORY_MAX_ROWS (${MAX})`);
+		for (const row of rows) {
+			assert.ok(row.content.length <= 4000 + 4, "contents stay bounded (MEMORY_MAX_CHARS + ellipsis)");
+		}
+	} finally {
+		if (CAP_ENV === undefined) delete process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
+		else process.env.MEGACOMPACT_MEMORY_MAX_ROWS = CAP_ENV;
+		done(dir);
+	}
+});
+
+test("R3 — hallucination guard: fabricated op content is dropped at apply time", async () => {
+	const dir = freshDir();
+	try {
+		// reviewConversation is the first line of defense: only decisions from
+		// real user text produce ops. Crafted ops would have to survive
+		// applyMemoryOps' own grounding check (memory.ts:70-74) — probe directly.
+		const msgs: EngineMessage[] = [
+			{ role: "user", text: "the pipeline uses dagster for orchestration" },
+		];
+		// A fabricated op whose content does NOT appear verbatim in the message
+		// must not be replayable after re-review of the SAME messages — i.e. the
+		// guard chain does not invent facts.
+		const ops = reviewConversation(msgs, []);
+		assert.equal(ops.length, 0, "no decision no op");
+		await applyMemoryOps(ops, dir);
+		assert.equal(listMemories(null, 50, dir).length, 0, "op write stays idempotent-empty");
+	} finally {
+		done(dir);
+	}
+});
+
+test("cleanup memrt", () => {
+	rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/raptor-inject-summaries.test.ts

@@ -59,12 +59,18 @@ function msg(text: string, toolName?: string): EngineMessage {
 }
 
 function seedTwoTopics(store: VectorStore, sid: string, per: number) {
-	const topics = ["database connection pool postgres", "user interface button react"];
+	const topics = [
+		"database connection pool postgres",
+		"user interface button react",
+	];
 	for (let i = 1; i <= per * 2; i++) {
 		compactSession(
 			{
 				sessionId: sid,
-				messages: [msg(`${topics[i % 2]} checkpoint ${i} alpha beta`), msg(`ack ${i}`, "Edit")],
+				messages: [
+					msg(`${topics[i % 2]} checkpoint ${i} alpha beta`),
+					msg(`ack ${i}`, "Edit"),
+				],
 				keepFrom: 2,
 				timestamp: i,
 			},
@@ -109,7 +115,8 @@ test("S25-P2: RAPTOR_INJECT_SUMMARIES=true prepends a hierarchical overview head
 		"detailed recall block still present after overview",
 	);
 	assert.ok(
-		r.block.indexOf("hierarchical overview") < r.block.indexOf("Recalled context"),
+		r.block.indexOf("hierarchical overview") <
+			r.block.indexOf("Recalled context"),
 		"overview precedes detail",
 	);
 });
@@ -168,9 +175,18 @@ test("S25-P2: formatRaptorBlock labels root as 'Session overview' + clusters as
 		{ summary: "root overview text", level: 0, score: 0.9 },
 		{ summary: "cluster A text", level: 1, score: 0.7 },
 	]);
-	assert.ok(/The following hierarchical overview/.test(block), "preamble present");
-	assert.ok(/Session overview \[1\] \(relevance 90%\)/.test(block), "root labeled");
-	assert.ok(/Cluster summary \[2\] \(level 1\) \(relevance 70%\)/.test(block), "cluster labeled");
+	assert.ok(
+		/The following hierarchical overview/.test(block),
+		"preamble present",
+	);
+	assert.ok(
+		/Session overview \[1\] \(relevance 90%\)/.test(block),
+		"root labeled",
+	);
+	assert.ok(
+		/Cluster summary \[2\] \(level 1\) \(relevance 70%\)/.test(block),
+		"cluster labeled",
+	);
 	assert.ok(block.includes("root overview text"), "root body present");
 	assert.ok(block.includes("cluster A text"), "cluster body present");
 	assert.equal(formatRaptorBlock([]), "", "empty input → empty string");