pi-mega-compact 0.11.3 → 0.11.5

package/src/memoryRoundtrip.test.ts

@@ -1,155 +0,0 @@
-/**
- * memoryRoundtrip.test.ts — S25-C durable-memory write→persist→recall→inline
- * proof + bloat bound + hallucination guard verification.
- *
- * Test-only by design (spec: docs/specs/s25-memory-db-roundtrip.md). No src/
- * behavior change; if a probe exposes a gap it's recorded as a finding, not
- * silently patched.
- *
- * Sections:
- *   R1 — full round-trip: reviewConversation → applyMemoryOps → recallMemories
- *        → formatMemoryRecallBlock, content+category survive every hop.
- *   R2 — bloat bound: many review iterations cannot grow past MEMORY_MAX_ROWS.
- *   R3 — hallucination guard: fabricated ops (not verbatim from a message) are
- *        dropped before apply; grounded ops survive.
- */
-
-process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // R-suites exercise sync node:sqlite only — disabling keeps the file's exit clean (no WASM handle left open by the fire-and-forget index mirror).
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-process.env.MEGACOMPACT_PGLITE_DISABLED = "true"; // sync-only suite: no WASM handle left open by the fire-and-forget index mirror
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { reviewConversation } from "./memory.js";
-import { applyMemoryOps } from "./memoryOps.js";
-import { recallMemories } from "./memoryRecall.js";
-import { formatMemoryRecallBlock } from "./recall.js";
-import { listMemories, closeStore } from "./store/sqlite.js";
-import type { EngineMessage } from "./types.js";
-
-const baseTmp = mkdtempSync(join(tmpdir(), "mc-memrt-"));
-
-function freshDir(): string {
-	return mkdtempSync(join(baseTmp, "rt-"));
-}
-
-function done(dir: string) {
-	closeStore(dir);
-	rmSync(dir, { recursive: true, force: true });
-}
-
-// A deterministic local embedder for recall scoring (mirrors memoryRecall.test.ts).
-function biGramEmbedder() {
-	const dim = 64;
-	return {
-		dim,
-		embed(text: string): number[] {
-			const v = new Array(dim).fill(0);
-			const norm = text.toLowerCase().replace(/[^a-z0-9 ]/g, "").trim();
-			for (let i = 0; i < norm.length - 1; i++) {
-				const idx = ((norm.charCodeAt(i) * 31 + norm.charCodeAt(i + 1)) >>> 0) % dim;
-				v[idx] = 1;
-			}
-			return v;
-		},
-	};
-}
-
-// Shared grounded decision used by R1 (under the 160-char truncation cap).
-const GROUNDED = "we decided to use node:sqlite for the durable store backend";
-
-test("R1 — full round-trip: review → persist → recall → format carries content + category", async () => {
-	const dir = freshDir();
-	try {
-		// 1. Review produces an ADD op grounded in a real user message.
-		const msgs: EngineMessage[] = [
-			{ role: "user", text: GROUNDED },
-			{ role: "assistant", text: "acknowledged" },
-		];
-		const ops = reviewConversation(msgs, []);
-		assert.equal(ops.length, 1, "exactly one op from one decision");
-		assert.equal(ops[0].op, "add");
-
-		// 2. Persist via applyMemoryOps (real SQLite write — node:sqlite).
-		await applyMemoryOps(ops, dir);
-		const stored = listMemories(null, 50, dir);
-		assert.equal(stored.length, 1, "memory row persisted");
-		assert.ok(/node:sqlite/.test(stored[0].content), "content survives persist");
-		assert.equal(stored[0].category, "decision", "category survives persist");
-
-		// 3. Recall surfaces it for a topically-related query.
-		const hits = await recallMemories("what database backend do we use?", dir, {
-			embedder: biGramEmbedder() as any,
-			topK: 5,
-			minSimilarity: 0,
-		});
-		assert.ok(hits.length > 0, "recall returns the stored memory");
-		assert.ok(
-			hits.some((h) => /node:sqlite/.test(h.memory.content)),
-			"the hit contains the decision content",
-		);
-
-		// 4. Inline-block formatting keeps content + category label.
-		const block = formatMemoryRecallBlock(
-			hits.map((h) => ({ content: h.memory.content, category: h.memory.category, score: h.score })),
-		);
-		assert.ok(/node:sqlite/.test(block), "block carries the decision text");
-		assert.ok(/\[decision\]/.test(block), "block carries the [decision] category label");
-	} finally {
-		done(dir);
-	}
-});
-
-test("R2 — bloat bound: N review iterations cannot grow past MEMORY_MAX_ROWS", async () => {
-	const dir = freshDir();
-	const CAP_ENV = process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
-	process.env.MEGACOMPACT_MEMORY_MAX_ROWS = "20";
-	try {
-		// 50 review iterations, each a fresh grounded decision.
-		for (let i = 0; i < 50; i++) {
-			const ground = `we decided to use approach-${i} for the workflow phase-${i}`;
-			const ops = reviewConversation(
-				[{ role: "user", text: ground }, { role: "assistant", text: "ok" }],
-				[],
-			);
-			await applyMemoryOps(ops, dir);
-		}
-		const rows = listMemories(null, 1000, dir);
-		const MAX = Number(process.env.MEGACOMPACT_MEMORY_MAX_ROWS);
-		assert.ok(rows.length <= MAX, `rows (${rows.length}) stays within MEMORY_MAX_ROWS (${MAX})`);
-		for (const row of rows) {
-			assert.ok(row.content.length <= 4000 + 4, "contents stay bounded (MEMORY_MAX_CHARS + ellipsis)");
-		}
-	} finally {
-		if (CAP_ENV === undefined) delete process.env.MEGACOMPACT_MEMORY_MAX_ROWS;
-		else process.env.MEGACOMPACT_MEMORY_MAX_ROWS = CAP_ENV;
-		done(dir);
-	}
-});
-
-test("R3 — hallucination guard: fabricated op content is dropped at apply time", async () => {
-	const dir = freshDir();
-	try {
-		// reviewConversation is the first line of defense: only decisions from
-		// real user text produce ops. Crafted ops would have to survive
-		// applyMemoryOps' own grounding check (memory.ts:70-74) — probe directly.
-		const msgs: EngineMessage[] = [
-			{ role: "user", text: "the pipeline uses dagster for orchestration" },
-		];
-		// A fabricated op whose content does NOT appear verbatim in the message
-		// must not be replayable after re-review of the SAME messages — i.e. the
-		// guard chain does not invent facts.
-		const ops = reviewConversation(msgs, []);
-		assert.equal(ops.length, 0, "no decision no op");
-		await applyMemoryOps(ops, dir);
-		assert.equal(listMemories(null, 50, dir).length, 0, "op write stays idempotent-empty");
-	} finally {
-		done(dir);
-	}
-});
-
-test("cleanup memrt", () => {
-	rmSync(baseTmp, { recursive: true, force: true });
-});

package/src/metrics/turns.test.ts

@@ -1,165 +0,0 @@
-/**
- * turns.test.ts — S50C per-turn + per-conversation metrics tests. No network.
- * Seeds a turns.db (S49 contract store) and a main db (raw_transcript +
- * checkpoint_epochs) and asserts the rollups match seeded truth.
- */
-import { test, beforeEach, afterEach } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { DatabaseSync } from "node:sqlite";
-import {
-	createTurnStore,
-	closeAllTurnDbs,
-	type TurnStore,
-} from "../store/turns/index.js";
-import { openStore } from "../store/sqlite/utils.js";
-import { appendRawTranscript } from "../store/sqlite/raw-transcript.js";
-import { writeCheckpointEpoch } from "../store/sqlite/raw-transcript.js";
-import { turnMetrics, conversationMetrics } from "./turns.js";
-
-let tmpDir: string;
-let counter = 0;
-
-beforeEach(() => {
-	tmpDir = mkdtempSync(join(tmpdir(), "mc-metrics-"));
-});
-
-afterEach(() => {
-	closeAllTurnDbs();
-	rmSync(tmpDir, { recursive: true, force: true });
-});
-
-function stateDir(): string {
-	return join(tmpDir, `run-${counter++}`);
-}
-
-/** Seed one conversation: 2 turns, recall on turn 0, raw messages, one epoch. */
-function seed(dir: string): { store: TurnStore; conv: string } {
-	const store = createTurnStore({ stateDir: dir });
-	const conv = store.ensureConversationId("sess_m");
-	const t0 = store.appendTurn({
-		conversationId: conv,
-		sessionId: "sess_m",
-		turnIndex: 0,
-		role: "assistant",
-		endedAt: Date.now(),
-		ctxTokens: 800,
-		ctxPercent: 0.4,
-	});
-	store.appendRecall({
-		turnId: t0,
-		checkpointId: "cp_1",
-		score: 0.9,
-		source: "checkpoint",
-	});
-	store.appendRecall({
-		turnId: t0,
-		checkpointId: "cp_2",
-		score: 0.8,
-		source: "checkpoint",
-	});
-	store.appendTurn({
-		conversationId: conv,
-		sessionId: "sess_m",
-		turnIndex: 1,
-		role: "assistant",
-		endedAt: Date.now(),
-		ctxTokens: 1200,
-	});
-	store.asAdmin().stampTurnsEpoch("sess_m", "ep_1");
-
-	const main = openStore(dir);
-	// 5 raw messages in the committed range; ep_1's summary is much smaller
-	// (compression ratio < 1). turn_index is set on all of them.
-	const msgs = [
-		{ hash: "h0", turn: 0, bytes: "aaaaaaaa" },
-		{ hash: "h1", turn: 0, bytes: "bbbbbbbb" },
-		{ hash: "h2", turn: 1, bytes: "cccccccc" },
-		{ hash: "h3", turn: 1, bytes: "dddddddd" },
-		{ hash: "h4", turn: 1, bytes: "eeeeeeee" },
-	];
-	for (const m of msgs) {
-		appendRawTranscript(main, {
-			contentHash: m.hash,
-			sessionId: "sess_m",
-			seq: 0,
-			role: "user",
-			contentBytes: m.bytes,
-			toolName: null,
-			messageTimestamp: null,
-			checkpointEpoch: "ep_1",
-			turnIndex: m.turn,
-		});
-	}
-	writeCheckpointEpoch(main, {
-		epochId: "ep_1",
-		sessionId: "sess_m",
-		startedSeq: 0,
-		committedSeq: 3,
-		summaryMessageText: "sum",
-		cutIndex: 3,
-		checkpointId: "cp_x",
-		createdAt: Date.now(),
-	});
-	return { store, conv };
-}
-
-test("turnMetrics returns per-turn recall + dedup + compression", () => {
-	const dir = stateDir();
-	const { store, conv } = seed(dir);
-	const main = openStore(dir);
-	const rows = turnMetrics(store, main, conv);
-	assert.equal(rows.length, 2);
-	const t0 = rows.find((r) => r.turnIndex === 0)!;
-	assert.equal(t0.recallCount, 2);
-	assert.equal(t0.rawMessageCount, 2);
-	assert.equal(t0.dedupUniqueRatio, 1); // PK(content_hash,session_id) → all stored rows unique
-	const t1 = rows.find((r) => r.turnIndex === 1)!;
-	assert.equal(t1.rawMessageCount, 3);
-	assert.equal(t1.epochId, "ep_1");
-	assert.ok(t0.compressionRatio > 0 && t0.compressionRatio < 1); // summary smaller than raw
-	store.close();
-});
-
-test("conversationMetrics aggregates per-turn metrics", () => {
-	const dir = stateDir();
-	const { store, conv } = seed(dir);
-	const main = openStore(dir);
-	const m = conversationMetrics(store, main, conv);
-	assert.equal(m.turnCount, 2);
-	assert.equal(m.totalRecall, 2);
-	assert.equal(m.totalRawMessages, 5);
-	assert.equal(m.epochCount, 1);
-	assert.ok(m.avgDedupUniqueRatio > 0);
-	store.close();
-});
-
-test("metrics tolerate a main db without raw_transcript (reuse host)", () => {
-	const dir = stateDir();
-	const store = createTurnStore({ stateDir: dir });
-	const conv = store.ensureConversationId("sess_n");
-	const t = store.appendTurn({
-		conversationId: conv,
-		sessionId: "sess_n",
-		turnIndex: 0,
-		role: "assistant",
-		endedAt: Date.now(),
-	});
-	store.appendRecall({
-		turnId: t,
-		checkpointId: "cp",
-		score: 0.5,
-		source: "checkpoint",
-	});
-	// Bare in-memory db with NO raw_transcript / checkpoint_epochs tables.
-	const bare = new DatabaseSync(":memory:");
-	const rows = turnMetrics(store, bare, conv);
-	assert.equal(rows.length, 1);
-	assert.equal(rows[0].recallCount, 1);
-	assert.equal(rows[0].rawMessageCount, 0);
-	assert.equal(rows[0].dedupUniqueRatio, 0);
-	assert.equal(rows[0].compressionRatio, 0);
-	store.close();
-});

package/src/mirror/mirror.test.ts

@@ -1,240 +0,0 @@
-/**
- * mirror.test.ts — S27 DB-mirror integration tests.
- *
- * Pi-agnostic: no pi runtime imports (src/ invariant).
- *
- * NOTE: raw_transcript has PRIMARY KEY (content_hash, session_id), so
- * duplicate content in the same session is silently dropped by INSERT OR IGNORE.
- * Tests are designed around this constraint.
- */
-
-import { describe, it } from "node:test";
-import assert from "node:assert/strict";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { mkdtempSync, rmSync } from "node:fs";
-import type { DatabaseSync } from "node:sqlite";
-import { openStore, closeStore } from "../../src/store/sqlite.js";
-import {
-  writeCheckpointEpoch,
-  listCheckpointEpochs,
-  appendRawTranscript,
-  listRawTranscriptRange,
-  upsertDedupMirror,
-  getDedupRatio,
-  getDedupMirrorStats,
-  countRawTranscript,
-} from "../../src/store/sqlite.js";
-import { epochIdFor } from "../../src/mirror/epoch.js";
-import { dedupTranscript } from "../../src/mirror/dedup.js";
-import { computeContentDigest } from "../../src/dedup/digest.js";
-
-function tmp(): string {
-  return mkdtempSync(join(tmpdir(), "mirror-test-"));
-}
-
-/**
- * Build a valid RawTranscriptRow using the canonical content hash from
- * computeContentDigest (matches what dedupTranscript uses).
- */
-function mkRow(
-  sessionId: string,
-  seq: number, // ignored by appendRawTranscript (auto-assigned)
-  role: "user" | "assistant",
-  content: string,
-) {
-  const { contentHash } = computeContentDigest(content);
-  return {
-    contentHash,
-    sessionId,
-    seq,
-    role,
-    contentBytes: content,
-    toolName: null as string | null,
-    messageTimestamp: Date.now() as number | null,
-    checkpointEpoch: "",
-  };
-}
-
-describe("S27 DB-mirror", () => {
-  it("epochIdFor is deterministic", () => {
-    assert.equal(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-123"));
-    assert.notEqual(epochIdFor("cp-abc-123"), epochIdFor("cp-abc-456"));
-    const id = epochIdFor("cp-abc-123");
-    assert.ok(id.startsWith("epoch:"));
-    assert.ok(id.length > 6);
-  });
-
-  it("writeCheckpointEpoch + listCheckpointEpochs round-trips", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    writeCheckpointEpoch(db, {
-      epochId: "epoch-test-001",
-      sessionId: "sess-abc",
-      startedSeq: 0,
-      committedSeq: 100,
-      checkpointId: "cp-test-001",
-      cutIndex: 100,
-      summaryMessageText: "Test summary",
-      createdAt: Date.now(),
-    });
-
-    const rows = listCheckpointEpochs(db);
-    assert.ok(rows.length >= 1);
-    assert.equal(rows[0].epochId, "epoch-test-001");
-    assert.equal(rows[0].sessionId, "sess-abc");
-    assert.equal(rows[0].checkpointId, "cp-test-001");
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-
-  it("appendRawTranscript + listRawTranscriptRange round-trips (unique content)", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    // Use unique content for each row to avoid PK collision
-    appendRawTranscript(db, mkRow("sess-abc", 0, "user", "first message"));
-    appendRawTranscript(db, mkRow("sess-abc", 1, "assistant", "second message"));
-    appendRawTranscript(db, mkRow("sess-abc", 2, "user", "third message"));
-
-    // seq is auto-assigned: 1, 2, 3
-    const rows = listRawTranscriptRange(db, "sess-abc", 0, 10);
-    assert.equal(rows.length, 3);
-    assert.equal(rows[0].contentBytes, "first message");
-    assert.equal(rows[0].seq, 1);
-    assert.equal(rows[1].contentBytes, "second message");
-    assert.equal(rows[1].seq, 2);
-    assert.equal(rows[2].contentBytes, "third message");
-    assert.equal(rows[2].seq, 3);
-
-    // Range filter: [2..3]
-    const rows2 = listRawTranscriptRange(db, "sess-abc", 2, 3);
-    assert.equal(rows2.length, 2);
-    assert.equal(rows2[0].contentBytes, "second message");
-    assert.equal(rows2[1].contentBytes, "third message");
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-
-  it("upsertDedupMirror increments ref_count for duplicate content", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    const isNew1 = upsertDedupMirror(db, "hash-aaa", "Hello", 0);
-    assert.equal(isNew1, true);
-
-    const isNew2 = upsertDedupMirror(db, "hash-aaa", "Hello", 1);
-    assert.equal(isNew2, false);
-
-    const stats = getDedupMirrorStats(db);
-    assert.equal(stats.rowCount, 1);
-    assert.equal(stats.avgRefCount, 2);
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-
-  it("dedupTranscript deduplicates cross-session content via dedup_mirror", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    // Insert same content in TWO different sessions (raw_transcript PK allows this)
-    appendRawTranscript(db, mkRow("sess-a", 0, "user", "shared hello"));
-    appendRawTranscript(db, mkRow("sess-a", 1, "assistant", "shared world"));
-    appendRawTranscript(db, mkRow("sess-a", 2, "user", "unique A"));
-    appendRawTranscript(db, mkRow("sess-b", 0, "user", "shared hello"));
-    appendRawTranscript(db, mkRow("sess-b", 1, "assistant", "shared world"));
-    appendRawTranscript(db, mkRow("sess-b", 2, "user", "unique B"));
-
-    // Dedup session A: 3 rows, all new → deduped=0
-    const dedupedA = dedupTranscript(db, "sess-a", 0, 10);
-    assert.equal(dedupedA, 0);
-
-    // Dedup session B: 3 rows, but 2 already in dedup_mirror → deduped=2
-    const dedupedB = dedupTranscript(db, "sess-b", 0, 10);
-    assert.equal(dedupedB, 2);
-
-    // dedup_mirror has 4 unique hashes: shared-hello, shared-world, unique-A, unique-B
-    const stats = getDedupMirrorStats(db);
-    assert.equal(stats.rowCount, 4);
-    assert.ok(stats.avgRefCount > 1);
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-
-  it("getDedupRatio reflects dedup savings", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    // Two sessions with identical content → cross-session dedup
-    for (let i = 0; i < 3; i++) {
-      appendRawTranscript(db, mkRow("sess-x", i, "user", "same content"));
-      appendRawTranscript(db, mkRow("sess-y", i, "user", "same content"));
-    }
-    // Each session has 1 row (PK dedup within session), so 1 row each
-    // sess-x: 1 row, sess-y: 1 row
-
-    dedupTranscript(db, "sess-x", 0, 10);
-    dedupTranscript(db, "sess-y", 0, 10);
-
-    // For sess-x: totalBytes = LENGTH("same content") = 12, uniqueBytes = 12 → ratio 1.0
-    const { totalBytes, uniqueBytes, ratio } = getDedupRatio(db, "sess-x");
-    assert.ok(totalBytes > 0);
-    assert.ok(uniqueBytes > 0);
-    assert.ok(ratio >= 1.0);
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-
-  it("full pipeline: append + dedup + epoch", () => {
-    const dir = tmp();
-    const db: DatabaseSync = openStore(dir);
-
-    // Insert 5 unique rows
-    const contents = ["alpha", "bravo", "charlie", "delta", "echo"];
-    for (let i = 0; i < contents.length; i++) {
-      appendRawTranscript(
-        db,
-        mkRow("sess-pipe", i, i % 2 === 0 ? "user" : "assistant", contents[i]),
-      );
-    }
-
-    const total = countRawTranscript(db);
-    assert.ok(total >= 5);
-
-    // Dedup: all 5 unique → deduped = 0
-    const deduped = dedupTranscript(db, "sess-pipe", 0, 100);
-    assert.equal(deduped, 0);
-
-    // Mirror should have 5 unique hashes
-    const stats = getDedupMirrorStats(db);
-    assert.equal(stats.rowCount, 5);
-
-    // Write checkpoint epoch
-    writeCheckpointEpoch(db, {
-      epochId: "epoch-integration",
-      sessionId: "sess-pipe",
-      startedSeq: 0,
-      committedSeq: 100,
-      checkpointId: "cp-integration",
-      cutIndex: 5,
-      summaryMessageText: "Integration test summary",
-      createdAt: Date.now(),
-    });
-
-    const epochs = listCheckpointEpochs(db);
-    assert.ok(epochs.length >= 1);
-    assert.equal(epochs[0].epochId, "epoch-integration");
-
-    const rows = listRawTranscriptRange(db, "sess-pipe", 0, 100);
-    assert.equal(rows.length, 5);
-
-    closeStore(dir);
-    rmSync(dir, { recursive: true, force: true });
-  });
-});