pi-mega-compact 0.13.1 → 0.13.2

package/src/memoryGraph-gates.test/_helpers.ts

@@ -0,0 +1,164 @@
+/**
+ * Shared helpers for the memoryGraph-gates split files.
+ * Extracted from src/memoryGraph-gates.test.ts: temp dirs, node/edge/ws
+ * builders, withEnv, and populateStore (real SQLite seeding).
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { MemoryGraphNode, MemoryGraphEdge } from "../memoryGraph.js";
+import type { GraphWorkingSet } from "../memoryGraph/gates.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { upsertCheckpoint } from "../store/sqlite/checkpoints.js";
+import { recordTurn, newConversationId } from "../store/sqlite/turns.js";
+import { appendRawTranscript } from "../store/sqlite/raw-transcript.js";
+import type { RawTranscriptRow } from "../store/sqlite/raw-transcript.js";
+
+export const baseTmp = mkdtempSync(join(tmpdir(), "mc-mgraph-gates-"));
+let counter = 0;
+
+export function tmpDir(): string {
+  return join(baseTmp, `run-${counter++}`);
+}
+
+export function n(
+  id: string,
+  overrides: Partial<MemoryGraphNode> = {},
+): MemoryGraphNode {
+  return {
+    id,
+    sessionId: "sess_test",
+    label: id,
+    summaryTruncated: "",
+    tokenEstimate: 0,
+    timestamp: 1000,
+    dedupStatus: undefined,
+    raptorLevel: 0,
+    topicSummary: undefined,
+    decisionCount: 0,
+    textSnippet: "",
+    nodeType: "checkpoint",
+    epochId: undefined,
+    ...overrides,
+  };
+}
+
+export function e(
+  source: string,
+  target: string,
+  overrides: Partial<MemoryGraphEdge> = {},
+): MemoryGraphEdge {
+  return { source, target, weight: 1.0, type: "temporal", ...overrides };
+}
+
+export function ws(
+  nodes: MemoryGraphNode[],
+  edges: MemoryGraphEdge[] = [],
+): GraphWorkingSet {
+  return { nodes, edges };
+}
+
+/** Save and restore env vars around a test. */
+export function withEnv(
+  env: Record<string, string | undefined>,
+  fn: () => void,
+): void {
+  const saved: Record<string, string | undefined> = {};
+  for (const k of Object.keys(env)) {
+    saved[k] = process.env[k];
+    if (env[k] === undefined) {
+      delete process.env[k];
+    } else {
+      process.env[k] = env[k]!;
+    }
+  }
+  try {
+    fn();
+  } finally {
+    for (const k of Object.keys(env)) {
+      if (saved[k] === undefined) {
+        delete process.env[k];
+      } else {
+        process.env[k] = saved[k];
+      }
+    }
+  }
+}
+
+export function populateStore(
+  dir: string,
+  sessionId: string,
+  sources: { checkpoint?: boolean; turn?: boolean; turnContent?: boolean; memory?: boolean },
+): void {
+  const db = openStore(dir);
+
+  if (sources.checkpoint) {
+    upsertCheckpoint(
+      {
+        checkpointId: "chkpt_001",
+        sessionId,
+        regionHash: "abc",
+        contentHash: "def",
+        normalizedText: "checkpoint one",
+        summary: "Checkpoint One",
+        tokenEstimate: 50,
+        timestamp: 1000,
+        embedding: [],
+        keyDecisions: [],
+        nextSteps: [],
+        filesModified: [],
+      },
+      dir,
+    );
+    upsertCheckpoint(
+      {
+        checkpointId: "chkpt_002",
+        sessionId,
+        regionHash: "ghi",
+        contentHash: "jkl",
+        normalizedText: "checkpoint two",
+        summary: "Checkpoint Two",
+        tokenEstimate: 60,
+        timestamp: 2000,
+        embedding: [],
+        keyDecisions: [],
+        nextSteps: [],
+        filesModified: [],
+      },
+      dir,
+    );
+  }
+
+  if (sources.turn) {
+    const convId = newConversationId();
+    recordTurn(
+      { conversationId: convId, sessionId, turnIndex: 0, role: "user", endedAt: 1500, ctxTokens: 100 },
+      dir,
+    );
+    recordTurn(
+      { conversationId: convId, sessionId, turnIndex: 1, role: "assistant", endedAt: 2500, ctxTokens: 200 },
+      dir,
+    );
+  }
+
+  if (sources.turnContent) {
+    const rows: RawTranscriptRow[] = [
+      { sessionId, seq: 0, role: "user", contentBytes: "hello from the user", contentHash: "hash_0", toolName: null, messageTimestamp: 1500, checkpointEpoch: "", turnIndex: 0 },
+      { sessionId, seq: 1, role: "assistant", contentBytes: "response from assistant", contentHash: "hash_1", toolName: null, messageTimestamp: 2500, checkpointEpoch: "", turnIndex: 1 },
+    ];
+    for (const row of rows) {
+      appendRawTranscript(db, row);
+    }
+  }
+
+  if (sources.memory) {
+    db.prepare(
+      `INSERT OR IGNORE INTO memories (content, kind, created_at, source_turn)
+       VALUES (?, ?, ?, ?)`,
+    ).run("remembered fact", "fact", 2000, 0);
+    db.prepare(
+      `INSERT OR IGNORE INTO memories (content, kind, created_at, source_turn)
+       VALUES (?, ?, ?, ?)`,
+    ).run("user preference", "preference", 3000, 1);
+  }
+}

package/src/ratio.bench.test/_helpers.ts

@@ -0,0 +1,268 @@
+/**
+ * Shared helpers for the ratio benchmark split test files.
+ *
+ * Extracted from ratio.bench.test.ts: message builders, tmp-dir lifecycle,
+ * and the generateMessages / generateRealisticConversation / generateNearDuplicates
+ * fixtures used across all benchmark describe blocks.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import type { EngineMessage } from "../types.js";
+
+export function makeMsg(role: EngineMessage["role"], text: string): EngineMessage {
+  return { role, text };
+}
+
+export function makeDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), "ratio-bench-"));
+}
+
+/** Generate N messages with controlled repetition. */
+export function generateMessages(
+  n: number,
+  opts: {
+    pattern: "unique" | "repetitive" | "mixed" | "code-review" | "debug-session";
+  },
+): EngineMessage[] {
+  const msgs: EngineMessage[] = [];
+  const templates = [
+    "I'm looking at the {file} module. The {component} needs refactoring because {reason}.",
+    "Found a bug in {file}: the {component} doesn't handle {case} correctly. Here's the fix: {fix}",
+    "Let me check the {file} implementation. The {component} uses {pattern} pattern which is fine for now.",
+    "Updated {file} to fix the {component} issue. The {reason} was causing {case}.",
+    "Running tests on {file}. The {component} test covers {case} and {fix}. Looks good.",
+    "The {component} in {file} needs to handle {case}. Currently it just {fix}.",
+    "Reviewed the {file} changes. The {component} refactor looks solid. {reason}.",
+    "Deployed the {component} fix to staging. {file} is now handling {case} correctly.",
+  ];
+
+  const files = [
+    "src/engine.ts",
+    "src/store.ts",
+    "src/vectorStore.ts",
+    "src/compact.ts",
+    "src/recall.ts",
+  ];
+  const components = [
+    "compression",
+    "dedup",
+    "embedding",
+    "search",
+    "checkpoint",
+    "supersede",
+    "recall",
+  ];
+  const reasons = [
+    "performance",
+    "correctness",
+    "maintainability",
+    "edge case handling",
+    "type safety",
+  ];
+  const cases = [
+    "empty input",
+    "large payloads",
+    "concurrent access",
+    "unicode content",
+    "timeout",
+  ];
+  const fixes = [
+    "added bounds checking",
+    "refactored the loop",
+    "added error handling",
+    "simplified the logic",
+    "added unit tests",
+  ];
+  const patterns = ["singleton", "observer", "strategy", "factory", "builder"];
+
+  for (let i = 0; i < n; i++) {
+    const template = templates[i % templates.length];
+    const fill = (s: string): string =>
+      s
+        .replace("{file}", files[i % files.length])
+        .replace("{component}", components[i % components.length])
+        .replace("{reason}", reasons[i % reasons.length])
+        .replace("{case}", cases[i % cases.length])
+        .replace("{fix}", fixes[i % fixes.length])
+        .replace("{pattern}", patterns[i % patterns.length]);
+
+    let text: string;
+
+    switch (opts.pattern) {
+      case "unique":
+        text = fill(template) + ` [turn ${i}]`;
+        break;
+
+      case "repetitive":
+        text = fill(templates[i % 5]);
+        break;
+
+      case "mixed":
+        text =
+          i % 5 < 3
+            ? fill(template) + ` [turn ${i}]`
+            : fill(templates[i % 3]);
+        break;
+
+      case "code-review":
+        if (i % 4 === 0) {
+          text = `Reviewing PR #${i}: ${fill(template)}`;
+        } else if (i % 4 === 1) {
+          text = "```typescript\nfunction handle" + components[i % components.length] + "() {\n  // " + fill(template) + "\n  return result;\n}\n```";
+        } else if (i % 4 === 2) {
+          text = `LGTM. ${fill(template)} The test coverage looks good.`;
+        } else {
+          text = `@user requested changes: ${fill(template)}`;
+        }
+        break;
+
+      case "debug-session":
+        if (i % 6 === 0) {
+          text = `Error in ${files[i % files.length]}: TypeError: Cannot read property '${components[i % components.length]}' of undefined`;
+        } else if (i % 6 === 1) {
+          text = `Stack trace:\n  at ${components[i % components.length]} (${files[i % files.length]}:${100 + i}:15)\n  at process (${files[(i + 1) % files.length]}:${50 + i}:3)`;
+        } else if (i % 6 === 2) {
+          text = `The issue is that ${cases[i % cases.length]} isn't being handled. ${fixes[i % fixes.length]}.`;
+        } else if (i % 6 === 3) {
+          text = `Applied fix: ${fill(template)}`;
+        } else if (i % 6 === 4) {
+          text = `Tests passing now. ${reasons[i % reasons.length]}.`;
+        } else {
+          text = `Committed fix for ${components[i % components.length]}. ${fill(template)}`;
+        }
+        break;
+
+      default:
+        text = fill(template);
+    }
+
+    const role: EngineMessage["role"] = i % 2 === 0 ? "user" : "assistant";
+    msgs.push(makeMsg(role, text));
+  }
+
+  return msgs;
+}
+
+/** Generate a large realistic conversation with tool reads, code blocks, discussion. */
+export function generateRealisticConversation(turns: number): EngineMessage[] {
+  const msgs: EngineMessage[] = [];
+  for (let i = 0; i < turns; i++) {
+    const phase = i % 8;
+    const mods = ["compression", "dedup", "embedding", "search"];
+    const fns = ["vectorStore.ts", "engine.ts", "compact.ts", "recall.ts"];
+    const issues = ["error handling", "type safety", "performance", "logging"];
+    const bugs = ["null pointer", "type mismatch", "race condition", "memory leak"];
+
+    switch (phase) {
+      case 0:
+        msgs.push(
+          makeMsg(
+            "user",
+            `I need help with the ${mods[i % 4]} module. It's not working correctly.`,
+          ),
+        );
+        break;
+      case 1:
+        msgs.push(
+          makeMsg(
+            "assistant",
+            `Let me look at the code. I'll check the ${fns[i % 4]} file.`,
+          ),
+        );
+        break;
+      case 2:
+        msgs.push(
+          makeMsg(
+            "tool",
+            `File content of src/${["vectorStore", "engine", "compact", "recall"][i % 4]}.ts:\n${"x".repeat(800 + (i % 5) * 200)}\n// Line ${i * 10}: function ${["search", "compact", "embed", "dedup"][i % 4]}() { ... }`,
+          ),
+        );
+        break;
+      case 3:
+        msgs.push(
+          makeMsg(
+            "assistant",
+            `I found the issue. The ${mods[i % 4]} function doesn't handle edge cases properly. Here's what I suggest:\n\n\`\`\`typescript\nfunction fixed${["Search", "Compact", "Embed", "Dedup"][i % 4]}() {\n  if (!input) return null;\n  return process(input);\n}\n\`\`\``,
+          ),
+        );
+        break;
+      case 4:
+        msgs.push(
+          makeMsg(
+            "user",
+            `That looks good. Can you also fix the ${issues[i % 4]} while you're at it?`,
+          ),
+        );
+        break;
+      case 5:
+        msgs.push(
+          makeMsg(
+            "assistant",
+            `Sure. I've updated the ${issues[i % 4]} as well. The changes affect:\n- src/engine.ts\n- src/store.ts`,
+          ),
+        );
+        break;
+      case 6:
+        msgs.push(makeMsg("user", "Run the tests to make sure nothing is broken."));
+        break;
+      case 7:
+        msgs.push(
+          makeMsg(
+            "assistant",
+            `All ${150 + i * 3} tests pass. The fix resolved the ${bugs[i % 4]} issue.`,
+          ),
+        );
+        break;
+    }
+  }
+  return msgs;
+}
+
+/** Generate near-duplicate messages with controlled edit distance. */
+export function generateNearDuplicates(
+  base: string,
+  count: number,
+  editLevel: "none" | "one-word" | "minor-rephrase" | "major-change",
+): string[] {
+  const results: string[] = [];
+  for (let i = 0; i < count; i++) {
+    switch (editLevel) {
+      case "none":
+        results.push(base);
+        break;
+      case "one-word":
+        results.push(
+          base.replace(
+            "bug",
+            ["issue", "defect", "error", "problem"][i % 4],
+          ),
+        );
+        break;
+      case "minor-rephrase":
+        results.push(
+          base
+            .replace(
+              "Found a bug",
+              [
+                "Discovered an issue",
+                "Spotted a defect",
+                "Located an error",
+                "Identified a problem",
+              ][i % 4],
+            )
+            .replace(
+              "in the",
+              ["in the", "within the", "inside the", "in our"][i % 4],
+            ),
+        );
+        break;
+      case "major-change":
+        results.push(
+          `Turn ${i}: ${base.split(" ").reverse().join(" ")}. Additional context: ${"y".repeat(200)}`,
+        );
+        break;
+    }
+  }
+  return results;
+}

package/src/recall/async.ts

@@ -0,0 +1,142 @@
+/**
+ * recall/async.ts — Slice 2 async cross-repo recall.
+ *
+ * Same dedup/bound/inline contract as `recallAndInline`, but backed by
+ * `VectorStore.searchAsync` so it can recall across repos (HNSW NN over the
+ * global PGlite index) when `opts.crossRepo` is set. The synchronous
+ * `recallAndInline` is unchanged and remains the default per-session path.
+ * Inline-window dedupe + token cap (Fix C) apply here too.
+ */
+import {
+	vectorSearchAsync,
+	vectorWasInjected,
+	vectorMarkInjected,
+	type SearchHit,
+	type VectorStore,
+} from "../vectorStore.js";
+import { estimateBlockTokens } from "../tokens.js";
+import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
+import { formatRecallBlock } from "./format.js";
+import type { RecallInjectOptions, RecallInjectResult } from "./types.js";
+
+/**
+ * `store` must provide `searchAsync` (the live VectorStore does). Errors fall
+ * back to an empty result — recall is a bonus, never a hard dependency.
+ */
+export async function recallAndInlineAsync(
+	opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
+	store: VectorStore,
+): Promise<RecallInjectResult> {
+	const limit = opts.limit ?? 3;
+	const skip = opts.skipInjected ?? true;
+	const maxTokens = opts.recallMaxTokens ?? 0;
+	const doWindowDedupe = opts.windowDedupe ?? false;
+	const dedupSim = opts.dedupSim ?? 0.9;
+
+	let hits: SearchHit[] = [];
+	try {
+		hits = await vectorSearchAsync(store, opts.sessionId, opts.query, limit, {
+			crossRepo: opts.crossRepo,
+			repoId: opts.repoId,
+		});
+	} catch {
+		hits = [];
+	}
+
+	// F1: hoist one embedder instance for inline dedupe. defaultEmbedder() is
+	// deterministic but creating it per call wastes allocations on large hit sets.
+	// (recallAndInline already hoisted this; applying the same fix here.)
+	const embedder = defaultEmbedder();
+	let liveEmbeddings: number[][] = [];
+	if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+		liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+	}
+
+	const toInject: SearchHit[] = [];
+	let blockTokens = 0;
+
+	// F2: when cross-repo is on but no global index dir could be resolved, skip
+	// foreign hits rather than injecting them undeduped — otherwise a foreign
+	// checkpoint with no machine-wide injected-set to consult would re-inject in
+	// every new session. Same-repo hits (no repoId) are unaffected. Warn once so
+	// the silent degradation is observable. (The extension resolver normally
+	// supplies a default globalIndexDir, so this is belt-and-braces.)
+	const skipCrossRepoHits = !!opts.crossRepo && !opts.globalIndexDir;
+	if (skipCrossRepoHits) {
+		try {
+			console.warn(
+				"[mega-compact:recall] cross-repo recall enabled but globalIndexDir is unset — " +
+					"skipping cross-repo injection to avoid re-injecting undeduped foreign checkpoints",
+			);
+		} catch {
+			/* ignore */
+		}
+	}
+
+	for (const h of hits) {
+		// F2: skip foreign hits when we can't dedup them machine-wide.
+		if (skipCrossRepoHits && h.repoId) continue;
+		if (
+			skip &&
+			vectorWasInjected(store, opts.sessionId, h.checkpoint.checkpointId)
+		)
+			continue;
+		// S18: machine-wide injected-set — a foreign checkpoint already injected
+		// (in any session) is never re-injected. Only applies to cross-repo hits
+		// (same-repo hits have no repoId and are handled by the per-session set).
+		if (opts.globalIndexDir && h.repoId) {
+			try {
+				const { wasInjectedGlobal } = await import("./../store/sqlite.js");
+				if (
+					wasInjectedGlobal(
+						h.checkpoint.checkpointId,
+						opts.sessionId,
+						opts.globalIndexDir,
+					)
+				)
+					continue;
+			} catch {
+				/* non-fatal: degrade to per-session injected-set only */
+			}
+		}
+		// Inline dedupe: skip a hit already resident in the live window (F1: hoisted embedder).
+		if (doWindowDedupe && liveEmbeddings.length > 0) {
+			const hitVec = embedder.embed(h.checkpoint.summary);
+			if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim))
+				continue;
+		}
+		// F3: build the hit list first; format ONCE at the end so the block carries
+		// exactly one preamble and numbering [1..n] rather than one per hit.
+		const partTokens = estimateBlockTokens(h.checkpoint.summary);
+		if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+		toInject.push(h);
+		blockTokens += partTokens;
+		vectorMarkInjected(store, opts.sessionId, h.checkpoint.checkpointId);
+		// S18: record the cross-repo injection machine-wide so it's not re-injected
+		// by a later recall (same or different session).
+		if (opts.globalIndexDir && h.repoId) {
+			try {
+				const { markInjectedGlobal } = await import("./../store/sqlite.js");
+				markInjectedGlobal(
+					h.checkpoint.checkpointId,
+					h.repoId,
+					opts.sessionId,
+					opts.globalIndexDir,
+				);
+			} catch {
+				/* non-fatal */
+			}
+		}
+	}
+
+	// F3: format once — one preamble, correct [1..n] numbering, token cap counted
+	// against one preamble (not N). Pass the full toInject array so formatRecallBlock
+	// has repoId + score for proper labeling.
+	const block = toInject.length > 0 ? formatRecallBlock(toInject) : "";
+	const report = toInject.map(
+		(h) =>
+			`  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
+	);
+
+	return { toInject, report, block, empty: toInject.length === 0 };
+}