pi-mega-compact 0.4.0

package/src/recall.integration.test.ts

@@ -0,0 +1,96 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { compactSession } from "./engine.js";
+import { recallAndInline } from "./recall.js";
+import type { EngineMessage } from "./types.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-resume-"));
+let counter = 0;
+/** Two instances, SAME disk dir — simulates a fresh process / resumed session. */
+function storeForDir(dir: string) {
+  return new VectorStore({ dedupSim: 0.9, stateDir: dir });
+}
+function msg(role: EngineMessage["role"], text: string, toolName?: string): EngineMessage {
+  return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
+}
+const SESS = "sess_resume";
+
+/**
+ * Simulate the extension's `recentUserQuery`: the resume query is built from the
+ * newest user message in the (re-loaded) session. We model "newest user msg"
+ * directly rather than going through pi's session manager.
+ */
+function latestUserQuery(messages: EngineMessage[]): string {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i].role === "user") return messages[i].text;
+  }
+  return "";
+}
+
+test("resume contract: compact in one process, recall in a fresh one from disk", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+
+  // --- Process 1: the original session compacts ---
+  const writer = storeForDir(dir);
+  const session: EngineMessage[] = [
+    msg("user", "investigated src/compact.ts and added a truncate helper"),
+    msg("assistant", "added truncate", "Edit"),
+    msg("user", "then wired it into the summary pipeline"),
+    msg("assistant", "wired it in", "Edit"),
+  ];
+  const ran = compactSession(
+    { sessionId: SESS, messages: session, keepFrom: session.length, timestamp: 1 },
+    writer,
+  );
+  assert.equal(ran.skipped, false);
+  assert.ok(ran.checkpointId, "a checkpoint was persisted");
+
+  // --- Process 2: pi restarts, session resumes from disk ---
+  const reader = storeForDir(dir); // brand new instance, same dir
+  const resumeQuery = latestUserQuery(session); // newest user msg
+  const r = recallAndInline(
+    { sessionId: SESS, query: resumeQuery, limit: 3, source: "resume" },
+    reader,
+  );
+
+  assert.equal(r.empty, false, "resume must re-surface the compacted context");
+  assert.equal(r.toInject.length, 1);
+  assert.equal(r.toInject[0].checkpoint.checkpointId, ran.checkpointId);
+  assert.ok(r.block.includes("Recalled context"), "block is model-visible system-prompt text");
+});
+
+test("resume contract: nothing to recall for a brand-new session", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const fresh = storeForDir(dir);
+  const r = recallAndInline(
+    { sessionId: "sess_never_seen", query: "anything at all", limit: 3, source: "resume" },
+    fresh,
+  );
+  assert.equal(r.empty, true);
+  assert.equal(r.block, "");
+});
+
+test("resume contract: re-inject is deduped after the first recall", () => {
+  const dir = join(baseTmp, `run-${counter++}`);
+  const writer = storeForDir(dir);
+  const session: EngineMessage[] = [
+    msg("user", "built the trigram embedder for the vector store"),
+    msg("assistant", "built it", "Edit"),
+  ];
+  compactSession({ sessionId: SESS, messages: session, keepFrom: 2, timestamp: 1 }, writer);
+
+  const reader = storeForDir(dir);
+  const q = latestUserQuery(session);
+  const first = recallAndInline({ sessionId: SESS, query: q, limit: 3, source: "resume" }, reader);
+  const second = recallAndInline({ sessionId: SESS, query: q, limit: 3, source: "resume" }, reader);
+  assert.equal(first.empty, false);
+  assert.equal(second.empty, true, "second resume does not re-inject (sentinel)");
+});
+
+test("cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/recall.test.ts

@@ -0,0 +1,59 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { compactSession } from "./engine.js";
+import { recallAndInline, formatRecallBlock } from "./recall.js";
+import type { EngineMessage } from "./types.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
+let counter = 0;
+function store() {
+  return new VectorStore({ dedupSim: 0.9, stateDir: join(baseTmp, `run-${counter++}`) });
+}
+function msg(role: EngineMessage["role"], text: string, toolName?: string): EngineMessage {
+  return toolName ? { role, text, toolName, input: text, output: text } : { role, text };
+}
+const SESS = "sess_recall";
+
+test("recallAndInline injects new hits and marks them injected", () => {
+  const s = store();
+  compactSession({ sessionId: SESS, messages: [msg("user", "investigated src/vectorStore.ts embedding"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+  compactSession({ sessionId: SESS, messages: [msg("user", "fixed the dedupe race in store.ts"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 2 }, s);
+
+  const r1 = recallAndInline({ sessionId: SESS, query: "vectorStore embedding", limit: 1, source: "command" }, s as any);
+  assert.equal(r1.empty, false);
+  assert.equal(r1.toInject.length, 1);
+  assert.ok(r1.block.includes("Recalled context"));
+
+  // Second call with the same query must NOT re-inject (shared dedup).
+  const r2 = recallAndInline({ sessionId: SESS, query: "vectorStore embedding", limit: 1, source: "command" }, s as any);
+  assert.equal(r2.empty, true);
+  assert.equal(r2.toInject.length, 0);
+});
+
+test("recallAndInline skipInjected=false re-returns hits", () => {
+  const s = store();
+  compactSession({ sessionId: SESS, messages: [msg("user", "configured the fast gate threshold"), msg("assistant", "ok", "Edit")], keepFrom: 2, timestamp: 1 }, s);
+  const r1 = recallAndInline({ sessionId: SESS, query: "fast gate threshold", limit: 5, source: "resume" }, s as any);
+  const r2 = recallAndInline({ sessionId: SESS, query: "fast gate threshold", limit: 5, source: "resume", skipInjected: false }, s as any);
+  assert.equal(r1.toInject.length, 1);
+  assert.equal(r2.toInject.length, 1);
+});
+
+test("formatRecallBlock is empty for no hits", () => {
+  assert.equal(formatRecallBlock([]), "");
+});
+
+test("recallAndInline empty when store has nothing for query", () => {
+  const s = store();
+  const r = recallAndInline({ sessionId: SESS, query: "no such topic exists here", limit: 5, source: "command" }, s as any);
+  assert.equal(r.empty, true);
+  assert.equal(r.block, "");
+});
+
+test("cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/recall.ts

@@ -0,0 +1,100 @@
+/**
+ * recall.ts — Layer 5 (RECALL / INLINE): the unified injection path.
+ *
+ * ONE vector store, THREE entry points, ONE dedup engine. Every way context
+ * gets re-injected into the window (auto-inline on resume, on-demand
+ * /recall-context, and the dedup sentinel) goes through `recallAndInline`.
+ * It always does: search -> dedupe -> inject. The only thing that differs per
+ * entry point is *what triggers it* and *what query it uses*.
+ *
+ * Injection respects PREVENT-PI-003: pi has no `system` message role, so we
+ * prepend our recall block to the system prompt via the `before_agent_start`
+ * hook's `systemPrompt` result (the extension wires that). This module is
+ * pi-agnostic: it returns an injectable text block and records injections; the
+ * extension decides where it lands.
+ */
+
+import { recall as searchRecall } from "./engine.js";
+import type { SearchHit, VectorStore } from "./vectorStore.js";
+
+export type RecallSource = "resume" | "command" | "sentinel";
+
+export interface RecallInjectOptions {
+  sessionId: string;
+  query: string;
+  limit?: number;
+  source: RecallSource;
+  /** Skip checkpoints already injected this session (recall dedup). */
+  skipInjected?: boolean;
+}
+
+export interface RecallInjectResult {
+  /** Blocks that are ready to inline (already deduped against the window). */
+  toInject: SearchHit[];
+  /** Human-readable lines for status/notify reporting. */
+  report: string[];
+  /** The concatenated, model-visible recall block (empty when nothing new). */
+  block: string;
+  /** True when nothing new was inlined. */
+  empty: boolean;
+}
+
+/** Wrap a recall block so the model reads it as restored compacted context. */
+export function formatRecallBlock(hits: SearchHit[]): string {
+  if (hits.length === 0) return "";
+  const parts = hits.map((h, i) => {
+    const score = (h.score * 100).toFixed(0);
+    return (
+      `### Recalled context [${i + 1}] (relevance ${score}%)\n` +
+      `${h.checkpoint.summary.trim()}\n` +
+      (h.checkpoint.filesModified.length
+        ? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
+        : "")
+    );
+  });
+  return (
+    "The following compacted context was recalled from earlier in this session " +
+    "and is relevant to the current request. Treat it as background you already know:\n\n" +
+    parts.join("\n")
+  );
+}
+
+/**
+ * Run the unified recall+dudupe+prepare-inject pipeline. Does NOT touch pi;
+ * it records injections via `markInjected` so the next call dedupes. The
+ * `store` is passed by the extension (defaults to the engine's default store).
+ */
+export function recallAndInline(
+  opts: RecallInjectOptions,
+  store: Pick<VectorStore, "search" | "wasInjected" | "markInjected">,
+): RecallInjectResult {
+  const limit = opts.limit ?? 3;
+  const skip = opts.skipInjected ?? true;
+
+  const { hits } = searchRecall(
+    { sessionId: opts.sessionId, query: opts.query, limit, skipInjected: false },
+    store as VectorStore,
+  );
+
+  // Shared dedup: drop checkpoints already injected this session, then mark the
+  // survivors so repeated triggers are free. (Cosine near-dup collapse already
+  // happened inside store.search.)
+  const toInject: SearchHit[] = [];
+  for (const h of hits) {
+    if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+    toInject.push(h);
+    store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+  }
+
+  const block = formatRecallBlock(toInject);
+  const report = toInject.map(
+    (h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
+  );
+
+  return {
+    toInject,
+    report,
+    block,
+    empty: toInject.length === 0,
+  };
+}

package/src/sprint14.test.ts

@@ -0,0 +1,245 @@
+/**
+ * sprint14.test.ts — Sprint 14 full-pipeline wiring (flags, backfill, monitoring, canary).
+ * Hermetic: isolated state dirs, no network, no remote.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "./vectorStore.js";
+import { defaultEmbedder } from "./embedder.js";
+import { loadDedupConfig, type DedupConfigShape } from "./config/dedup.js";
+import {
+  loadMetrics,
+  saveMetrics,
+  recordDecision,
+  evaluateAlerts,
+  fpRate,
+  p95,
+  type DedupMetrics,
+} from "./monitoring.js";
+import { backfillPhase, backfillRaptor } from "./store/backfill.js";
+import { listCheckpoints, closeStore } from "./store/sqlite.js";
+import { CanaryController, runCanary } from "./canary.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-s14-"));
+
+function cfg(over: Partial<DedupConfigShape> = {}): DedupConfigShape {
+  return { ...loadDedupConfig(), ...over };
+}
+
+function store(over: Partial<DedupConfigShape> = {}, eventsPath?: string): VectorStore {
+  const dir = join(baseTmp, `run-${Math.floor(performance.now() * 1000)}-${Math.random()}`);
+  return new VectorStore({ stateDir: dir, config: cfg(over), eventsPath });
+}
+
+// --- 1. Flag matrix: 16 combos don't crash add()/search() -------------------
+
+test("flag matrix: all 16 L0/L1/L2/RAPTOR enable combos are safe", () => {
+  const flags = [false, true];
+  let combos = 0;
+  for (const l0 of flags)
+    for (const l1 of flags)
+      for (const l2 of flags)
+        for (const raptor of flags) {
+          combos++;
+          const s = store({ L0_ENABLED: l0, L1_ENABLED: l1, L2_ENABLED: l2, RAPTOR_ENABLED: raptor });
+          s.add({ sessionId: "s", summary: "x", regionText: `region A for combo ${combos} about the cache`, timestamp: 1 });
+          const r2 = s.add({ sessionId: "s", summary: "x", regionText: `region B for combo ${combos} about the parser`, timestamp: 2 });
+          // search must not throw under any combination
+          const hits = s.search("s", "cache", 3);
+          assert.ok(Array.isArray(hits));
+          // With all tiers off, the second add is always "new" (no collapse).
+          if (!l0 && !l1 && !l2) assert.equal(r2.deduped, false);
+        }
+  assert.equal(combos, 16);
+});
+
+// --- 2. MARK_ONLY_L1 records but doesn't collapse ---------------------------
+
+test("MARK_ONLY_L1: L1 match is recorded but not collapsed (new checkpoint)", () => {
+  // Disable L2 so we isolate L1 behavior (L2 cosine would otherwise catch the near-dup).
+  const s = store({ L1_ENABLED: true, MARK_ONLY_L1: true, L2_ENABLED: false });
+  const a = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loop", timestamp: 1 });
+  const b = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loops", timestamp: 2 });
+  assert.equal(a.deduped, false);
+  // MARK_ONLY → b is NOT collapsed into a; both stored as active.
+  assert.equal(b.deduped, false);
+  const all = listCheckpoints("s", (s as any).stateDir);
+  assert.equal(all.length, 2);
+  assert.ok(all.every((c) => c.dedupStatus === "active"));
+});
+
+test("MARK_ONLY_L1 off: L1 match IS collapsed", () => {
+  const s = store({ L1_ENABLED: true, MARK_ONLY_L1: false });
+  s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loop", timestamp: 1 });
+  const b = s.add({ sessionId: "s", summary: "x", regionText: "the parser optimized the hot loops", timestamp: 2 });
+  assert.equal(b.deduped, true);
+  assert.equal(b.reason, "l1MinHash");
+});
+
+// --- 3. Backfill resumes after interrupt -------------------------------------
+
+test("backfill L1 resumes after simulated interrupt", () => {
+  const dir = join(baseTmp, `bf-${Math.floor(performance.now())}`);
+  // Seed 12 checkpoints, 2 per batch of 5.
+  const s = new VectorStore({ stateDir: dir, config: cfg() });
+  const texts = [
+    "The walrus drifted past the lighthouse while the baker kneaded sourdough at dawn",
+    "Quantum entanglement linked the two photons across the lab in a cryogenic chamber",
+    "A medieval scribe copied the gospel by candlelight atop a windswept cliff",
+    "The rover sampled basalt from the crater and transmitted spectra to mission control",
+    "Jazz musicians improvised a syncopated triangle rhythm beneath the streetlamp",
+    "The glacier calved a towering iceberg into the fjord with a thunderous crack",
+    "A botanist cataloged the orchid species thriving in the cloud forest canopy",
+    "The blacksmith forged a horseshoe while sparks danced across the anvil",
+    "Astronomers imaged a distant nebula glowing with newborn stellar furnaces",
+    "The ferry crossed the strait as gulls wheeled above the churning wake",
+    "A weaver threaded crimson silk through the loom in the mountain village",
+    "The surgeon sutured the incision with steady hands under the theatre lights",
+  ];
+  for (let i = 0; i < 12; i++) {
+    s.add({ sessionId: "sess_bf", summary: `n${i}`, regionText: texts[i], timestamp: i });
+  }
+  // Interrupt after batch 1 (5 rows).
+  const r1 = backfillPhase("L1", "sess_bf", dir, { batchSize: 5, interruptAfterBatches: 1 });
+  assert.equal(r1.interrupted, true);
+  assert.equal(r1.processed, 5);
+  assert.ok(r1.cursor === "chkpt_005" || r1.cursor === "chkpt_05");
+  // Resume: should continue from the cursor → process the remaining 7.
+  const r2 = backfillPhase("L1", "sess_bf", dir, { batchSize: 5 });
+  assert.equal(r2.interrupted, false);
+  assert.equal(r2.processed, 12); // total across both runs (cursor-based resume)
+  closeStore(dir);
+});
+
+// --- 4. Alert fires on injected FP spike -------------------------------------
+
+test("alert: FP spike breaches threshold → MARK_ONLY flagged + warning", () => {
+  const config = cfg();
+  const m: DedupMetrics = loadMetrics("/dev/null");
+  // Inject 100 L1 decisions, 20 false positives → 20% > FP_RATE_L1L2 (5%).
+  for (let i = 0; i < 100; i++) {
+    recordDecision(m, "L1", i < 20 ? "deduped" : "new", 5, i < 20);
+  }
+  const res = evaluateAlerts(m, config);
+  assert.ok(res.breached.includes("L1"));
+  assert.ok(res.warnings.some((w) => w.includes("DEDUP FP BREACH tier=L1")));
+  assert.ok(fpRate(m, "L1") > config.FP_RATE_L1L2);
+});
+
+test("alert: clean run does NOT breach", () => {
+  const config = cfg();
+  const m = loadMetrics("/dev/null");
+  for (let i = 0; i < 100; i++) recordDecision(m, "L1", "new", 5, false);
+  const res = evaluateAlerts(m, config);
+  assert.equal(res.breached.length, 0);
+});
+
+// --- 5. Canary auto-disables a tier whose p95 exceeds budget ---------------
+
+test("canary: auto-disables a tier whose p95 exceeds budget", () => {
+  // Feed where L2 always has high latency (breaches P95_BUDGET_MS).
+  const feed = (_step: number, c: DedupConfigShape): DedupMetrics => {
+    const m = loadMetrics("/dev/null");
+    if (c.L0_ENABLED) recordDecision(m, "L0", "new", 1, false);
+    if (c.L1_ENABLED) recordDecision(m, "L1", "new", 1, false);
+    if (c.L2_ENABLED) recordDecision(m, "L2", "new", 500, false); // > 100ms budget
+    if (c.RAPTOR_ENABLED) recordDecision(m, "RAPTOR", "new", 1, false);
+    return m;
+  };
+  const { controller, disabled } = runCanary(feed, cfg({ P95_BUDGET_MS: 100 }));
+  assert.ok(disabled.includes("L2"), `expected L2 auto-disabled, got ${JSON.stringify(disabled)}`);
+  assert.equal(controller.config.L2_ENABLED, false);
+  // Lower tiers should still be enabled.
+  assert.equal(controller.config.L0_ENABLED, true);
+});
+
+test("canary: sequential enablement order L0→L1→L2→RAPTOR", () => {
+  const c = new CanaryController(cfg());
+  assert.deepEqual([...c.getState().enabled], ["L0"]);
+  const t1 = c.stepForward();
+  assert.equal(t1, "L1");
+  assert.equal(c.stepForward(), "L2");
+  assert.equal(c.stepForward(), "RAPTOR");
+  assert.equal(c.stepForward(), null); // all enabled
+});
+
+// --- 6. Monitoring: structured decision events written ----------------------
+
+test("monitoring: add() writes structured decision events to events.log", () => {
+  const dir = join(baseTmp, `mon-${Math.floor(performance.now())}`);
+  const eventsPath = join(dir, "events.log");
+  const s = new VectorStore({
+    stateDir: dir,
+    config: cfg(),
+    eventsPath,
+  });
+  s.add({ sessionId: "s", summary: "x", regionText: "unique region alpha one", timestamp: 1 });
+  s.add({ sessionId: "s", summary: "x", regionText: "unique region alpha one", timestamp: 2 }); // L0 content dup
+  assert.ok(existsSync(eventsPath));
+  const lines = readFileSync(eventsPath, "utf-8").trim().split("\n").filter(Boolean);
+  assert.ok(lines.length >= 2);
+  const ev = JSON.parse(lines[0]);
+  assert.ok(["L0"].includes(ev.tier));
+  assert.ok(["new", "deduped", "mark_only"].includes(ev.result));
+  closeStore(dir);
+});
+
+// --- 7. RAPTOR backfill builds + persists a tree ---------------------------
+
+test("backfill RAPTOR builds + persists a tree for a session", () => {
+  const dir = join(baseTmp, `raptorbf-${Math.floor(performance.now())}`);
+  const s = new VectorStore({ stateDir: dir, config: cfg() });
+  const texts = [
+    "The whale breached beside the research vessel near the polar ice shelf",
+    "A potter shaped the clay vessel on the spinning wheel at the riverside studio",
+    "The comet streaked across the pre dawn sky witnessed by the hilltop observatory",
+    "Lumberjacks felled the ancient cedar while the river carried the logs downstream",
+    "The chemist titrated the solution until the indicator turned faint violet",
+    "A flock of cranes migrated northward over the thawing wetland at first light",
+    "The locksmith picked the stubborn tumbler and opened the oak cabinet",
+    "Geologists hammered the schist sample from the canyon wall into the satchel",
+    "The chocolatier tempered the couverture until it snapped with a clean gloss",
+    "A fisher cast the line into the mist where the trout rose to the fly",
+    "The archivist unsealed the parchment scroll recovered from the coastal ruin",
+    "Beekeepers harvested the golden comb while the orchard blossoms drifted down",
+    "The pilot navigated the canyon winds using only the instrument panel glow",
+    "A tailor stitched the velvet cuff with silk thread by the window",
+    "The miner extracted the quartz crystal from the vein deep in the shaft",
+    "Cartographers plotted the uncharted island onto the worn leather map",
+    "The gardener pruned the rosebush and tied the canes to the cedar trellis",
+    "A violinist tuned the gut strings until the chamber rang pure and bright",
+    "The diver surfaced with the amphora lifted from the sunken galleon",
+    "Shepherds guided the flock across the high pasture toward the stone bothy",
+  ];
+  for (let i = 0; i < 20; i++) {
+    s.add({ sessionId: "sess_rb", summary: `n${i}`, regionText: texts[i], timestamp: i });
+  }
+  const res = backfillRaptor("sess_rb", dir, defaultEmbedder());
+  assert.ok(res.processed > 0);
+  // Sanity: source checkpoints remain intact (backfill is additive).
+  const nodes = listCheckpoints("sess_rb", dir).length;
+  assert.ok(nodes >= 20);
+  closeStore(dir);
+});
+
+// --- 8. dashboard.json metrics round-trip -----------------------------------
+
+test("metrics: dashboard.json persists + p95 computed", () => {
+  const path = join(baseTmp, `dash-${Math.floor(performance.now())}.json`);
+  const m = loadMetrics(path);
+  for (let i = 0; i < 10; i++) recordDecision(m, "L2", "new", i * 10, false);
+  saveMetrics(path, m);
+  const reloaded = loadMetrics(path);
+  assert.equal(reloaded.decisions.L2, 10);
+  assert.equal(p95(reloaded.latency.L2), 90); // 95th pct of [0,10,..,90]
+});
+
+// --- cleanup ----------------------------------------------------------------
+
+test("Sprint 14 cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});