pi-mega-compact 0.4.28 → 0.5.0

package/src/memoryRecall.test.ts

@@ -0,0 +1,100 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { recallMemories } from "./memoryRecall.js";
+import { addMemory, getMemory } from "./store/sqlite.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-memrec-"));
+
+function biGramEmbedder() {
+  // Deterministic test embedder — bigger dim + binary encoding so two semantically
+  // related strings have higher cosine than unrelated ones.
+  const dim = 64;
+  return {
+    dim,
+    embed(text: string): number[] {
+      const v = new Array(dim).fill(0);
+      const norm = text.toLowerCase().replace(/[^a-z0-9 ]/g, "").trim();
+      for (let i = 0; i < norm.length - 1; i++) {
+        const idx = ((norm.charCodeAt(i) * 31 + norm.charCodeAt(i + 1)) >>> 0) % dim;
+        v[idx] = 1;
+      }
+      return v;
+    },
+  };
+}
+
+test("recallMemories: ranks relevant memory above unrelated", async () => {
+  const dir = join(baseTmp, "rank");
+  addMemory({ content: "we use sqlite for the durable store", category: "decision" }, null, dir);
+  addMemory({ content: "the threshold is 100 thousand tokens", category: "decision" }, null, dir);
+  addMemory({ content: "katz says hi", category: "note" }, null, dir);
+  const hits = await recallMemories("what store do we use?", dir, {
+    embedder: biGramEmbedder(),
+    topK: 3,
+    minSimilarity: 0.0,
+  });
+  assert.ok(hits.length >= 2, "finds relevant");
+  assert.ok(/sqlite/.test(hits[0].memory.content), "top hit is sqlite one");
+  assert.ok(!/katz/.test(hits[0].memory.content), "unrelated not on top");
+});
+
+test("recallMemories: marks referenced hits (last_referenced updated)", async () => {
+  const dir = join(baseTmp, "ref");
+  addMemory({ content: "policy is local-only", category: "rule" }, null, dir);
+  const hits = await recallMemories("local-only policy", dir, { embedder: biGramEmbedder() });
+  assert.ok(hits.length >= 1);
+  const fresh = getMemory(hits[0].memory.id, dir);
+  assert.ok(fresh && fresh.lastReferenced && fresh.lastReferenced > 0, "lastReferenced set");
+});
+
+test("recallMemories: empty store returns []", async () => {
+  const dir = join(baseTmp, "empty");
+  const hits = await recallMemories("anything", dir, { embedder: biGramEmbedder() });
+  assert.deepEqual(hits, []);
+});
+
+test("recallMemories: decision category beats fact category at equal similarity", async () => {
+  const dir = join(baseTmp, "category");
+  // Two memories that share many bigrams with the query — both will have very
+  // similar cosine. The decision category should win because of categoryWeight.
+  addMemory({ content: "we use redis for cache" }, null, dir);
+  const aId = addMemory({ content: "we use redis as primary cache key", category: "fact" }, null, dir);
+  const dId = addMemory({ content: "we use redis as primary cache layer", category: "decision" }, null, dir);
+  const hits = await recallMemories("redis cache layer", dir, {
+    embedder: biGramEmbedder(),
+    topK: 3,
+    minSimilarity: 0.0,
+  });
+  assert.ok(hits.length >= 2);
+  assert.equal(hits[0].memory.id, dId, "decision-tagged memory outranks fact-tagged");
+  assert.notEqual(hits[0].memory.id, aId, "top is the decision row, not the unknown-category row");
+});
+
+test("recallMemories: fresher reference beats older at equal similarity", async () => {
+  const dir = join(baseTmp, "recency");
+  const aId = addMemory({ content: "we use redis for cache", category: "fact" }, null, dir);
+  const bId = addMemory({ content: "we use redis for cache layer", category: "fact" }, null, dir);
+  // Backdate aId so its last_referenced is older; bId stays fresh.
+  const { openStore, closeStore } = await import("./store/sqlite.js");
+  const db = openStore(dir);
+  const longAgo = Math.floor(Date.now() / 1000) - 30 * 86_400; // 30d
+  db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(longAgo, aId);
+  // closeStore evicts the cached handle — calling db.close() directly would
+  // leave a stale closed DB in the openStore cache and break subsequent
+  // callers (the "database is not open" failure under parallel runs).
+  closeStore(dir);
+  const hits = await recallMemories("redis cache layer", dir, {
+    embedder: biGramEmbedder(),
+    topK: 3,
+    minSimilarity: 0.0,
+  });
+  assert.ok(hits.length >= 2);
+  assert.equal(hits[0].memory.id, bId, "freshly-referenced memory outranks 30-day-old one");
+});
+
+test("cleanup memrec", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/memoryRecall.ts

@@ -0,0 +1,83 @@
+/**
+ * memoryRecall.ts — semantic recall over the durable memories table (S21).
+ * Embeds a query with the same local embedder used by RAPTOR, ranks every
+ * memory in the current repo's SQLite by cosine similarity combined with a
+ * category-weighted + recency-boosted score, and returns the top-k. Side
+ * effect: marks returned memories as referenced (last_referenced) so drift
+ * can be measured. PREVENT-PI-004 — embedder is the same local one used
+ * everywhere else; no remote calls are introduced here.
+ * @module
+ */
+import { defaultEmbedder, cosineSimilarity } from "./embedder.js";
+import { listMemories, referenceMemory, type MemoryRecord } from "./store/sqlite.js";
+
+export interface RecallMemoriesOptions {
+  /** Max memories to return. Default 10. */
+  topK?: number;
+  /** Min cosine similarity (0..1) to include. Default 0.2 — filters unrelated. */
+  minSimilarity?: number;
+  /** If true, mark returned memories as referenced. Default true. */
+  markReferenced?: boolean;
+  /** Override the embedder (test-only seam). */
+  embedder?: { embed(text: string): number[] };
+  /** Repo filter; null = current repo's memories. */
+  repo?: string | null;
+  /** Per-category ranking weights. Categories not listed default to 1.0. */
+  categoryWeights?: Record<string, number>;
+  /** Recency boost strength. Default 0.05 (5% bonus per log-day since reference). */
+  recencyWeight?: number;
+}
+
+/** Default category weights — `decision` wins ties on tied cosine. */
+export const DEFAULT_CATEGORY_WEIGHTS: Record<string, number> = {
+  decision: 1.10,
+  preference: 1.05,
+  fact: 1.0,
+};
+
+const DEFAULT_RECENCY_WEIGHT = 0.05;
+
+/**
+ * Rank memories by a blended score of cosine similarity, category weight, and
+ * recency of last_referenced (with createdAt as a fallback). Returns the
+ * top-k above minSimilarity, sorted by blended score descending. Empty on
+ * no matches.
+ */
+export async function recallMemories(
+  query: string,
+  stateDir: string,
+  opts: RecallMemoriesOptions = {},
+): Promise<Array<{ memory: MemoryRecord; score: number }>> {
+  const topK = opts.topK ?? 10;
+  const minSimilarity = opts.minSimilarity ?? 0.2;
+  const markReferenced = opts.markReferenced ?? true;
+  const embedder = opts.embedder ?? defaultEmbedder();
+  const repo = opts.repo === undefined ? null : opts.repo;
+  const categoryWeights = { ...DEFAULT_CATEGORY_WEIGHTS, ...(opts.categoryWeights ?? {}) };
+  const recencyWeight = opts.recencyWeight ?? DEFAULT_RECENCY_WEIGHT;
+
+  const queryVec = embedder.embed(query);
+  const memories = listMemories(repo, 1000, stateDir);
+  if (!memories.length || !query.trim()) return [];
+
+  const nowSec = Math.floor(Date.now() / 1000);
+  const scored: Array<{ memory: MemoryRecord; score: number }> = [];
+  for (const mem of memories) {
+    const vec = embedder.embed(mem.content);
+    const sim = cosineSimilarity(queryVec, vec);
+    if (sim < minSimilarity) continue;
+    const categoryW = categoryWeights[mem.category ?? "fact"] ?? 1.0;
+    const lastTouch = mem.lastReferenced ?? mem.createdAt ?? nowSec;
+    // log(1 + daysSince) grows slowly — half-life-style decay.
+    const daysSince = Math.max(0, (nowSec - lastTouch) / 86_400);
+    const recencyBoost = Math.log1p(daysSince) * recencyWeight;
+    const finalScore = sim * categoryW * (1 + recencyBoost);
+    scored.push({ memory: mem, score: finalScore });
+  }
+  scored.sort((a, b) => b.score - a.score);
+  const top = scored.slice(0, topK);
+  if (markReferenced) {
+    for (const hit of top) referenceMemory(hit.memory.id, stateDir);
+  }
+  return top;
+}

package/src/recall.test.ts

@@ -5,7 +5,8 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { VectorStore } from "./vectorStore.js";
 import { compactSession } from "./engine.js";
-import { recallAndInline, formatRecallBlock } from "./recall.js";
+import { recallAndInline, recallAndInlineAsync, formatRecallBlock } from "./recall.js";
+import { markInjectedGlobal, wasInjectedGlobal, closeIndexStore } from "./store/sqlite.js";
 import type { EngineMessage } from "./types.js";
 
 const baseTmp = mkdtempSync(join(tmpdir(), "mc-recall-"));
@@ -47,6 +48,26 @@ test("formatRecallBlock is empty for no hits", () => {
   assert.equal(formatRecallBlock([]), "");
 });
 
+test("formatRecallBlock (S17): labels a cross-repo hit with its source repo", () => {
+  const hit = {
+    checkpoint: { checkpointId: "chkpt_x", summary: "did thing Y", filesModified: ["a.ts"] },
+    score: 0.91,
+    repoId: "/home/u/rad-gateway",
+  } as any;
+  const block = formatRecallBlock([hit]);
+  assert.ok(block.includes("from repo"), "labels cross-repo source");
+  assert.ok(block.includes("rad-gateway"), "includes the repo display name");
+});
+
+test("formatRecallBlock (S17): omits the label for same-repo hits (no repoId)", () => {
+  const hit = {
+    checkpoint: { checkpointId: "c1", summary: "s", filesModified: [] },
+    score: 0.9,
+  } as any;
+  const block = formatRecallBlock([hit]);
+  assert.ok(!block.includes("from repo"), "no source label for same-repo hits");
+});
+
 test("recallAndInline empty when store has nothing for query", () => {
   const s = store();
   const r = recallAndInline({ sessionId: SESS, query: "no such topic exists here", limit: 5, source: "command" }, s as any);
@@ -98,6 +119,61 @@ test("Fix C: inline dedupe drops a hit already resident in the live window", ()
   );
 });
 
+test("S18: global injected-set skips a foreign checkpoint already injected machine-wide", async () => {
+  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi-"));
+  try {
+    const sess = "sess_cross";
+    // A foreign checkpoint already marked injected globally (in this session).
+    markInjectedGlobal("chkpt_foreign", "/repo/other", sess, indexDir);
+    assert.equal(wasInjectedGlobal("chkpt_foreign", sess, indexDir), true);
+    // searchAsync returns the foreign hit; recallAndInlineAsync must skip it
+    // (globally injected) → toInject is empty.
+    const mockStore = {
+      searchAsync: async () => [{
+        checkpoint: { checkpointId: "chkpt_foreign", summary: "foreign work", filesModified: [], dedupStatus: "active" },
+        score: 0.92,
+        repoId: "/repo/other",
+      }],
+      wasInjected: () => false,
+      markInjected: () => {},
+    } as any;
+    const r = await recallAndInlineAsync(
+      { sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
+      mockStore,
+    );
+    assert.equal(r.toInject.length, 0, "globally-injected foreign checkpoint skipped");
+  } finally {
+    closeIndexStore();
+    rmSync(indexDir, { recursive: true, force: true });
+  }
+});
+
+test("S18: a fresh foreign checkpoint is injected AND recorded globally", async () => {
+  const indexDir = mkdtempSync(join(tmpdir(), "mc-gi2-"));
+  try {
+    const sess = "sess_fresh";
+    assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), false);
+    const mockStore = {
+      searchAsync: async () => [{
+        checkpoint: { checkpointId: "chkpt_new", summary: "brand new foreign work", filesModified: [], dedupStatus: "active" },
+        score: 0.93,
+        repoId: "/repo/alpha",
+      }],
+      wasInjected: () => false,
+      markInjected: () => {},
+    } as any;
+    const r = await recallAndInlineAsync(
+      { sessionId: sess, query: "foreign", limit: 3, source: "command", crossRepo: true, globalIndexDir: indexDir },
+      mockStore,
+    );
+    assert.equal(r.toInject.length, 1, "fresh foreign checkpoint injected");
+    assert.equal(wasInjectedGlobal("chkpt_new", sess, indexDir), true, "recorded machine-wide");
+  } finally {
+    closeIndexStore();
+    rmSync(indexDir, { recursive: true, force: true });
+  }
+});
+
 test("cleanup", () => {
   rmSync(baseTmp, { recursive: true, force: true });
 });

package/src/recall.ts

@@ -38,6 +38,10 @@ export interface RecallInjectOptions {
   liveWindow?: string[];
   /** Similarity threshold for inline dedupe (defaults to 0.9). */
   dedupSim?: number;
+  /** S18: index dir of the machine-wide injected-set. When set on a cross-repo
+   *  recall, a foreign checkpoint already injected (in any session) is skipped
+   *  and a fresh injection is recorded globally. */
+  globalIndexDir?: string;
 }
 
 export interface RecallInjectResult {
@@ -56,8 +60,12 @@ export function formatRecallBlock(hits: SearchHit[]): string {
   if (hits.length === 0) return "";
   const parts = hits.map((h, i) => {
     const score = (h.score * 100).toFixed(0);
+    // S17: label a cross-repo hit with its source repo (the repoId doubles as
+    // that repo's stateDir, so the last path segment is the repo's display
+    // name). Same-repo hits (no repoId) stay unlabeled.
+    const repoName = h.repoId ? ` (from repo ${h.repoId.split("/").filter(Boolean).pop() ?? h.repoId})` : "";
     return (
-      `### Recalled context [${i + 1}] (relevance ${score}%)\n` +
+      `### Recalled context [${i + 1}] (relevance ${score}%)${repoName}\n` +
       `${h.checkpoint.summary.trim()}\n` +
       (h.checkpoint.filesModified.length
         ? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
@@ -138,6 +146,70 @@ export function recallAndInline(
   };
 }
 
+// --- S21: memory recall ----------------------------------------------------
+// Durables (decisions, rules, user-saved facts) live in the `memories` table.
+// We mirror the checkpoint recall path: rank by cosine, format a block, respect
+// a token cap so it can never net-inflate the system prompt.
+
+export interface MemoryRecallInjectOptions {
+  query: string;
+  stateDir: string;
+  limit?: number;
+  /** Token ceiling; defaults to the same `recallMaxTokens` used for checkpoints. */
+  recallMaxTokens?: number;
+  /** Cosine threshold; default 0.2. */
+  minSimilarity?: number;
+}
+
+/** Format one memory hit for the recall block. Category + score for traceability. */
+export function formatMemoryRecallBlock(
+  hits: Array<{ content: string; category: string | null; score: number }>,
+): string {
+  if (hits.length === 0) return "";
+  const parts = hits.map((h, i) => {
+    const pct = (h.score * 100).toFixed(0);
+    const cat = h.category ? `[${h.category}] ` : "";
+    return `### Recalled memory [${i + 1}] (relevance ${pct}%)\n${cat}${h.content.trim()}`;
+  });
+  return (
+    "The following facts about this project were saved from earlier turns " +
+    "and are relevant to the current request. Treat them as established:\n\n" +
+    parts.join("\n")
+  );
+}
+
+/** Recall top-k durable memories, format into a token-capped block. */
+export async function recallMemoriesAndInline(
+  opts: MemoryRecallInjectOptions,
+): Promise<{ empty: boolean; block: string; report: string[] }> {
+  const limit = opts.limit ?? 5;
+  const maxTokens = opts.recallMaxTokens ?? 0;
+  const { recallMemories } = await import("./memoryRecall.js");
+  const hits = await recallMemories(opts.query, opts.stateDir, {
+    topK: limit,
+    minSimilarity: opts.minSimilarity ?? 0.2,
+  });
+  if (hits.length === 0) return { empty: true, block: "", report: [] };
+
+  // Same incremental token cap pattern as checkpoint recall.
+  const parts: string[] = [];
+  const report: string[] = [];
+  let blockTokens = 0;
+  for (const h of hits) {
+    const part = formatMemoryRecallBlock([
+      { content: h.memory.content, category: h.memory.category, score: h.score },
+    ]);
+    const partTokens = estimateBlockTokens(part);
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+    parts.push(part);
+    report.push(
+      `  • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`,
+    );
+    blockTokens += partTokens;
+  }
+  return { empty: parts.length === 0, block: parts.join("\n"), report };
+}
+
 /**
  * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
  * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
@@ -180,6 +252,17 @@ export async function recallAndInlineAsync(
 
   for (const h of hits) {
     if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+    // S18: machine-wide injected-set — a foreign checkpoint already injected
+    // (in any session) is never re-injected. Only applies to cross-repo hits
+    // (same-repo hits have no repoId and are handled by the per-session set).
+    if (opts.globalIndexDir && h.repoId) {
+      try {
+        const { wasInjectedGlobal } = await import("./store/sqlite.js");
+        if (wasInjectedGlobal(h.checkpoint.checkpointId, opts.sessionId, opts.globalIndexDir)) continue;
+      } catch {
+        /* non-fatal: degrade to per-session injected-set only */
+      }
+    }
     if (doWindowDedupe && liveEmbeddings.length > 0) {
       const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
       if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
@@ -191,6 +274,16 @@ export async function recallAndInlineAsync(
     toInject.push(h);
     blockTokens += partTokens;
     store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+    // S18: record the cross-repo injection machine-wide so it's not re-injected
+    // by a later recall (same or different session).
+    if (opts.globalIndexDir && h.repoId) {
+      try {
+        const { markInjectedGlobal } = await import("./store/sqlite.js");
+        markInjectedGlobal(h.checkpoint.checkpointId, h.repoId, opts.sessionId, opts.globalIndexDir);
+      } catch {
+        /* non-fatal */
+      }
+    }
   }
 
   const block = parts.join("\n");