pi-mega-compact 0.4.28 → 0.5.1

package/src/recall.ts

@@ -38,6 +38,10 @@ export interface RecallInjectOptions {
   liveWindow?: string[];
   /** Similarity threshold for inline dedupe (defaults to 0.9). */
   dedupSim?: number;
+  /** S18: index dir of the machine-wide injected-set. When set on a cross-repo
+   *  recall, a foreign checkpoint already injected (in any session) is skipped
+   *  and a fresh injection is recorded globally. */
+  globalIndexDir?: string;
 }
 
 export interface RecallInjectResult {
@@ -56,8 +60,12 @@ export function formatRecallBlock(hits: SearchHit[]): string {
   if (hits.length === 0) return "";
   const parts = hits.map((h, i) => {
     const score = (h.score * 100).toFixed(0);
+    // S17: label a cross-repo hit with its source repo (the repoId doubles as
+    // that repo's stateDir, so the last path segment is the repo's display
+    // name). Same-repo hits (no repoId) stay unlabeled.
+    const repoName = h.repoId ? ` (from repo ${h.repoId.split("/").filter(Boolean).pop() ?? h.repoId})` : "";
     return (
-      `### Recalled context [${i + 1}] (relevance ${score}%)\n` +
+      `### Recalled context [${i + 1}] (relevance ${score}%)${repoName}\n` +
       `${h.checkpoint.summary.trim()}\n` +
       (h.checkpoint.filesModified.length
         ? `Key files: ${h.checkpoint.filesModified.join(", ")}.\n`
@@ -138,6 +146,70 @@ export function recallAndInline(
   };
 }
 
+// --- S21: memory recall ----------------------------------------------------
+// Durables (decisions, rules, user-saved facts) live in the `memories` table.
+// We mirror the checkpoint recall path: rank by cosine, format a block, respect
+// a token cap so it can never net-inflate the system prompt.
+
+export interface MemoryRecallInjectOptions {
+  query: string;
+  stateDir: string;
+  limit?: number;
+  /** Token ceiling; defaults to the same `recallMaxTokens` used for checkpoints. */
+  recallMaxTokens?: number;
+  /** Cosine threshold; default 0.2. */
+  minSimilarity?: number;
+}
+
+/** Format one memory hit for the recall block. Category + score for traceability. */
+export function formatMemoryRecallBlock(
+  hits: Array<{ content: string; category: string | null; score: number }>,
+): string {
+  if (hits.length === 0) return "";
+  const parts = hits.map((h, i) => {
+    const pct = (h.score * 100).toFixed(0);
+    const cat = h.category ? `[${h.category}] ` : "";
+    return `### Recalled memory [${i + 1}] (relevance ${pct}%)\n${cat}${h.content.trim()}`;
+  });
+  return (
+    "The following facts about this project were saved from earlier turns " +
+    "and are relevant to the current request. Treat them as established:\n\n" +
+    parts.join("\n")
+  );
+}
+
+/** Recall top-k durable memories, format into a token-capped block. */
+export async function recallMemoriesAndInline(
+  opts: MemoryRecallInjectOptions,
+): Promise<{ empty: boolean; block: string; report: string[] }> {
+  const limit = opts.limit ?? 5;
+  const maxTokens = opts.recallMaxTokens ?? 0;
+  const { recallMemories } = await import("./memoryRecall.js");
+  const hits = await recallMemories(opts.query, opts.stateDir, {
+    topK: limit,
+    minSimilarity: opts.minSimilarity ?? 0.2,
+  });
+  if (hits.length === 0) return { empty: true, block: "", report: [] };
+
+  // Same incremental token cap pattern as checkpoint recall.
+  const parts: string[] = [];
+  const report: string[] = [];
+  let blockTokens = 0;
+  for (const h of hits) {
+    const part = formatMemoryRecallBlock([
+      { content: h.memory.content, category: h.memory.category, score: h.score },
+    ]);
+    const partTokens = estimateBlockTokens(part);
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+    parts.push(part);
+    report.push(
+      `  • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`,
+    );
+    blockTokens += partTokens;
+  }
+  return { empty: parts.length === 0, block: parts.join("\n"), report };
+}
+
 /**
  * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
  * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
@@ -180,6 +252,17 @@ export async function recallAndInlineAsync(
 
   for (const h of hits) {
     if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+    // S18: machine-wide injected-set — a foreign checkpoint already injected
+    // (in any session) is never re-injected. Only applies to cross-repo hits
+    // (same-repo hits have no repoId and are handled by the per-session set).
+    if (opts.globalIndexDir && h.repoId) {
+      try {
+        const { wasInjectedGlobal } = await import("./store/sqlite.js");
+        if (wasInjectedGlobal(h.checkpoint.checkpointId, opts.sessionId, opts.globalIndexDir)) continue;
+      } catch {
+        /* non-fatal: degrade to per-session injected-set only */
+      }
+    }
     if (doWindowDedupe && liveEmbeddings.length > 0) {
       const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
       if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
@@ -191,6 +274,16 @@ export async function recallAndInlineAsync(
     toInject.push(h);
     blockTokens += partTokens;
     store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+    // S18: record the cross-repo injection machine-wide so it's not re-injected
+    // by a later recall (same or different session).
+    if (opts.globalIndexDir && h.repoId) {
+      try {
+        const { markInjectedGlobal } = await import("./store/sqlite.js");
+        markInjectedGlobal(h.checkpoint.checkpointId, h.repoId, opts.sessionId, opts.globalIndexDir);
+      } catch {
+        /* non-fatal */
+      }
+    }
   }
 
   const block = parts.join("\n");

package/src/store/sqlite.ts

@@ -57,7 +57,17 @@ const cache = new Map<string, DatabaseSync>();
 /** Open (or reuse) the SQLite store for a state dir. */
 export function openStore(stateDir: string = getStateDir()): DatabaseSync {
   const existing = cache.get(stateDir);
-  if (existing) return existing;
+  if (existing) {
+    // A closed handle in the cache (e.g. a test calling db.close() directly
+    // instead of closeStore) would surface as "database is not open" on the
+    // next reuse. Detect and evict so callers never see a dead handle.
+    try {
+      existing.prepare("SELECT 1");
+      return existing;
+    } catch {
+      cache.delete(stateDir);
+    }
+  }
 
   if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
   const db = new DatabaseSync(join(stateDir, "sqlite.db"));
@@ -121,6 +131,19 @@ export function openIndexStore(indexDir: string = getIndexDir()): DatabaseSync {
       model_captured_at         INTEGER
     );
     CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
+    -- S18: machine-wide injected-set. A foreign checkpoint injected in repo A is
+    -- recorded here so repo B's recall never re-injects it. Keyed by checkpoint
+    -- + session (a checkpoint may be injected once per session); repo_id is the
+    -- source repo (the foreign repo's stateDir) for tracking/source labels.
+    -- PRAMETERIZED queries (PREVENT-002); local node:sqlite (PREVENT-PI-004).
+    CREATE TABLE IF NOT EXISTS injected_global (
+      checkpoint_id TEXT NOT NULL,
+      repo_id       TEXT NOT NULL,
+      session_id    TEXT NOT NULL,
+      injected_at   INTEGER NOT NULL,
+      PRIMARY KEY (checkpoint_id, session_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_injected_global_cid ON injected_global(checkpoint_id);
   `);
   indexCache = iddb;
   indexCacheDir = indexDir;
@@ -160,6 +183,19 @@ export function upsertRepoRegistry(
     tokensSaved: number;
     compressedOriginalBytes: number;
     lastCompactedAt?: number | null;
+    // The fields below are optional passthroughs so test fixtures and the
+    // /api/repos active-window filter can seed them directly. They're also
+    // written by other paths (recordRepoModel, registry refresh) — passing
+    // them here is harmless because the ON CONFLICT clause keeps first_seen
+    // and the model columns from being clobbered.
+    firstSeen?: number;
+    lastSeen?: number;
+    provider?: string | null;
+    providerName?: string | null;
+    modelName?: string | null;
+    inputRate?: number | null;
+    outputRate?: number | null;
+    modelCapturedAt?: number | null;
   },
   indexDir: string = getIndexDir(),
 ): void {
@@ -168,26 +204,42 @@ export function upsertRepoRegistry(
   db.prepare(
     `INSERT INTO repo_registry
        (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
-        checkpoint_count, tokens_saved, compressed_original_bytes)
-     VALUES (@repo_root, @display_name, @state_dir, @now, @now, @last_compacted_at,
-             @checkpoint_count, @tokens_saved, @compressed_original_bytes)
+        checkpoint_count, tokens_saved, compressed_original_bytes,
+        provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
+     VALUES (@repo_root, @display_name, @state_dir, @first_seen, @last_seen, @last_compacted_at,
+             @checkpoint_count, @tokens_saved, @compressed_original_bytes,
+             @provider, @provider_name, @model_name, @input_rate, @output_rate, @model_captured_at)
      ON CONFLICT(repo_root) DO UPDATE SET
        display_name = excluded.display_name,
        state_dir = excluded.state_dir,
-       last_seen = excluded.last_seen,
+       last_seen = COALESCE(excluded.last_seen, @now),
        last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
        checkpoint_count = excluded.checkpoint_count,
        tokens_saved = excluded.tokens_saved,
-       compressed_original_bytes = excluded.compressed_original_bytes`,
+       compressed_original_bytes = excluded.compressed_original_bytes,
+       provider = COALESCE(excluded.provider, repo_registry.provider),
+       provider_name = COALESCE(excluded.provider_name, repo_registry.provider_name),
+       model_name = COALESCE(excluded.model_name, repo_registry.model_name),
+       input_rate = COALESCE(excluded.input_rate, repo_registry.input_rate),
+       output_rate = COALESCE(excluded.output_rate, repo_registry.output_rate),
+       model_captured_at = COALESCE(excluded.model_captured_at, repo_registry.model_captured_at)`,
   ).run({
     repo_root: row.repoRoot,
     display_name: row.displayName,
     state_dir: row.stateDir,
     now,
+    first_seen: row.firstSeen ?? null,
+    last_seen: row.lastSeen ?? null,
     last_compacted_at: row.lastCompactedAt ?? null,
     checkpoint_count: row.checkpointCount,
     tokens_saved: row.tokensSaved,
     compressed_original_bytes: row.compressedOriginalBytes,
+    provider: row.provider ?? null,
+    provider_name: row.providerName ?? null,
+    model_name: row.modelName ?? null,
+    input_rate: row.inputRate ?? null,
+    output_rate: row.outputRate ?? null,
+    model_captured_at: row.modelCapturedAt ?? null,
   });
 }
 
@@ -281,6 +333,51 @@ export function closeIndexStore(): void {
   }
 }
 
+// ---------------------------------------------------------------------------
+// S18: machine-wide injected-set (cross-repo dedup markers)
+//
+// A foreign checkpoint injected in repo A is recorded here so repo B's recall
+// never re-injects it (a stronger, machine-wide version of the per-session
+// injected-set in the local store). Keyed by (checkpoint_id, session_id); the
+// session_id here is the RECEIVING session, so the same foreign checkpoint can
+// be injected into different sessions but never twice into the same one.
+// PRAMETERIZED queries (PREVENT-002); local node:sqlite + WAL (PREVENT-PI-004),
+// multi-process safe.
+// ---------------------------------------------------------------------------
+
+/** Record that a (foreign) checkpoint was injected into `sessionId`. Idempotent. */
+export function markInjectedGlobal(
+  checkpointId: string,
+  repoId: string,
+  sessionId: string,
+  indexDir: string = getIndexDir(),
+): void {
+  const db = openIndexStore(indexDir);
+  db.prepare(
+    "INSERT OR IGNORE INTO injected_global (checkpoint_id, repo_id, session_id, injected_at) VALUES ($cid, $rid, $sid, $ts)",
+  ).run({ $cid: checkpointId, $rid: repoId, $sid: sessionId, $ts: Date.now() });
+}
+
+/** True when a checkpoint was already injected into `sessionId` (machine-wide). */
+export function wasInjectedGlobal(
+  checkpointId: string,
+  sessionId: string,
+  indexDir: string = getIndexDir(),
+): boolean {
+  const db = openIndexStore(indexDir);
+  const row = db.prepare(
+    "SELECT 1 FROM injected_global WHERE checkpoint_id = $cid AND session_id = $sid LIMIT 1",
+  ).get({ $cid: checkpointId, $sid: sessionId }) as { "1": number } | undefined;
+  return row !== undefined;
+}
+
+/** Count of cross-repo injections recorded (for /mega-status stats). */
+export function countInjectedGlobal(indexDir: string = getIndexDir()): number {
+  const db = openIndexStore(indexDir);
+  const row = db.prepare("SELECT COUNT(*) AS n FROM injected_global").get() as { n: number } | undefined;
+  return row?.n ?? 0;
+}
+
 function initSchema(db: DatabaseSync): void {
   db.exec(`
     CREATE TABLE IF NOT EXISTS context_chunks (
@@ -430,7 +527,12 @@ function initSchema(db: DatabaseSync): void {
       content           TEXT NOT NULL,
       tags              TEXT,                   -- JSON array of strings
       created_at        INTEGER,
-      last_recalled_at  INTEGER
+      last_recalled_at  INTEGER,
+      -- S20 memory-RAG extension (auto-review add/replace/remove ops).
+      category          TEXT,                   -- typed bucket, e.g. decision | fact | preference
+      target            TEXT,                   -- optional subject/scope this memory targets
+      last_referenced   INTEGER,               -- last time memory was referenced by recall (epoch s)
+      source_turn       INTEGER                -- conversation turn that produced this memory
     );
     CREATE INDEX IF NOT EXISTS idx_memories_repo ON memories(repo);
 
@@ -447,6 +549,12 @@ function initSchema(db: DatabaseSync): void {
   // databases created by an older version — otherwise repoStats()/upsert crash
   // with "no such column" and the extension fails to load. Additive only.
   ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
+  // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
+  // only alters DBs created by an older version that lack these columns.
+  ensureColumn(db, "memories", "category", "TEXT");
+  ensureColumn(db, "memories", "target", "TEXT");
+  ensureColumn(db, "memories", "last_referenced", "INTEGER");
+  ensureColumn(db, "memories", "source_turn", "INTEGER");
   const v = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get() as
     | { value: string }
     | undefined;
@@ -613,11 +721,15 @@ export interface MemoryRecord {
   tags: string[];
   createdAt: number;
   lastRecalledAt: number | null;
+  category: string | null;
+  target: string | null;
+  lastReferenced: number | null;
+  sourceTurn: number | null;
 }
 
 /** Save a memory to the current repo's store. Returns the new row id. */
 export function addMemory(
-  memory: { kind?: string; content: string; tags?: string[] },
+  memory: { kind?: string; content: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
   repo: string | null,
   stateDir: string = getStateDir(),
 ): number {
@@ -625,10 +737,19 @@ export function addMemory(
   const now = Math.floor(Date.now() / 1000);
   const res = db
     .prepare(
-      `INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
-       VALUES(?, ?, ?, ?, ?, NULL)`,
+      `INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
+       VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`,
     )
-    .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
+    .run(
+      repo ?? null,
+      memory.kind ?? "note",
+      memory.content,
+      JSON.stringify(memory.tags ?? []),
+      now,
+      memory.category ?? null,
+      memory.target ?? null,
+      memory.sourceTurn ?? null,
+    );
   return Number(res.lastInsertRowid);
 }
 
@@ -659,6 +780,58 @@ export function recallMemory(id: number, stateDir: string = getStateDir()): bool
   return res.changes > 0;
 }
 
+/** Mark a memory as referenced (updates last_referenced). Returns true if found. */
+export function referenceMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
+  return res.changes > 0;
+}
+
+/** Replace a memory's mutable fields by id. Returns true if a row was updated. */
+export function replaceMemory(
+  id: number,
+  patch: { kind?: string; content?: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
+  stateDir: string = getStateDir(),
+): boolean {
+  const db = openStore(stateDir);
+  const res = db
+    .prepare(
+      `UPDATE memories
+       SET kind = COALESCE(?, kind),
+           content = COALESCE(?, content),
+           tags = COALESCE(?, tags),
+           category = COALESCE(?, category),
+           target = COALESCE(?, target),
+           source_turn = COALESCE(?, source_turn)
+       WHERE id = ?`,
+    )
+    .run(
+      patch.kind ?? null,
+      patch.content ?? null,
+      patch.tags ? JSON.stringify(patch.tags) : null,
+      "category" in patch ? (patch.category ?? null) : null,
+      "target" in patch ? (patch.target ?? null) : null,
+      "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null,
+      id,
+    );
+  return res.changes > 0;
+}
+
+/** Remove a memory by id. Returns true if a row was deleted. */
+export function removeMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
+  return res.changes > 0;
+}
+
+/** Look up a single memory by id (or undefined). */
+export function getMemory(id: number, stateDir: string = getStateDir()): MemoryRecord | undefined {
+  const db = openStore(stateDir);
+  const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
+  return row ? mapMemoryRow(row) : undefined;
+}
+
 function mapMemoryRow(row: any): MemoryRecord {
   return {
     id: row.id,
@@ -668,6 +841,10 @@ function mapMemoryRow(row: any): MemoryRecord {
     tags: row.tags ? JSON.parse(row.tags) : [],
     createdAt: row.created_at ?? 0,
     lastRecalledAt: row.last_recalled_at ?? null,
+    category: row.category ?? null,
+    target: row.target ?? null,
+    lastReferenced: row.last_referenced ?? null,
+    sourceTurn: row.source_turn ?? null,
   };
 }
 

package/src/store.ts

@@ -49,6 +49,9 @@ export function normalizeSessionId(sessionId: string | undefined | null): string
 export interface StoredCheckpoint {
   checkpointId: string;
   sessionId: string;
+  /** Source repo id (foreign stateDir) when this checkpoint came from a global
+   *  cross-repo index entry. Undefined for same-repo checkpoints. Spec S17.1. */
+  repoId?: string;
   summary: string;
   /** Compressed topic summary (extractive, ~2K tokens vs ~70K raw). */
   topicSummary?: string;

package/src/vectorStore.ts

@@ -51,6 +51,10 @@ import { migrateJsonToSqlite } from "./store/migrate.js";
 export interface SearchHit {
   checkpoint: StoredCheckpoint;
   score: number;
+  /** Source repo id (the foreign repo's stateDir) for cross-repo hits, set by
+   *  `searchAsync` so the recall block can label foreign checkpoints. Undefined
+   *  for same-repo hits (the default path). */
+  repoId?: string;
 }
 
 export interface AddInput {
@@ -326,6 +330,7 @@ export class VectorStore {
     const checkpoint: StoredCheckpoint = {
       checkpointId,
       sessionId,
+      repoId: this.repoId,
       summary: input.summary,
       topicSummary: input.topicSummary,
       summaryHash,
@@ -525,11 +530,15 @@ export class VectorStore {
     }
     // Hydrate each index hit from the authoritative node:sqlite store. repoId is
     // that repo's stateDir, so cross-repo hits resolve against their own store.
+    // Tag cross-repo hits with their source repoId so the recall block can label
+    // them ("from repo <name>"); same-repo hits stay unlabeled.
+    const selfRepo = this.repoId;
     const hydrated: SearchHit[] = [];
     for (const h of indexHits) {
       const cp = getCheckpoint(h.sessionId, h.checkpointId, h.repoId);
       if (cp && cp.dedupStatus !== "removed") {
-        hydrated.push({ checkpoint: cp, score: h.score });
+        const crossRepo = opts.crossRepo && selfRepo && h.repoId && h.repoId !== selfRepo;
+        hydrated.push({ checkpoint: cp, score: h.score, repoId: crossRepo ? h.repoId : undefined });
       }
     }
     if (hydrated.length === 0) return this.search(sid, query, k);