pi-mega-compact 0.6.1 → 0.6.3

package/src/recall.ts

@@ -159,6 +159,10 @@ export interface MemoryRecallInjectOptions {
   recallMaxTokens?: number;
   /** Cosine threshold; default 0.2. */
   minSimilarity?: number;
+  /** When true, augment same-repo recall with cross-repo PGlite NN (S24). */
+  crossRepo?: boolean;
+  /** Stricter cosine floor for cross-repo memory hits (S24). Default 0.3. */
+  crossRepoCosine?: number;
 }
 
 /** Format one memory hit for the recall block. Category + score for traceability. */
@@ -184,28 +188,49 @@ export async function recallMemoriesAndInline(
 ): Promise<{ empty: boolean; block: string; report: string[] }> {
   const limit = opts.limit ?? 5;
   const maxTokens = opts.recallMaxTokens ?? 0;
-  const { recallMemories } = await import("./memoryRecall.js");
+  const { recallMemories, recallMemoriesCrossRepo } = await import("./memoryRecall.js");
   const hits = await recallMemories(opts.query, opts.stateDir, {
     topK: limit,
     minSimilarity: opts.minSimilarity ?? 0.2,
   });
-  if (hits.length === 0) return { empty: true, block: "", report: [] };
+
+  // S24 cross-repo augmentation: if same-repo recall is thin, pull additional
+  // memories from OTHER repos via the PGlite HNSW index. Non-fatal: a failure
+  // degrades to the same-repo hits only.
+  const crossHits: Array<{ memory: any; score: number; repoId: string }> = [];
+  if (opts.crossRepo && hits.length < limit) {
+    try {
+      const x = await recallMemoriesCrossRepo(opts.query, opts.stateDir, {
+        repo: null,
+        limit: limit - hits.length,
+        crossRepoCosine: opts.crossRepoCosine ?? 0.3,
+      });
+      for (const h of x) crossHits.push(h);
+    } catch {
+      /* non-fatal — cross-repo failure → same-repo only */
+    }
+  }
+  if (hits.length === 0 && crossHits.length === 0) return { empty: true, block: "", report: [] };
 
   // Same incremental token cap pattern as checkpoint recall.
   const parts: string[] = [];
   const report: string[] = [];
   let blockTokens = 0;
-  for (const h of hits) {
-    const part = formatMemoryRecallBlock([
-      { content: h.memory.content, category: h.memory.category, score: h.score },
-    ]);
+  const pushHit = (content: string, category: string | null, score: number, label: string) => {
+    const part = formatMemoryRecallBlock([{ content, category, score }]);
     const partTokens = estimateBlockTokens(part);
-    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) return false;
     parts.push(part);
-    report.push(
-      `  • memory#${h.memory.id} (${(h.score * 100).toFixed(0)}%): ${h.memory.content.slice(0, 60).replace(/\n/g, " ")}…`,
-    );
+    report.push(`  • ${label} (${(score * 100).toFixed(0)}%): ${content.slice(0, 60).replace(/\n/g, " ")}…`);
     blockTokens += partTokens;
+    return true;
+  };
+  for (const h of hits) {
+    if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id}`)) break;
+  }
+  for (const h of crossHits) {
+    const repoLabel = h.repoId.split(/[\\/]/).filter(Boolean).pop() ?? h.repoId;
+    if (!pushHit(h.memory.content, h.memory.category, h.score, `memory#${h.memory.id} (from ${repoLabel})`)) break;
   }
   return { empty: parts.length === 0, block: parts.join("\n"), report };
 }

package/src/store/memoryIndex.test.ts

@@ -0,0 +1,61 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { defaultEmbedder } from "../embedder.js";
+import {
+  upsertMemoryEmbedding,
+  searchMemoriesAsync,
+  initMemoryIndex,
+  closeMemoryIndex,
+  isMemoryIndexDisabled,
+} from "./memoryIndex.js";
+
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-memidx-"));
+
+test("memoryIndex: disabled when MEGACOMPACT_PGLITE_DISABLED", async () => {
+  process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
+  try {
+    assert.equal(isMemoryIndexDisabled(), true, "kill-switch honored");
+    const hits = await searchMemoriesAsync(defaultEmbedder().embed("anything"), { k: 3 });
+    assert.deepEqual(hits, [], "search returns [] when disabled");
+  } finally {
+    delete process.env.MEGACOMPACT_PGLITE_DISABLED;
+  }
+});
+
+test("memoryIndex: cross-repo upsert + NN search returns the right repo's memory", async () => {
+  // Isolate the global PGlite dir so concurrent test runs don't collide.
+  process.env.MEGACOMPACT_INDEX_DIR = join(baseTmp, "index");
+  const repoA = "/tmp/repo-a";
+  const repoB = "/tmp/repo-b";
+  try {
+    await initMemoryIndex();
+    // Two memories in different repos, with clearly distinct content so their
+    // trigram embeddings separate.
+    const vecA = defaultEmbedder().embed("We standardized on node:sqlite for the store backend");
+    const vecB = defaultEmbedder().embed("The deployment target is a raspberry pi in the closet");
+    await upsertMemoryEmbedding(repoA, 1, "We standardized on node:sqlite for the store backend", vecA);
+    await upsertMemoryEmbedding(repoB, 7, "The deployment target is a raspberry pi in the closet", vecB);
+
+    // Query close to A's content → top hit should be A's memory, not B's.
+    const q = defaultEmbedder().embed("standardized node:sqlite store backend choice");
+    const hits = await searchMemoriesAsync(q, { k: 3 });
+    assert.ok(hits.length >= 1, "at least one hit");
+    assert.equal(hits[0].repoId, repoA, "nearest neighbor is repo A");
+    assert.equal(hits[0].memoryId, 1, "correct memory id");
+    assert.ok(hits[0].score > 0.5, "high cosine for the matching memory");
+
+    // Scope to repoB only → A must not appear.
+    const scoped = await searchMemoriesAsync(q, { k: 3, repoId: repoB });
+    assert.ok(scoped.every((h) => h.repoId === repoB), "scoped search stays within repoB");
+  } finally {
+    await closeMemoryIndex();
+    delete process.env.MEGACOMPACT_INDEX_DIR;
+  }
+});
+
+test("memoryIndex: cleanup", () => {
+  rmSync(baseTmp, { recursive: true, force: true });
+});

package/src/store/memoryIndex.ts

@@ -0,0 +1,269 @@
+/**
+ * memoryIndex.ts — cross-repo async vector index for durable memories (S24).
+ *
+ * A REDUNDANT, additive, ASYNC index layered over the authoritative node:sqlite
+ * `memories` table. The same-repo linear cosine scan over the in-repo memories
+ * (src/memoryRecall.ts) stays the DEFAULT recall path; this global PGlite index
+ * exists only to provide real cross-repo nearest-neighbor memory recall — so a
+ * decision you saved in repo A can be inlined as RAG context when you start a
+ * session in repo B. It is best-effort and non-fatal: any init/write failure
+ * degrades to the same-repo scan and must NEVER break memory write, recall, or
+ * extension load.
+ *
+ * PREVENT-PI-004: PGlite is WASM Postgres — fully local, zero network. Memory
+ * remains AUTHORITATIVE in SQLite; this index only holds (repo_id, memory_id,
+ * content, embedding) for NN lookup and is rebuilt from SQLite at any time.
+ *
+ * Topology mirrors vectorIndex.ts (Slice 2): ONE global PGlite DB, `repo_id` is
+ * a first-class column. `searchMemoriesAsync(q, k, {repoId?})` → omit repoId for
+ * cross-repo NN, pass repoId to scope to a single repo. Hit content is stored
+ * inline because the recall process cannot open every other repo's SQLite dir.
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { mkdirSync, rmSync, existsSync } from "node:fs";
+
+// PGlite + pgvector are script-free WASM (no native build) → survive pi's
+// install-script block. The VALUE import is LAZY (dynamic import inside
+// openPgLite) so a missing/broken package degrades to the same-repo scan instead
+// of crashing module load. A static top-level `import { PGlite }` would throw
+// "Cannot find module" at pi startup and take down the whole extension. The
+// `import type` below is erased at compile time and emits NO runtime load.
+import type { PGlite as PGliteInstance, Extension } from "@electric-sql/pglite";
+
+/** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
+export const MEMORY_INDEX_DIM = 512;
+
+/** A single cross-repo memory hit returned by the async index. */
+export interface MemoryIndexHit {
+  repoId: string;
+  memoryId: number;
+  /** Inline content so recall can read it without opening the other repo's db. */
+  content: string;
+  /** Cosine similarity in [0,1] (1 = identical). */
+  score: number;
+}
+
+let db: PGliteInstance | undefined;
+let initPromise: Promise<PGliteInstance | undefined> | undefined;
+let disabled = false;
+let warned = false;
+/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
+let pgliteMod: {
+  PGlite: typeof import("@electric-sql/pglite")["PGlite"];
+  vector: Extension;
+} | undefined;
+let pgliteLoadFailed = false;
+
+function indexDir(): string {
+  const override = process.env.MEGACOMPACT_INDEX_DIR;
+  if (override && override.trim() !== "") return join(override, "memory");
+  try {
+    return join(homedir(), ".pi", "mega-compact-vector", "memory");
+  } catch {
+    return join("/tmp", ".mega-compact-vector", "memory");
+  }
+}
+
+function logWarn(msg: string): void {
+  // Never throw — degradation is the whole point. One warning per process.
+  if (warned) return;
+  warned = true;
+  try {
+    console.warn(`[mega-compact:memoryIndex] ${msg} (falling back to same-repo scan)`);
+  } catch {
+    /* ignore */
+  }
+}
+
+/** Honor the emergency kill-switch (shared with the checkpoint index). */
+export function isMemoryIndexDisabled(): boolean {
+  return (
+    disabled ||
+    process.env.MEGACOMPACT_PGLITE_DISABLED === "true" ||
+    process.env.MEGACOMPACT_PGLITE_DISABLED === "1"
+  );
+}
+
+/**
+ * Lazily open + schema-init the global PGlite DB. Idempotent and safe to call
+ * from many places. Returns undefined when disabled/unavailable so callers can
+ * fall back to the synchronous scan. Never throws.
+ */
+export function initMemoryIndex(): Promise<PGliteInstance | undefined> {
+  if (isMemoryIndexDisabled()) return Promise.resolve(undefined);
+  if (db) return Promise.resolve(db);
+  if (initPromise) return initPromise;
+  initPromise = openPgLite(/* retryOnCorrupt */ true);
+  return initPromise;
+}
+
+/**
+ * Lazily load the PGlite module + pgvector extension via dynamic import. Caches
+ * success and permanent failure. Returns undefined (once, then forever) when the
+ * package is missing/broken so callers fall back to the same-repo scan. Never throws.
+ */
+async function loadPgLite(): Promise<
+  { PGlite: typeof import("@electric-sql/pglite")["PGlite"]; vector: Extension } | undefined
+> {
+  if (pgliteMod) return pgliteMod;
+  if (pgliteLoadFailed) return undefined;
+  try {
+    const [pglitePkg, pgvectorPkg] = await Promise.all([
+      import("@electric-sql/pglite"),
+      import("@electric-sql/pglite-pgvector"),
+    ]);
+    pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
+    return pgliteMod;
+  } catch (err) {
+    pgliteLoadFailed = true;
+    logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
+    return undefined;
+  }
+}
+
+/**
+ * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level abort
+ * (typically from a corrupted/torn data dir) triggers a delete + one retry.
+ */
+async function openPgLite(
+  retryOnCorrupt: boolean,
+): Promise<PGliteInstance | undefined> {
+  try {
+    const mod = await loadPgLite();
+    if (!mod) return undefined;
+    const dir = indexDir();
+    mkdirSync(dir, { recursive: true });
+    const pg = await new mod.PGlite({
+      dataDir: dir,
+      extensions: { vector: mod.vector },
+    });
+    await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
+    await pg.exec(`
+      CREATE TABLE IF NOT EXISTS memory_index (
+        repo_id    TEXT NOT NULL,
+        memory_id  INTEGER NOT NULL,
+        content    TEXT NOT NULL,
+        embedding  vector(${MEMORY_INDEX_DIM}) NOT NULL,
+        PRIMARY KEY (repo_id, memory_id)
+      );
+    `);
+    await pg.exec(
+      "CREATE INDEX IF NOT EXISTS memory_index_hnsw ON memory_index USING hnsw (embedding vector_cosine_ops);",
+    );
+    db = pg;
+    return pg;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (retryOnCorrupt && (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
+      try {
+        const dir = indexDir();
+        if (existsSync(dir)) rmSync(dir, { recursive: true, force: true });
+        initPromise = undefined;
+        return openPgLite(/* retryOnCorrupt */ false);
+      } catch {
+        /* self-heal failed — fall through to disable */
+      }
+    }
+    disabled = true;
+    logWarn(`init failed: ${msg}`);
+    return undefined;
+  }
+}
+
+function toVectorLiteral(v: number[]): string {
+  const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
+  return `[${parts.join(",")}]`;
+}
+
+/**
+ * Best-effort upsert of one memory embedding into the global index.
+ * Dimension-mismatched vectors (e.g. a BYO embedder with dim ≠ 512) are skipped.
+ * Fire-and-forget: callers must NOT await this on the sync write path. Never
+ * throws. `content` is stored inline so cross-repo recall can read it directly.
+ */
+export async function upsertMemoryEmbedding(
+  repoId: string,
+  memoryId: number,
+  content: string,
+  embedding: number[],
+): Promise<void> {
+  if (isMemoryIndexDisabled()) return;
+  if (!embedding || embedding.length !== MEMORY_INDEX_DIM) return;
+  try {
+    const pg = await initMemoryIndex();
+    if (!pg) return;
+    const lit = toVectorLiteral(embedding);
+    await pg.query(
+      `INSERT INTO memory_index (repo_id, memory_id, content, embedding)
+       VALUES ($1, $2, $3, $4::vector)
+       ON CONFLICT (repo_id, memory_id)
+       DO UPDATE SET content = EXCLUDED.content, embedding = EXCLUDED.embedding;`,
+      [repoId, memoryId, content, lit],
+    );
+  } catch (err) {
+    disabled = true;
+    logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+export interface SearchMemoriesAsyncOpts {
+  /** When provided, scope the NN search to a single repo; omit for cross-repo. */
+  repoId?: string;
+  /** Max hits (default 5). */
+  k?: number;
+}
+
+/**
+ * Cross-repo (or single-repo) HNSW nearest-neighbor memory search. Returns hits
+ * sorted by descending similarity. Never throws — on any failure returns [].
+ */
+export async function searchMemoriesAsync(
+  query: number[],
+  opts: SearchMemoriesAsyncOpts = {},
+): Promise<MemoryIndexHit[]> {
+  if (isMemoryIndexDisabled() || !query || query.length !== MEMORY_INDEX_DIM) return [];
+  const k = opts.k ?? 5;
+  const repoId = opts.repoId;
+  try {
+    const pg = await initMemoryIndex();
+    if (!pg) return [];
+    const lit = toVectorLiteral(query);
+    const params: unknown[] = [lit, k];
+    let sql =
+      "SELECT repo_id, memory_id, content, 1 - (embedding <=> $1::vector) AS score " +
+      "FROM memory_index";
+    if (repoId) {
+      sql += " WHERE repo_id = $3";
+      params.push(repoId);
+    }
+    sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
+    const res = await pg.query(sql, params);
+    return res.rows.map((r: any) => ({
+      repoId: r.repo_id as string,
+      memoryId: Number(r.memory_id),
+      content: r.content as string,
+      score: r.score as number,
+    }));
+  } catch (err) {
+    disabled = true;
+    logWarn(`search failed: ${err instanceof Error ? err.message : String(err)}`);
+    return [];
+  }
+}
+
+/** Close the index (test teardown / shutdown). Safe to call when unopened. */
+export async function closeMemoryIndex(): Promise<void> {
+  if (db) {
+    try {
+      await db.close();
+    } catch {
+      /* ignore */
+    }
+  }
+  db = undefined;
+  initPromise = undefined;
+  disabled = false;
+  warned = false;
+}

package/src/store/vectorIndex.ts

@@ -21,10 +21,12 @@ import { join } from "node:path";
 import { mkdirSync, rmSync, existsSync } from "node:fs";
 
 // PGlite + pgvector are script-free WASM (no native build) → survive pi's
-// install-script block. Imported lazily so a missing/broken package degrades
-// gracefully instead of crashing module load.
-import { PGlite, type PGlite as PGliteInstance } from "@electric-sql/pglite";
-import { vector } from "@electric-sql/pglite-pgvector";
+// install-script block. The VALUE import is LAZY (dynamic import inside
+// openPgLite) so a missing/broken package degrades to the sync scan instead of
+// crashing module load. A static top-level `import { PGlite }` would throw
+// "Cannot find module" at pi startup and take down the whole extension. The
+// `import type` below is erased at compile time and emits NO runtime load.
+import type { PGlite as PGliteInstance, Extension } from "@electric-sql/pglite";
 
 /** Vector dimension produced by the default TrigramEmbedder (src/embedder.ts). */
 export const EMBEDDING_DIM = 512;
@@ -42,6 +44,12 @@ let db: PGliteInstance | undefined;
 let initPromise: Promise<PGliteInstance | undefined> | undefined;
 let disabled = false;
 let warned = false;
+/** Lazily-loaded PGlite module + pgvector extension (see loadPgLite). */
+let pgliteMod: {
+  PGlite: typeof import("@electric-sql/pglite")["PGlite"];
+  vector: Extension;
+} | undefined;
+let pgliteLoadFailed = false;
 
 function indexDir(): string {
   const override = process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
@@ -86,6 +94,30 @@ export function initVectorIndex(): Promise<PGliteInstance | undefined> {
   return initPromise;
 }
 
+/**
+ * Lazily load the PGlite module + pgvector extension via dynamic import. Caches
+ * success and permanent failure. Returns undefined (once, then forever) when the
+ * package is missing/broken so callers fall back to the sync scan. Never throws.
+ */
+async function loadPgLite(): Promise<
+  { PGlite: typeof import("@electric-sql/pglite")["PGlite"]; vector: Extension } | undefined
+> {
+  if (pgliteMod) return pgliteMod;
+  if (pgliteLoadFailed) return undefined;
+  try {
+    const [pglitePkg, pgvectorPkg] = await Promise.all([
+      import("@electric-sql/pglite"),
+      import("@electric-sql/pglite-pgvector"),
+    ]);
+    pgliteMod = { PGlite: pglitePkg.PGlite, vector: pgvectorPkg.vector };
+    return pgliteMod;
+  } catch (err) {
+    pgliteLoadFailed = true;
+    logWarn(`package unavailable: ${err instanceof Error ? err.message : String(err)}`);
+    return undefined;
+  }
+}
+
 /**
  * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
  * abort (typically from a corrupted/torn data dir) triggers a delete + one
@@ -95,11 +127,13 @@ async function openPgLite(
   retryOnCorrupt: boolean,
 ): Promise<PGliteInstance | undefined> {
   try {
+    const mod = await loadPgLite();
+    if (!mod) return undefined;
     const dir = indexDir();
     mkdirSync(dir, { recursive: true });
-    const pg = await new PGlite({
+    const pg = await new mod.PGlite({
       dataDir: dir,
-      extensions: { vector },
+      extensions: { vector: mod.vector },
     });
     await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
     await pg.exec(`