pi-mega-compact 0.4.25 → 0.4.26

package/dist/extensions/mega-pipeline.js

@@ -15,6 +15,7 @@ import { C, MARKER_TYPE, } from "./mega-runtime.js";
 import { resolveRepoRoot, preserveRecentForPressure } from "./mega-config.js";
 import { runRaptor } from "../src/dedup/raptor/index.js";
 import { loadDedupConfig } from "../src/config/dedup.js";
+import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
 /** Run the full compaction pipeline and persist a checkpoint. Returns the result. */
 export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
     runtime.bindRepo(ctx.cwd);
@@ -131,6 +132,24 @@ export function runCompact(pi, runtime, config, ctx, messages, opts = {}) {
             /* non-fatal: tree refresh never blocks a compaction */
         }
     }
+    // Slice 2: best-effort mirror of the new checkpoint into the async global
+    // PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
+    // shared global dir is never hammered by concurrent test workers.
+    // Non-fatal: a WASM init failure degrades to the sync scan silently.
+    if (!result.deduped) {
+        try {
+            const all = runtime.store.list(sid);
+            const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
+            if (latest?.embedding) {
+                void indexUpsertEmbedding(runtime.currentStateDir, sid, latest.checkpointId, latest.embedding).catch(() => {
+                    /* non-fatal: index refresh never blocks a compaction */
+                });
+            }
+        }
+        catch {
+            /* non-fatal: index refresh never blocks a compaction */
+        }
+    }
     runtime.setStatus(ctx, runtime.rt.persistedThisSession
         ? `mega-compact: ${result.checkpointId} · ${saved} tok saved`
         : `mega-compact: ready`);

package/dist/src/store/vectorIndex.js

@@ -17,7 +17,7 @@
  */
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { mkdirSync } from "node:fs";
+import { mkdirSync, rmSync, existsSync } from "node:fs";
 // PGlite + pgvector are script-free WASM (no native build) → survive pi's
 // install-script block. Imported lazily so a missing/broken package degrades
 // gracefully instead of crashing module load.
@@ -70,37 +70,61 @@ export function initVectorIndex() {
         return Promise.resolve(db);
     if (initPromise)
         return initPromise;
-    initPromise = (async () => {
-        try {
-            const dir = indexDir();
-            mkdirSync(dir, { recursive: true });
-            const pg = await new PGlite({
-                dataDir: dir,
-                extensions: { vector },
-            });
-            await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
-            await pg.exec(`
-        CREATE TABLE IF NOT EXISTS vector_index (
-          repo_id       TEXT NOT NULL,
-          session_id    TEXT NOT NULL,
-          checkpoint_id TEXT NOT NULL,
-          embedding     vector(${EMBEDDING_DIM}) NOT NULL,
-          PRIMARY KEY (repo_id, session_id, checkpoint_id)
-        );
-      `);
-            // HNSW index over cosine distance for fast NN. Created idempotently.
-            await pg.exec("CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);");
-            db = pg;
-            return pg;
-        }
-        catch (err) {
-            disabled = true;
-            logWarn(`init failed: ${err instanceof Error ? err.message : String(err)}`);
-            return undefined;
-        }
-    })();
+    initPromise = openPgLite(/* retryOnCorrupt */ true);
     return initPromise;
 }
+/**
+ * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
+ * abort (typically from a corrupted/torn data dir) triggers a delete + one
+ * retry — the dir is rebuilt from scratch by PGlite's initdb.
+ */
+async function openPgLite(retryOnCorrupt) {
+    try {
+        const dir = indexDir();
+        mkdirSync(dir, { recursive: true });
+        const pg = await new PGlite({
+            dataDir: dir,
+            extensions: { vector },
+        });
+        await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
+        await pg.exec(`
+      CREATE TABLE IF NOT EXISTS vector_index (
+        repo_id       TEXT NOT NULL,
+        session_id    TEXT NOT NULL,
+        checkpoint_id TEXT NOT NULL,
+        embedding     vector(${EMBEDDING_DIM}) NOT NULL,
+        PRIMARY KEY (repo_id, session_id, checkpoint_id)
+      );
+    `);
+        // HNSW index over cosine distance for fast NN. Created idempotently.
+        await pg.exec("CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);");
+        db = pg;
+        return pg;
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        // Self-heal: a WASM Aborted() typically means the data dir is corrupted
+        // (torn WAL from concurrent access). Delete it and retry once.
+        if (retryOnCorrupt &&
+            (msg.includes("Aborted") || msg.includes("RuntimeError"))) {
+            try {
+                const dir = indexDir();
+                if (existsSync(dir)) {
+                    rmSync(dir, { recursive: true, force: true });
+                }
+                // Clear singleton state so the retry starts fresh.
+                initPromise = undefined;
+                return openPgLite(/* retryOnCorrupt */ false);
+            }
+            catch {
+                // Self-heal failed — fall through to disable.
+            }
+        }
+        disabled = true;
+        logWarn(`init failed: ${msg}`);
+        return undefined;
+    }
+}
 function toVectorLiteral(v) {
     // pgvector text form: [a,b,c]. Guard against NaN/Inf for a clean literal.
     const parts = v.map((x) => (Number.isFinite(x) ? x : 0));
@@ -182,6 +206,8 @@ export async function closeVectorIndex() {
     }
     db = undefined;
     initPromise = undefined;
+    disabled = false;
+    warned = false;
 }
 /**
  * Rebuild the entire index from the authoritative node:sqlite store. Used for

package/dist/src/vectorStore.js

@@ -20,7 +20,7 @@ import { mmrRerank } from "./dedup/mmr.js";
 import { topK } from "./dedup/topk.js";
 import { openBloom, saveBloom } from "./store/bloom.js";
 import { listCheckpoints, nextCheckpointId, upsertCheckpoint, getCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, setDedupStatus, addTokensSaved, getDedupStats, bumpDedupStats, repoStats as repoStatsFromStore, dataInvariantStats, } from "./store/sqlite.js";
-import { upsertEmbedding as indexUpsertEmbedding, initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
+import { initVectorIndex, searchAsync as vectorIndexSearch, } from "./store/vectorIndex.js";
 import { rehydrateRaptorTree } from "./dedup/raptor/index.js";
 import { stagedExpansion } from "./dedup/raptor/retrieval.js";
 import { migrateJsonToSqlite } from "./store/migrate.js";
@@ -292,12 +292,6 @@ export class VectorStore {
         // Cumulative store-wide dedup accounting (attempt, not collapsed).
         bumpDedupStats(false, this.stateDir);
         onTier?.({ tier: "new", status: "stored" });
-        // Slice 2: best-effort, fire-and-forget mirror of this new checkpoint into
-        // the async global PGlite/HNSW index. NEVER awaited — must not block or
-        // throw into the synchronous add() path. On failure the index degrades to
-        // the sync scan (handled inside vectorIndex). The node:sqlite store remains
-        // authoritative; the index is rebuildable from it at any time.
-        void indexUpsertEmbedding(this.repoId, sessionId, checkpointId, checkpoint.embedding);
         return { checkpoint, deduped: false };
     }
     /**

package/extensions/mega-pipeline.ts

@@ -22,6 +22,7 @@ import {
 import { resolveRepoRoot, preserveRecentForPressure, type MegaConfig } from "./mega-config.js";
 import { runRaptor } from "../src/dedup/raptor/index.js";
 import { loadDedupConfig } from "../src/config/dedup.js";
+import { upsertEmbedding as indexUpsertEmbedding } from "../src/store/vectorIndex.js";
 
 export type RunCompactResult =
   | { skipped: true }
@@ -164,6 +165,29 @@ export function runCompact(
     }
   }
 
+  // Slice 2: best-effort mirror of the new checkpoint into the async global
+  // PGlite/HNSW vector index. Fires once per compaction (not per-add), so the
+  // shared global dir is never hammered by concurrent test workers.
+  // Non-fatal: a WASM init failure degrades to the sync scan silently.
+  if (!result.deduped) {
+    try {
+      const all = runtime.store.list(sid);
+      const latest = all.find((cp) => cp.checkpointId === result.checkpointId);
+      if (latest?.embedding) {
+        void indexUpsertEmbedding(
+          runtime.currentStateDir,
+          sid,
+          latest.checkpointId,
+          latest.embedding,
+        ).catch(() => {
+          /* non-fatal: index refresh never blocks a compaction */
+        });
+      }
+    } catch {
+      /* non-fatal: index refresh never blocks a compaction */
+    }
+  }
+
   runtime.setStatus(
     ctx,
     runtime.rt.persistedThisSession

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.4.25",
+  "version": "0.4.26",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",

package/src/store/vectorIndex.ts

@@ -18,7 +18,7 @@
 
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { mkdirSync } from "node:fs";
+import { mkdirSync, rmSync, existsSync } from "node:fs";
 
 // PGlite + pgvector are script-free WASM (no native build) → survive pi's
 // install-script block. Imported lazily so a missing/broken package degrades
@@ -82,37 +82,65 @@ export function initVectorIndex(): Promise<PGliteInstance | undefined> {
   if (isVectorIndexDisabled()) return Promise.resolve(undefined);
   if (db) return Promise.resolve(db);
   if (initPromise) return initPromise;
-  initPromise = (async (): Promise<PGliteInstance | undefined> => {
-    try {
-      const dir = indexDir();
-      mkdirSync(dir, { recursive: true });
-      const pg = await new PGlite({
-        dataDir: dir,
-        extensions: { vector },
-      });
-      await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
-      await pg.exec(`
-        CREATE TABLE IF NOT EXISTS vector_index (
-          repo_id       TEXT NOT NULL,
-          session_id    TEXT NOT NULL,
-          checkpoint_id TEXT NOT NULL,
-          embedding     vector(${EMBEDDING_DIM}) NOT NULL,
-          PRIMARY KEY (repo_id, session_id, checkpoint_id)
-        );
-      `);
-      // HNSW index over cosine distance for fast NN. Created idempotently.
-      await pg.exec(
-        "CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);",
+  initPromise = openPgLite(/* retryOnCorrupt */ true);
+  return initPromise;
+}
+
+/**
+ * Open + schema-init PGlite. When `retryOnCorrupt` is true, a WASM-level
+ * abort (typically from a corrupted/torn data dir) triggers a delete + one
+ * retry — the dir is rebuilt from scratch by PGlite's initdb.
+ */
+async function openPgLite(
+  retryOnCorrupt: boolean,
+): Promise<PGliteInstance | undefined> {
+  try {
+    const dir = indexDir();
+    mkdirSync(dir, { recursive: true });
+    const pg = await new PGlite({
+      dataDir: dir,
+      extensions: { vector },
+    });
+    await pg.exec("CREATE EXTENSION IF NOT EXISTS vector;");
+    await pg.exec(`
+      CREATE TABLE IF NOT EXISTS vector_index (
+        repo_id       TEXT NOT NULL,
+        session_id    TEXT NOT NULL,
+        checkpoint_id TEXT NOT NULL,
+        embedding     vector(${EMBEDDING_DIM}) NOT NULL,
+        PRIMARY KEY (repo_id, session_id, checkpoint_id)
       );
-      db = pg;
-      return pg;
-    } catch (err) {
-      disabled = true;
-      logWarn(`init failed: ${err instanceof Error ? err.message : String(err)}`);
-      return undefined;
+    `);
+    // HNSW index over cosine distance for fast NN. Created idempotently.
+    await pg.exec(
+      "CREATE INDEX IF NOT EXISTS vector_index_hnsw ON vector_index USING hnsw (embedding vector_cosine_ops);",
+    );
+    db = pg;
+    return pg;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    // Self-heal: a WASM Aborted() typically means the data dir is corrupted
+    // (torn WAL from concurrent access). Delete it and retry once.
+    if (
+      retryOnCorrupt &&
+      (msg.includes("Aborted") || msg.includes("RuntimeError"))
+    ) {
+      try {
+        const dir = indexDir();
+        if (existsSync(dir)) {
+          rmSync(dir, { recursive: true, force: true });
+        }
+        // Clear singleton state so the retry starts fresh.
+        initPromise = undefined;
+        return openPgLite(/* retryOnCorrupt */ false);
+      } catch {
+        // Self-heal failed — fall through to disable.
+      }
     }
-  })();
-  return initPromise;
+    disabled = true;
+    logWarn(`init failed: ${msg}`);
+    return undefined;
+  }
 }
 
 function toVectorLiteral(v: number[]): string {
@@ -211,6 +239,8 @@ export async function closeVectorIndex(): Promise<void> {
   }
   db = undefined;
   initPromise = undefined;
+  disabled = false;
+  warned = false;
 }
 
 /**

package/src/vectorStore.ts

@@ -40,7 +40,6 @@ import {
   dataInvariantStats,
 } from "./store/sqlite.js";
 import {
-  upsertEmbedding as indexUpsertEmbedding,
   initVectorIndex,
   searchAsync as vectorIndexSearch,
   type VectorIndexHit,
@@ -388,12 +387,6 @@ export class VectorStore {
     // Cumulative store-wide dedup accounting (attempt, not collapsed).
     bumpDedupStats(false, this.stateDir);
     onTier?.({ tier: "new", status: "stored" });
-    // Slice 2: best-effort, fire-and-forget mirror of this new checkpoint into
-    // the async global PGlite/HNSW index. NEVER awaited — must not block or
-    // throw into the synchronous add() path. On failure the index degrades to
-    // the sync scan (handled inside vectorIndex). The node:sqlite store remains
-    // authoritative; the index is rebuildable from it at any time.
-    void indexUpsertEmbedding(this.repoId, sessionId, checkpointId, checkpoint.embedding);
     return { checkpoint, deduped: false };
   }