pi-mega-compact 0.4.23 → 0.4.25

package/extensions/dashboard-server.test.ts

@@ -10,6 +10,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { spawn } from "node:child_process";
 
 // ---------------------------------------------------------------------------
 // helpers
@@ -122,3 +123,79 @@ describe("port.pid file", () => {
     rmSync(dir, { recursive: true });
   });
 });
+
+// ---------------------------------------------------------------------------
+// Lifecycle integration — launch the compiled server as a real subprocess
+// (the same way the /dashboard command spawns it) and assert the two failure
+// modes that historically produced a silent "failed to start":
+//   1. a stale port.pid pointing at a dead port is dropped, and the server
+//      binds fresh (instead of returning the dead port);
+//   2. a module-load crash is captured to the launch log instead of going
+//      silent under stdio:"ignore".
+// ---------------------------------------------------------------------------
+
+const SERVER_ENTRY = new URL("./dashboard-server.js", import.meta.url).pathname;
+
+function waitFor(cond: () => boolean | Promise<boolean>, timeoutMs = 6000): Promise<void> {
+  const start = Date.now();
+  return new Promise((resolve, reject) => {
+    const tick = async () => {
+      if (await cond()) return resolve();
+      if (Date.now() - start > timeoutMs) return reject(new Error("timeout"));
+      setTimeout(tick, 50);
+    };
+    tick();
+  });
+}
+
+describe("server lifecycle", () => {
+  test("drops a stale port.pid and binds a fresh port", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "dash-stale-"));
+    // A marker claiming a port where nothing is listening.
+    writeFileSync(join(dir, "port.pid"), JSON.stringify({ port: 9325, pid: 999999 }));
+
+    const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
+    try {
+      // Wait for the server to actually be live (not just any port.pid — the
+      // stale marker already exists at t=0 and would pass a naive check).
+      await waitFor(async () => {
+        try {
+          const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+          const res = await fetch(`http://localhost:${raw.port}/api/version`);
+          return res.ok;
+        } catch {
+          return false;
+        }
+      });
+      const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+      assert.equal(typeof raw.port, "number");
+      assert.notEqual(raw.port, 9325, "should not reuse the dead port from the stale marker");
+      // And a real server must answer on it.
+      const res = await fetch(`http://localhost:${raw.port}/api/version`);
+      assert.equal(res.ok, true);
+    } finally {
+      child.kill("SIGTERM");
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  test("writes a dashboard.log with startup lines", async () => {
+    const dir = mkdtempSync(join(tmpdir(), "dash-log-"));
+    const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
+    try {
+      await waitFor(() => {
+        try {
+          return /server running/.test(readFileSync(join(dir, "dashboard.log"), "utf-8"));
+        } catch {
+          return false;
+        }
+      });
+      const log = readFileSync(join(dir, "dashboard.log"), "utf-8");
+      assert.match(log, /\[mega-compact\]\[dashboard\]/);
+      assert.match(log, /server running/);
+    } finally {
+      child.kill("SIGTERM");
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

package/extensions/dashboard-server.ts

@@ -12,12 +12,32 @@
  */
 
 import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
-import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, unlinkSync, watch, writeFileSync, appendFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { DatabaseSync } from "node:sqlite";
 
+// ---------------------------------------------------------------------------
+// Local runtime log
+//
+// The dashboard server is spawned as a DETACHED child. When it is launched with
+// `stdio: "ignore"` (the old default) any crash before the first console.log is
+// invisible — there is no log to "check". We therefore mirror every lifecycle
+// line to a file in the state dir so a failed start is always diagnosable. The
+// launcher also captures stderr, so this doubles as defense-in-depth.
+// ---------------------------------------------------------------------------
+
+let LOG_PATH: string | null = null;
+function log(...parts: unknown[]): void {
+  const line = `[mega-compact][dashboard] ${parts.map((p) => (typeof p === "string" ? p : JSON.stringify(p))).join(" ")}`;
+  // eslint-disable-next-line no-console
+  console.error(line); // stderr — captured by the launcher pipe
+  if (LOG_PATH) {
+    try { appendFileSync(LOG_PATH, new Date().toISOString() + " " + line + "\n"); } catch { /* non-fatal */ }
+  }
+}
+
 // --- Multi-repo index (Phase 5b) ------------------------------------------------
 // The extension writes a machine-wide repo registry into a single SQLite DB
 // (<indexDir>/index.sqlite) as the concurrency-safe write path; the dashboard
@@ -738,7 +758,7 @@ function dashboardHtml(tierName: string): string {
 // Server
 // ---------------------------------------------------------------------------
 
-export function launchDashboardServer(stateDir: string): Promise<{ port: number; url: string }> {
+export async function launchDashboardServer(stateDir: string): Promise<{ port: number; url: string }> {
   // Our own package version — exposed at /api/version so the launcher can
   // detect a stale server (started by an older build) and replace it on
   // upgrade instead of reuse it.
@@ -757,17 +777,37 @@ export function launchDashboardServer(stateDir: string): Promise<{ port: number;
   const portFile = join(stateDir, "port.pid");
   const snapshotPath = join(stateDir, "dashboard.json");
   const eventsPath = join(stateDir, "events.log");
-
-  // ── Existing server? ──────────────────────────────────────────────────────
+  LOG_PATH = join(stateDir, "dashboard.log");
+  log("launch invoked", { stateDir });
+
+  // ── Existing server? ───────────────────────────────────────────────────────
+  // A stale port.pid pointing at a dead/competing process is the classic cause
+  // of "dashboard failed to start" — we return a port that is NOT actually
+  // serving. Probe for a live server on that port first; only reuse the marker
+  // when something real answers /api/version. Otherwise drop it and start fresh.
   if (existsSync(portFile)) {
     try {
       const info = JSON.parse(readFileSync(portFile, "utf-8"));
       if (info && info.port) {
-        return Promise.resolve({ port: info.port, url: `http://localhost:${info.port}` });
+        let live = false;
+        try {
+          const probe = await fetch(`http://localhost:${info.port}/api/version`, { signal: AbortSignal.timeout(800) });
+          live = probe.ok;
+        } catch {
+          live = false;
+        }
+        if (live) {
+          log("reusing live server from port.pid", { port: info.port });
+          return { port: info.port, url: `http://localhost:${info.port}` };
+        }
+        log("port.pid present but no live server — treating as stale", { port: info.port });
       }
     } catch {
-      // stale file, overwrite
+      log("port.pid unparseable — treating as stale");
     }
+    // stale file, remove so the fresh bind does not collide with a lingering
+    // process that still holds the port
+    try { unlinkSync(portFile); } catch { /* ignore */ }
   }
 
   // ── New server ────────────────────────────────────────────────────────────
@@ -891,20 +931,26 @@ export function launchDashboardServer(stateDir: string): Promise<{ port: number;
     function tryPort(port: number) {
       server.once("error", (err: NodeJS.ErrnoException) => {
         if (err.code === "EADDRINUSE" && port < TARGET_PORT + PORT_RANGE - 1) {
+          log("port in use, trying next", { port });
           tryPort(port + 1);
         } else {
+          log("listen failed", { port, code: err.code, message: err.message });
           reject(err);
         }
       });
 
       server.listen(port, "127.0.0.1", () => {
         const url = `http://localhost:${port}`;
+        log("server running", { url });
+        // eslint-disable-next-line no-console
         console.log(`[mega-compact] dashboard server running: ${url}`);
 
         // Write port.pid
         try {
           writeFileSync(portFile, JSON.stringify({ port, pid: process.pid }));
-        } catch { /* non-fatal */ }
+        } catch (e) {
+          log("could not write port.pid", { error: String(e) });
+        }
 
         // Graceful cleanup
         const cleanup = () => {

package/extensions/mega-dashboard-cmds.ts

@@ -9,7 +9,7 @@
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { join, dirname, sep } from "node:path";
 import { fileURLToPath } from "node:url";
-import { existsSync, writeFileSync, readFileSync, unlinkSync } from "node:fs";
+import { existsSync, writeFileSync, readFileSync, unlinkSync, openSync, closeSync } from "node:fs";
 import { spawn } from "node:child_process"; // guardrails-allow PREVENT-PI-004: spawns the optional, user-triggered localhost dashboard server only
 import { MegaRuntime } from "./mega-runtime.js";
 
@@ -209,11 +209,32 @@ export function registerDashboardCommands(pi: ExtensionAPI, runtime: MegaRuntime
         return;
       }
 
+      // Clear any stale marker so a fresh bind never collides with a lingering
+      // orphan, and truncate the launch log so the next error report shows only
+      // this attempt's output.
+      try { unlinkSync(portFile); } catch { /* ignore */ }
+      try { writeFileSync(launchLog, ""); } catch { /* ignore */ }
+
       const args = dashboardNeedsStrip ? ["--experimental-strip-types", runnerFile] : [runnerFile];
+      // Redirect the child's stderr to the launch log so that a CRASH BEFORE the
+      // runner's own __fail handler runs (e.g. an ESM module-load / parse error,
+      // or a missing entry) is still captured. With the old `stdio: "ignore"`
+      // these failures were completely silent and the "check logs" message
+      // pointed at an empty file. We open the fd in the parent and pass it to the
+      // child; once spawned we close our copy (the child keeps its own dup).
+      let stderrFd: number;
+      try {
+        stderrFd = openSync(launchLog, "a");
+      } catch {
+        stderrFd = -1; // fall back to ignored stderr
+      }
       const child = spawn(process.execPath, args, {
         detached: true,
-        stdio: "ignore",
+        stdio: ["ignore", "ignore", stderrFd >= 0 ? stderrFd : "ignore"],
       });
+      if (stderrFd >= 0) {
+        try { closeSync(stderrFd); } catch { /* ignore */ }
+      }
       child.unref();
 
       // Poll for a live server (port 9320–9329) instead of relying solely on the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.4.23",
+  "version": "0.4.25",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",
@@ -57,6 +57,8 @@
     "typescript": "^5.4.0"
   },
   "dependencies": {
+    "@electric-sql/pglite": "^0.5.4",
+    "@electric-sql/pglite-pgvector": "^0.0.5",
     "@mongodb-js/zstd": "^7.0.0"
   }
 }

package/src/recall.ts

@@ -137,3 +137,66 @@ export function recallAndInline(
     empty: toInject.length === 0,
   };
 }
+
+/**
+ * Slice 2 async cross-repo recall. Same dedup/bound/inline contract as
+ * `recallAndInline`, but backed by `VectorStore.searchAsync` so it can recall
+ * across repos (HNSW NN over the global PGlite index) when `opts.crossRepo` is
+ * set. The synchronous `recallAndInline` is unchanged and remains the default
+ * per-session path. Inline-window dedupe + token cap (Fix C) apply here too.
+ *
+ * `store` must provide `searchAsync` (the live VectorStore does). Errors fall
+ * back to an empty result — recall is a bonus, never a hard dependency.
+ */
+export async function recallAndInlineAsync(
+  opts: RecallInjectOptions & { crossRepo?: boolean; repoId?: string },
+  store: Pick<VectorStore, "searchAsync" | "wasInjected" | "markInjected">,
+): Promise<RecallInjectResult> {
+  const limit = opts.limit ?? 3;
+  const skip = opts.skipInjected ?? true;
+  const maxTokens = opts.recallMaxTokens ?? 0;
+  const doWindowDedupe = opts.windowDedupe ?? false;
+  const dedupSim = opts.dedupSim ?? 0.9;
+
+  let hits: SearchHit[] = [];
+  try {
+    hits = await store.searchAsync(opts.sessionId, opts.query, limit, {
+      crossRepo: opts.crossRepo,
+      repoId: opts.repoId,
+    });
+  } catch {
+    hits = [];
+  }
+
+  let liveEmbeddings: number[][] = [];
+  if (doWindowDedupe && opts.liveWindow && opts.liveWindow.length > 0) {
+    const embedder = defaultEmbedder();
+    liveEmbeddings = opts.liveWindow.map((m) => embedder.embed(m));
+  }
+
+  const toInject: SearchHit[] = [];
+  const parts: string[] = [];
+  let blockTokens = 0;
+
+  for (const h of hits) {
+    if (skip && store.wasInjected(opts.sessionId, h.checkpoint.checkpointId)) continue;
+    if (doWindowDedupe && liveEmbeddings.length > 0) {
+      const hitVec = defaultEmbedder().embed(h.checkpoint.summary);
+      if (liveEmbeddings.some((v) => cosineSimilarity(v, hitVec) >= dedupSim)) continue;
+    }
+    const part = formatRecallBlock([h]);
+    const partTokens = estimateBlockTokens(part);
+    if (maxTokens > 0 && blockTokens + partTokens > maxTokens) break;
+    parts.push(part);
+    toInject.push(h);
+    blockTokens += partTokens;
+    store.markInjected(opts.sessionId, h.checkpoint.checkpointId);
+  }
+
+  const block = parts.join("\n");
+  const report = toInject.map(
+    (h) => `  • ${h.checkpoint.checkpointId} (${h.checkpoint.summary.slice(0, 60).replace(/\n/g, " ")}…)`,
+  );
+
+  return { toInject, report, block, empty: toInject.length === 0 };
+}

package/src/store/sqlite.ts

@@ -872,6 +872,19 @@ export function hasCheckpoint(sessionId: string, checkpointId: string, stateDir:
   return row !== undefined;
 }
 
+/** Fetch a single checkpoint by (session, id), or undefined if absent. */
+export function getCheckpoint(
+  sessionId: string,
+  checkpointId: string,
+  stateDir: string = getStateDir(),
+): StoredCheckpoint | undefined {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+    .get(normalizeSessionId(sessionId), checkpointId) as any;
+  return row ? rowToCheckpoint(row) : undefined;
+}
+
 /** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
 export function setDedupStatus(
   checkpointId: string,

package/src/store/vectorIndex.test.ts

@@ -0,0 +1,116 @@
+/**
+ * vectorIndex.test.ts — Slice 2 async PGlite/HNSW vector index.
+ *
+ * Proves: cross-repo nearest-neighbor recall, repoId scoping, the dimension
+ * guard (non-512 vectors skipped, never corrupt the index), and graceful
+ * degradation when the index is disabled (kill-switch) — all without touching
+ * the synchronous node:sqlite store.
+ *
+ * The index is a WASM Postgres (PGlite) — fully local, zero network
+ * (PREVENT-PI-004). Each test isolates state via MEGACOMPACT_VECTOR_INDEX_DIR.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  EMBEDDING_DIM,
+  initVectorIndex,
+  upsertEmbedding,
+  searchAsync,
+  closeVectorIndex,
+  isVectorIndexDisabled,
+} from "./vectorIndex.js";
+
+/** A 512-dim unit-ish vector with a single spike at `idx` (deterministic NN). */
+function spikeVec(idx: number, magnitude = 1): number[] {
+  const v = new Array<number>(EMBEDDING_DIM).fill(0);
+  v[idx % EMBEDDING_DIM] = magnitude;
+  return v;
+}
+
+function isolateIndexDir(): string {
+  const dir = mkdtempSync(join(tmpdir(), "mc-vecidx-"));
+  process.env.MEGACOMPACT_VECTOR_INDEX_DIR = dir;
+  return dir;
+}
+
+test("cross-repo HNSW nearest-neighbor recall across repos + repoId scoping", async () => {
+  delete process.env.MEGACOMPACT_PGLITE_DISABLED;
+  const dir = isolateIndexDir();
+  try {
+    await closeVectorIndex(); // ensure a fresh singleton for this dir
+    const pg = await initVectorIndex();
+    assert.ok(pg, "index should initialize (PGlite WASM available)");
+
+    // repoA: two checkpoints; repoB: one checkpoint. Distinct spike directions.
+    await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_001", spikeVec(0));
+    await upsertEmbedding("/repoA/.pi/mega-compact", "sessA", "chkpt_002", spikeVec(5));
+    await upsertEmbedding("/repoB/.pi/mega-compact", "sessB", "chkpt_001", spikeVec(0));
+
+    // Cross-repo query near spike(0): nearest are the two spike(0) rows, one per repo.
+    const cross = await searchAsync(spikeVec(0), { k: 2 });
+    assert.equal(cross.length, 2, "cross-repo returns two nearest");
+    const repos = new Set(cross.map((h) => h.repoId));
+    assert.ok(repos.has("/repoA/.pi/mega-compact"), "hit from repoA");
+    assert.ok(repos.has("/repoB/.pi/mega-compact"), "hit from repoB");
+    assert.ok(cross[0].score > 0.99, "top hit is near-identical (cosine ~1)");
+
+    // Scoped to repoA only: excludes repoB even though repoB has an identical vec.
+    const scoped = await searchAsync(spikeVec(0), { k: 5, repoId: "/repoA/.pi/mega-compact" });
+    assert.ok(scoped.length >= 1, "scoped returns repoA hits");
+    assert.ok(
+      scoped.every((h) => h.repoId === "/repoA/.pi/mega-compact"),
+      "repoId filter excludes other repos",
+    );
+  } finally {
+    await closeVectorIndex();
+    rmSync(dir, { recursive: true, force: true });
+    delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
+  }
+});
+
+test("dimension guard: non-512 vectors are skipped, never corrupt the index", async () => {
+  delete process.env.MEGACOMPACT_PGLITE_DISABLED;
+  const dir = isolateIndexDir();
+  try {
+    await closeVectorIndex();
+    await initVectorIndex();
+    // Wrong-dimension vector (BYO embedder mismatch) must be silently skipped.
+    await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_001", [1, 2, 3]);
+    const hits = await searchAsync(spikeVec(0), { k: 5 });
+    assert.equal(hits.length, 0, "no rows stored for a mismatched-dim vector");
+
+    // A correct-dim vector still stores fine afterward (index not corrupted).
+    await upsertEmbedding("/repoC/.pi/mega-compact", "sessC", "chkpt_002", spikeVec(3));
+    const ok = await searchAsync(spikeVec(3), { k: 1 });
+    assert.equal(ok.length, 1, "valid vector stored after a skipped one");
+    assert.equal(ok[0].checkpointId, "chkpt_002");
+  } finally {
+    await closeVectorIndex();
+    rmSync(dir, { recursive: true, force: true });
+    delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
+  }
+});
+
+test("kill-switch: MEGACOMPACT_PGLITE_DISABLED disables the index gracefully", async () => {
+  const dir = isolateIndexDir();
+  process.env.MEGACOMPACT_PGLITE_DISABLED = "true";
+  try {
+    await closeVectorIndex();
+    assert.equal(isVectorIndexDisabled(), true, "kill-switch reported disabled");
+    const pg = await initVectorIndex();
+    assert.equal(pg, undefined, "init returns undefined when disabled");
+    // Upsert + search are no-ops that never throw and return empty.
+    await upsertEmbedding("/repoD/.pi/mega-compact", "sessD", "chkpt_001", spikeVec(0));
+    const hits = await searchAsync(spikeVec(0), { k: 3 });
+    assert.deepEqual(hits, [], "search returns [] when disabled");
+  } finally {
+    delete process.env.MEGACOMPACT_PGLITE_DISABLED;
+    await closeVectorIndex();
+    rmSync(dir, { recursive: true, force: true });
+    delete process.env.MEGACOMPACT_VECTOR_INDEX_DIR;
+  }
+});