pi-mega-compact 0.8.21 → 0.8.23

package/src/sprint14.test.ts

@@ -8,7 +8,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, rmSync, readFileSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { VectorStore } from "./vectorStore.js";
+import { VectorStore, vectorSearch } from "./vectorStore.js";
 import { defaultEmbedder } from "./embedder.js";
 import { loadDedupConfig, type DedupConfigShape } from "./config/dedup.js";
 import {
@@ -49,7 +49,7 @@ test("flag matrix: all 16 L0/L1/L2/RAPTOR enable combos are safe", () => {
           s.add({ sessionId: "s", summary: "x", regionText: `region A for combo ${combos} about the cache`, timestamp: 1 });
           const r2 = s.add({ sessionId: "s", summary: "x", regionText: `region B for combo ${combos} about the parser`, timestamp: 2 });
           // search must not throw under any combination
-          const hits = s.search("s", "cache", 3);
+          const hits = vectorSearch(s, "s", "cache", 3);
           assert.ok(Array.isArray(hits));
           // With all tiers off, the second add is always "new" (no collapse).
           if (!l0 && !l1 && !l2) assert.equal(r2.deduped, false);

package/src/store/migrate.test.ts

@@ -16,7 +16,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, rmSync, existsSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { VectorStore } from "../vectorStore.js";
+import { VectorStore, vectorMarkInjected, vectorWasInjected, vectorSearch } from "../vectorStore.js";
 import { writeGzJson } from "../store.js";
 import type { StoredCheckpoint } from "../store.js";
 import { migrateJsonToSqlite, readLegacyCheckpointFile } from "../store/migrate.js";
@@ -123,7 +123,7 @@ test("cross-process recall: fresh VectorStore over same dir recalls prior checkp
 
   // Process B: brand-new VectorStore, same stateDir.
   const b = new VectorStore({ stateDir: dir });
-  const hits = b.search(sid, "cross process recall proof", 5);
+  const hits = vectorSearch(b, sid, "cross process recall proof", 5);
   assert.equal(hits.length, 1, "checkpoint survives cross-process reopen");
   assert.equal(hits[0].checkpoint.checkpointId, "chkpt_001");
   assert.ok(hits[0].score > 0.5, "recall is relevant");
@@ -142,13 +142,13 @@ test("cross-process recall: injected state persists across reopen", () => {
     regionText: "injected state persists across process reopen test",
     timestamp: 1,
   });
-  a.markInjected(sid, added.checkpoint.checkpointId);
-  assert.equal(a.wasInjected(sid, added.checkpoint.checkpointId), true);
+  vectorMarkInjected(a, sid, added.checkpoint.checkpointId);
+  assert.equal(vectorWasInjected(a, sid, added.checkpoint.checkpointId), true);
 
   closeStore(dir);
 
   const b = new VectorStore({ stateDir: dir });
-  assert.equal(b.wasInjected(sid, added.checkpoint.checkpointId), true, "injection remembered");
+  assert.equal(vectorWasInjected(b, sid, added.checkpoint.checkpointId), true, "injection remembered");
 
   closeStore(dir);
 });

package/src/store/sprint10.test.ts

@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { VectorStore } from "../vectorStore.js";
+import { VectorStore, vectorList, vectorRepoStats } from "../vectorStore.js";
 import { normalize } from "../dedup/normalize.js";
 import { openBloom, closeBloom } from "./bloom.js";
 import { backfillContentHashes, isBackfillComplete } from "./backfill.js";
@@ -29,7 +29,7 @@ test("Sprint 10 L0: case/whitespace/ANSI variants dedup to one row", () => {
     const r = s.add({ sessionId: "sess_norm", summary: "x", regionText: v, timestamp: added + 1 });
     if (!r.deduped) added++;
   }
-  assert.equal(s.list("sess_norm").length, 1);
+  assert.equal(vectorList(s, "sess_norm").length, 1);
 });
 
 test("normalize case-folds so Foo/foo/FOO are equal", () => {
@@ -74,7 +74,7 @@ test("duplicate add is idempotent (no partial rows, single checkpoint)", () => {
   const b = s.add({ sessionId: "sess_atom", summary: "x", regionText: raw, timestamp: 2 });
   assert.equal(a.deduped, false);
   assert.equal(b.deduped, true);
-  assert.equal(s.list("sess_atom").length, 1);
+  assert.equal(vectorList(s, "sess_atom").length, 1);
 });
 
 // --- Backfill orchestrator -------------------------------------------------
@@ -196,14 +196,14 @@ test("Sprint 10 migration: pre-0.4.2 db gains original_token_estimate and repoSt
     originalTokenEstimate: 500,
     timestamp: 1,
   });
-  const repo = vs.repoStats();
+  const repo = vectorRepoStats(vs);
   // No "no such column" crash = migration succeeded; totals reflect the new col.
   assert.equal(repo.checkpointCount, 1);
   assert.equal(repo.originalTokens, 500, "originalTokens read from migrated column");
   // Re-open a second time to prove idempotency (column already exists).
   closeStore(dir);
   const vs2 = new VectorStore({ dedupSim: 0.9, stateDir: dir });
-  assert.equal(vs2.repoStats().originalTokens, 500, "stable across re-open");
+  assert.equal(vectorRepoStats(vs2).originalTokens, 500, "stable across re-open");
 });
 
 // --- cleanup ---------------------------------------------------------------

package/src/store/sqlite/global-index.ts

@@ -12,10 +12,27 @@
  * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
  */
 import { DatabaseSync } from "node:sqlite";
-import { appendFileSync, existsSync, mkdirSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
+import { existsSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 
+// Back-compat re-exports: session heartbeat + timeseries helpers moved to
+// global-sessions.ts (file-size split). Existing imports from this module
+// keep resolving; behavior is unchanged.
+export {
+  recordSessionHeartbeat,
+  appendTokenSample,
+  pruneStaleSessions,
+  pruneTokenSamples,
+  readActiveSessions,
+  readSessionTimeseries,
+  clearSessionHeartbeat,
+  type ActiveSessionRow,
+  type TokenSamplePoint,
+  type SessionSeries,
+  type SessionTimeseriesResult,
+} from "./global-sessions.js";
+
 /** Resolve the machine-wide index directory (env-overridable). */
 export function getIndexDir(): string {
   const override = process.env.MEGACOMPACT_INDEX_DIR;
@@ -329,292 +346,3 @@ export function countInjectedGlobal(indexDir: string = getIndexDir()): number {
   return row?.n ?? 0;
 }
 
-// ---------------------------------------------------------------------------
-// S39: session heartbeats + token time-series (multi-pi stacked memory graph)
-//
-// `session_heartbeats`: one row per (pid, session_id), upserted on every
-// material snapshot() change. This is the reliable liveness signal the
-// dashboard /api/sessions endpoint reads (repo_registry.last_seen only updates
-// on repo-switch / bindRepo, so a long-running single-repo process looks
-// inactive; these heartbeats update on every material change).
-//
-// `token_samples`: append-only rows with (session_id, tokens, percent, ts) for
-// the stacked-memory graph. Garbage-collected by pruneTokenSamples.
-//
-// All queries use @named/$named bind parameters (PREVENT-002); local node:sqlite
-// + WAL (PREVENT-PI-004), multi-process safe.
-// ---------------------------------------------------------------------------
-
-/** A live active session row (joined heartbeat + latest token sample). */
-export interface ActiveSessionRow {
-  pid: number;
-  sessionId: string;
-  repoRoot: string | null;
-  stateDir: string | null;
-  ctxWindow: number;
-  lastSeen: number;
-  tokens: number | null;
-  percent: number | null;
-}
-
-/** A single time-series data point for a session. */
-export interface TokenSamplePoint {
-  ts: number;
-  tokens: number;
-  percent: number;
-}
-
-/** A recharts-ready per-session series with a stable color. */
-export interface SessionSeries {
-  sessionId: string;
-  label: string;
-  color: string;
-  data: TokenSamplePoint[];
-}
-
-/** Timeseries response shape (recharts-ready). */
-export interface SessionTimeseriesResult {
-  series: SessionSeries[];
-  totals: { ts: number; tokens: number }[];
-}
-
-/** Stable color palette for per-session series (hash-based, no randomness). */
-const SESSION_COLORS = [
-  "#60a5fa", // blue-400
-  "#34d399", // emerald-400
-  "#fbbf24", // amber-400
-  "#f87171", // red-400
-  "#a78bfa", // violet-400
-  "#f472b6", // pink-400
-  "#22d3ee", // cyan-400
-  "#a3e635", // lime-400
-];
-
-/** Hash a sessionId to a stable color index. */
-function sessionColor(sessionId: string): string {
-  let h = 0;
-  for (let i = 0; i < sessionId.length; i++) {
-    h = (h * 31 + sessionId.charCodeAt(i)) | 0;
-  }
-  return SESSION_COLORS[Math.abs(h) % SESSION_COLORS.length];
-}
-
-/**
- * Record (upsert) a session heartbeat. Called from snapshot() on every material
- * change. PRIMARY KEY (pid, session_id) means concurrent pi processes each get
- * their own row. Non-fatal on conflict (INSERT ... ON CONFLICT DO UPDATE).
- */
-export function recordSessionHeartbeat(
-  pid: number,
-  sessionId: string,
-  repoRoot: string,
-  stateDir: string,
-  ctxWindow: number,
-  indexDir: string = getIndexDir(),
-): void {
-  const db = openIndexStore(indexDir);
-  const now = Date.now();
-  db.prepare(
-    `INSERT INTO session_heartbeats (pid, session_id, repo_root, state_dir, ctx_window, last_seen)
-     VALUES (@pid, @session_id, @repo_root, @state_dir, @ctx_window, @last_seen)
-     ON CONFLICT(pid, session_id) DO UPDATE SET
-       repo_root = excluded.repo_root,
-       state_dir = excluded.state_dir,
-       ctx_window = excluded.ctx_window,
-       last_seen = excluded.last_seen`,
-  ).run({
-    pid,
-    session_id: sessionId,
-    repo_root: repoRoot,
-    state_dir: stateDir,
-    ctx_window: ctxWindow,
-    last_seen: now,
-  });
-}
-
-/**
- * Append a token sample row + optionally a session_sample line to events.log
- * (for SSE real-time push via /api/events). The eventsLogPath is optional —
- * callers without an events.log (e.g. tests) can omit it.
- */
-export function appendTokenSample(
-  sessionId: string,
-  repoRoot: string,
-  tokens: number,
-  percent: number,
-  ctxWindow: number,
-  eventsLogPath: string | null,
-  indexDir: string = getIndexDir(),
-): void {
-  const db = openIndexStore(indexDir);
-  const now = Date.now();
-  db.prepare(
-    `INSERT INTO token_samples (session_id, repo_root, tokens, percent, ctx_window, ts)
-     VALUES (@session_id, @repo_root, @tokens, @percent, @ctx_window, @ts)`,
-  ).run({
-    session_id: sessionId,
-    repo_root: repoRoot,
-    tokens,
-    percent,
-    ctx_window: ctxWindow,
-    ts: now,
-  });
-  // Step 5: also append a session_sample JSON line to events.log so the
-  // existing /api/events SSE tail streams it for free (real-time chart push).
-  // Mirrors the DashboardEmitter events.log append pattern in
-  // extensions/mega-dashboard.ts:{ event(type, data) }: a JSON object with
-  // `ts` (ISO 8601 string), `type`, and the event-specific payload. The ISO
-  // timestamp matches the shape of every other SSE variant (SseSessionSample
-  // contract; every SSE variant's `ts` is a string); a numeric ms `ts` would
-  // violate the contract union's "every SSE variant has a ts field of type
-  // string" invariant and break DashboardEmitter consumers.
-  if (eventsLogPath) {
-    try {
-      const dir = eventsLogPath.includes("/") ? eventsLogPath.slice(0, eventsLogPath.lastIndexOf("/")) : ".";
-      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-      appendFileSync(
-        eventsLogPath,
-        JSON.stringify({
-          ts: new Date(now).toISOString(),
-          type: "session_sample",
-          sessionId,
-          tokens,
-          percent,
-        }) + "\n",
-      );
-    } catch {
-      /* non-fatal: SSE push is best-effort */
-    }
-  }
-}
-
-/**
- * Prune stale session heartbeats (sessions not seen within maxAgeMs).
- * Default 30-min retention. Called by /api/sessions.
- */
-export function pruneStaleSessions(
-  maxAgeMs: number = 1_800_000,
-  indexDir: string = getIndexDir(),
-): number {
-  const db = openIndexStore(indexDir);
-  const cutoff = Date.now() - maxAgeMs;
-  const result = db.prepare(
-    "DELETE FROM session_heartbeats WHERE last_seen < @cutoff",
-  ).run({ cutoff });
-  return Number(result.changes);
-}
-
-/**
- * Prune old token samples older than maxAgeMs. Default 30-min retention.
- * Called by /api/sessions/timeseries.
- */
-export function pruneTokenSamples(
-  maxAgeMs: number = 1_800_000,
-  indexDir: string = getIndexDir(),
-): number {
-  const db = openIndexStore(indexDir);
-  const cutoff = Date.now() - maxAgeMs;
-  const result = db.prepare(
-    "DELETE FROM token_samples WHERE ts < @cutoff",
-  ).run({ cutoff });
-  return Number(result.changes);
-}
-
-/**
- * Read all active sessions with their latest token sample (if any).
- * JOINs session_heartbeats with the latest token_samples per session_id
- * via a correlated subquery. Returns rows sorted by last_seen descending.
- */
-export function readActiveSessions(
-  indexDir: string = getIndexDir(),
-): ActiveSessionRow[] {
-  const db = openIndexStore(indexDir);
-  const rows = db.prepare(
-    `SELECT h.pid, h.session_id, h.repo_root, h.state_dir, h.ctx_window, h.last_seen,
-            s.tokens, s.percent
-     FROM session_heartbeats h
-     LEFT JOIN token_samples s ON s.id = (
-       SELECT id FROM token_samples t
-       WHERE t.session_id = h.session_id
-       ORDER BY t.ts DESC LIMIT 1
-     )
-     ORDER BY h.last_seen DESC`,
-  ).all() as Array<{
-    pid: number;
-    session_id: string;
-    repo_root: string | null;
-    state_dir: string | null;
-    ctx_window: number;
-    last_seen: number;
-    tokens: number | null;
-    percent: number | null;
-  }>;
-  return rows.map((r) => ({
-    pid: r.pid,
-    sessionId: r.session_id,
-    repoRoot: r.repo_root,
-    stateDir: r.state_dir,
-    ctxWindow: r.ctx_window ?? 0,
-    lastSeen: r.last_seen,
-    tokens: r.tokens,
-    percent: r.percent,
-  }));
-}
-
-/**
- * Read token samples since sinceMs, returning a recharts-ready stacked shape:
- * per-session `SessionSeries` (with stable color) + a `totals` array
- * [{ts, tokens}] (sum of all sessions at each timestamp).
- */
-export function readSessionTimeseries(
-  sinceMs: number,
-  indexDir: string = getIndexDir(),
-): SessionTimeseriesResult {
-  const db = openIndexStore(indexDir);
-  const rows = db.prepare(
-    `SELECT session_id, tokens, percent, ts FROM token_samples WHERE ts >= @since ORDER BY ts ASC`,
-  ).all({ since: sinceMs }) as Array<{
-    session_id: string;
-    tokens: number;
-    percent: number;
-    ts: number;
-  }>;
-  // Group by session_id → series; + accumulate totals per timestamp.
-  const seriesMap = new Map<string, TokenSamplePoint[]>();
-  const totalsMap = new Map<number, number>();
-  for (const r of rows) {
-    let pts = seriesMap.get(r.session_id);
-    if (!pts) {
-      pts = [];
-      seriesMap.set(r.session_id, pts);
-    }
-    pts.push({ ts: r.ts, tokens: r.tokens, percent: r.percent });
-    totalsMap.set(r.ts, (totalsMap.get(r.ts) ?? 0) + r.tokens);
-  }
-  const series: SessionSeries[] = [];
-  for (const [sessionId, data] of seriesMap) {
-    const label = sessionId.length > 12 ? sessionId.slice(0, 12) : sessionId;
-    series.push({ sessionId, label, color: sessionColor(sessionId), data });
-  }
-  // Sort series by first-timestamp for stable legend order.
-  series.sort((a, b) => (a.data[0]?.ts ?? 0) - (b.data[0]?.ts ?? 0));
-  const totals = Array.from(totalsMap.entries())
-    .sort((a, b) => a[0] - b[0])
-    .map(([ts, tokens]) => ({ ts, tokens }));
-  return { series, totals };
-}
-
-/**
- * Clear a session's heartbeat row (e.g. on clean shutdown / session reset).
- * Non-fatal: no-op if the row doesn't exist.
- */
-export function clearSessionHeartbeat(
-  pid: number,
-  sessionId: string,
-  indexDir: string = getIndexDir(),
-): void {
-  const db = openIndexStore(indexDir);
-  db.prepare(
-    "DELETE FROM session_heartbeats WHERE pid = @pid AND session_id = @session_id",
-  ).run({ pid, session_id: sessionId });
-}