pi-mega-compact 0.8.21 → 0.8.23

package/src/store/sqlite/global-sessions.ts

@@ -0,0 +1,291 @@
+/**
+ * global-sessions.ts — session heartbeats + token time-series (S39).
+ *
+ * Split out of global-index.ts so each file stays under 500 lines. Shares the
+ * same machine-wide index DB (`indexCache` in global-index.ts) via
+ * `openIndexStore()`; no schema or behavior change — pure structural move.
+ *
+ * `session_heartbeats`: one row per (pid, session_id) live session, upserted on
+ * every material snapshot. `token_samples`: append-only rows with
+ * (session_id, tokens, percent, ts) for the stacked-memory graph. Garbage-
+ * collected by pruneTokenSamples.
+ *
+ * All queries use @named/$named bind parameters (PREVENT-002); local
+ * node:sqlite + WAL (PREVENT-PI-004), multi-process safe.
+ */
+import { appendFileSync, existsSync, mkdirSync } from "node:fs";
+import { openIndexStore, getIndexDir } from "./global-index.js";
+
+/** A live active session row (joined heartbeat + latest token sample). */
+export interface ActiveSessionRow {
+  pid: number;
+  sessionId: string;
+  repoRoot: string | null;
+  stateDir: string | null;
+  ctxWindow: number;
+  lastSeen: number;
+  tokens: number | null;
+  percent: number | null;
+}
+
+/** A single time-series data point for a session. */
+export interface TokenSamplePoint {
+  ts: number;
+  tokens: number;
+  percent: number;
+}
+
+/** A recharts-ready per-session series with a stable color. */
+export interface SessionSeries {
+  sessionId: string;
+  label: string;
+  color: string;
+  data: TokenSamplePoint[];
+}
+
+/** Timeseries response shape (recharts-ready). */
+export interface SessionTimeseriesResult {
+  series: SessionSeries[];
+  totals: { ts: number; tokens: number }[];
+}
+
+/** Stable color palette for per-session series (hash-based, no randomness). */
+const SESSION_COLORS = [
+  "#60a5fa", // blue-400
+  "#34d399", // emerald-400
+  "#fbbf24", // amber-400
+  "#f87171", // red-400
+  "#a78bfa", // violet-400
+  "#f472b6", // pink-400
+  "#22d3ee", // cyan-400
+  "#a3e635", // lime-400
+];
+
+/** Hash a sessionId to a stable color index. */
+function sessionColor(sessionId: string): string {
+  let h = 0;
+  for (let i = 0; i < sessionId.length; i++) {
+    h = (h * 31 + sessionId.charCodeAt(i)) | 0;
+  }
+  return SESSION_COLORS[Math.abs(h) % SESSION_COLORS.length];
+}
+
+/**
+ * Record (upsert) a session heartbeat. Called from snapshot() on every material
+ * change. PRIMARY KEY (pid, session_id) means concurrent pi processes each get
+ * their own row. Non-fatal on conflict (INSERT ... ON CONFLICT DO UPDATE).
+ */
+export function recordSessionHeartbeat(
+  pid: number,
+  sessionId: string,
+  repoRoot: string,
+  stateDir: string,
+  ctxWindow: number,
+  indexDir: string = getIndexDir(),
+): void {
+  const db = openIndexStore(indexDir);
+  const now = Date.now();
+  db.prepare(
+    `INSERT INTO session_heartbeats (pid, session_id, repo_root, state_dir, ctx_window, last_seen)
+     VALUES (@pid, @session_id, @repo_root, @state_dir, @ctx_window, @last_seen)
+     ON CONFLICT(pid, session_id) DO UPDATE SET
+       repo_root = excluded.repo_root,
+       state_dir = excluded.state_dir,
+       ctx_window = excluded.ctx_window,
+       last_seen = excluded.last_seen`,
+  ).run({
+    pid,
+    session_id: sessionId,
+    repo_root: repoRoot,
+    state_dir: stateDir,
+    ctx_window: ctxWindow,
+    last_seen: now,
+  });
+}
+
+/**
+ * Append a token sample row + optionally a session_sample line to events.log
+ * (for SSE real-time push via /api/events). The eventsLogPath is optional —
+ * callers without an events.log (e.g. tests) can omit it.
+ */
+export function appendTokenSample(
+  sessionId: string,
+  repoRoot: string,
+  tokens: number,
+  percent: number,
+  ctxWindow: number,
+  eventsLogPath: string | null,
+  indexDir: string = getIndexDir(),
+): void {
+  const db = openIndexStore(indexDir);
+  const now = Date.now();
+  db.prepare(
+    `INSERT INTO token_samples (session_id, repo_root, tokens, percent, ctx_window, ts)
+     VALUES (@session_id, @repo_root, @tokens, @percent, @ctx_window, @ts)`,
+  ).run({
+    session_id: sessionId,
+    repo_root: repoRoot,
+    tokens,
+    percent,
+    ctx_window: ctxWindow,
+    ts: now,
+  });
+  // Step 5: also append a session_sample JSON line to events.log so the
+  // existing /api/events SSE tail streams it for free (real-time chart push).
+  // Mirrors the DashboardEmitter events.log append pattern in
+  // extensions/mega-dashboard.ts:{ event(type, data) }: a JSON object with
+  // `ts` (ISO 8601 string), `type`, and the event-specific payload. The ISO
+  // timestamp matches the shape of every other SSE variant (SseSessionSample
+  // contract; every SSE variant's `ts` is a string); a numeric ms `ts` would
+  // violate the contract union's "every SSE variant has a ts field of type
+  // string" invariant and break DashboardEmitter consumers.
+  if (eventsLogPath) {
+    try {
+      const dir = eventsLogPath.includes("/") ? eventsLogPath.slice(0, eventsLogPath.lastIndexOf("/")) : ".";
+      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+      appendFileSync(
+        eventsLogPath,
+        JSON.stringify({
+          ts: new Date(now).toISOString(),
+          type: "session_sample",
+          sessionId,
+          tokens,
+          percent,
+        }) + "\n",
+      );
+    } catch {
+      /* non-fatal: SSE push is best-effort */
+    }
+  }
+}
+
+/**
+ * Prune stale session heartbeats (sessions not seen within maxAgeMs).
+ * Default 30-min retention. Called by /api/sessions.
+ */
+export function pruneStaleSessions(
+  maxAgeMs: number = 1_800_000,
+  indexDir: string = getIndexDir(),
+): number {
+  const db = openIndexStore(indexDir);
+  const cutoff = Date.now() - maxAgeMs;
+  const result = db.prepare(
+    "DELETE FROM session_heartbeats WHERE last_seen < @cutoff",
+  ).run({ cutoff });
+  return Number(result.changes);
+}
+
+/**
+ * Prune old token samples older than maxAgeMs. Default 30-min retention.
+ * Called by /api/sessions/timeseries.
+ */
+export function pruneTokenSamples(
+  maxAgeMs: number = 1_800_000,
+  indexDir: string = getIndexDir(),
+): number {
+  const db = openIndexStore(indexDir);
+  const cutoff = Date.now() - maxAgeMs;
+  const result = db.prepare(
+    "DELETE FROM token_samples WHERE ts < @cutoff",
+  ).run({ cutoff });
+  return Number(result.changes);
+}
+
+/**
+ * Read all active sessions with their latest token sample (if any).
+ * JOINs session_heartbeats with the latest token_samples per session_id
+ * via a correlated subquery. Returns rows sorted by last_seen descending.
+ */
+export function readActiveSessions(
+  indexDir: string = getIndexDir(),
+): ActiveSessionRow[] {
+  const db = openIndexStore(indexDir);
+  const rows = db.prepare(
+    `SELECT h.pid, h.session_id, h.repo_root, h.state_dir, h.ctx_window, h.last_seen,
+            s.tokens, s.percent
+     FROM session_heartbeats h
+     LEFT JOIN token_samples s ON s.id = (
+       SELECT id FROM token_samples t
+       WHERE t.session_id = h.session_id
+       ORDER BY t.ts DESC LIMIT 1
+     )
+     ORDER BY h.last_seen DESC`,
+  ).all() as Array<{
+    pid: number;
+    session_id: string;
+    repo_root: string | null;
+    state_dir: string | null;
+    ctx_window: number;
+    last_seen: number;
+    tokens: number | null;
+    percent: number | null;
+  }>;
+  return rows.map((r) => ({
+    pid: r.pid,
+    sessionId: r.session_id,
+    repoRoot: r.repo_root,
+    stateDir: r.state_dir,
+    ctxWindow: r.ctx_window ?? 0,
+    lastSeen: r.last_seen,
+    tokens: r.tokens,
+    percent: r.percent,
+  }));
+}
+
+/**
+ * Read token samples since sinceMs, returning a recharts-ready stacked shape:
+ * per-session `SessionSeries` (with stable color) + a `totals` array
+ * [{ts, tokens}] (sum of all sessions at each timestamp).
+ */
+export function readSessionTimeseries(
+  sinceMs: number,
+  indexDir: string = getIndexDir(),
+): SessionTimeseriesResult {
+  const db = openIndexStore(indexDir);
+  const rows = db.prepare(
+    `SELECT session_id, tokens, percent, ts FROM token_samples WHERE ts >= @since ORDER BY ts ASC`,
+  ).all({ since: sinceMs }) as Array<{
+    session_id: string;
+    tokens: number;
+    percent: number;
+    ts: number;
+  }>;
+  // Group by session_id → series; + accumulate totals per timestamp.
+  const seriesMap = new Map<string, TokenSamplePoint[]>();
+  const totalsMap = new Map<number, number>();
+  for (const r of rows) {
+    let pts = seriesMap.get(r.session_id);
+    if (!pts) {
+      pts = [];
+      seriesMap.set(r.session_id, pts);
+    }
+    pts.push({ ts: r.ts, tokens: r.tokens, percent: r.percent });
+    totalsMap.set(r.ts, (totalsMap.get(r.ts) ?? 0) + r.tokens);
+  }
+  const series: SessionSeries[] = [];
+  for (const [sessionId, data] of seriesMap) {
+    const label = sessionId.length > 12 ? sessionId.slice(0, 12) : sessionId;
+    series.push({ sessionId, label, color: sessionColor(sessionId), data });
+  }
+  // Sort series by first-timestamp for stable legend order.
+  series.sort((a, b) => (a.data[0]?.ts ?? 0) - (b.data[0]?.ts ?? 0));
+  const totals = Array.from(totalsMap.entries())
+    .sort((a, b) => a[0] - b[0])
+    .map(([ts, tokens]) => ({ ts, tokens }));
+  return { series, totals };
+}
+
+/**
+ * Clear a session's heartbeat row (e.g. on clean shutdown / session reset).
+ * Non-fatal: no-op if the row doesn't exist.
+ */
+export function clearSessionHeartbeat(
+  pid: number,
+  sessionId: string,
+  indexDir: string = getIndexDir(),
+): void {
+  const db = openIndexStore(indexDir);
+  db.prepare(
+    "DELETE FROM session_heartbeats WHERE pid = @pid AND session_id = @session_id",
+  ).run({ pid, session_id: sessionId });
+}

package/src/vector-read.ts

@@ -0,0 +1,237 @@
+/**
+ * vector-read.ts — free functions for VectorStore read-only operations.
+ *
+ * Extracted from vectorStore.ts (PR0 split) to bring it under 500 lines.
+ * All functions take `store: VectorStore` as first param and access store
+ * fields/props directly via type-cast to access private fields (acceptable
+ * since these fields were effectively public via the original methods).
+ *
+ * Call sites in src/ and extensions/ are rewritten from `store.stats(sid)`
+ * → `vectorStats(store, sid)`.
+ */
+
+import type { Vector } from "./embedder.js";
+import { cosineSimilarity } from "./embedder.js";
+import type { StoredCheckpoint, SessionState } from "./store.js";
+import {
+  listCheckpoints,
+  loadSessionState,
+  saveSessionState,
+  setDedupStatus,
+  getDedupStats,
+  repoStats as repoStatsFromStore,
+  dataInvariantStats,
+} from "./store/sqlite.js";
+import { normalizeSessionId } from "./store.js";
+import { computeRegionHash } from "./vectorStore.js";
+import type { VectorStore } from "./vectorStore.js";
+import type { SearchHit } from "./vectorStore.js";
+
+// ---------------------------------------------------------------------------
+// Cosine
+// ---------------------------------------------------------------------------
+
+/** Convenience wrapper for raw vector cosine similarity (exposed for tests). */
+export function vectorSimilarity(_store: VectorStore, a: Vector, b: Vector): number {
+  return cosineSimilarity(a, b);
+}
+
+// ---------------------------------------------------------------------------
+// SemDeDup
+// ---------------------------------------------------------------------------
+
+/**
+ * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
+ * lower-quality row of any pair scoring cosine > `threshold` as
+ * `dedup_status='removed'` (kept, not deleted — retrieval excludes it). Keeps
+ * the row with the higher `tokenEstimate` (more context preserved). Runs as a
+ * single scan; idempotent (re-running skips already-removed rows).
+ *
+ * Returns the number of rows marked removed.
+ */
+export function vectorSemDedup(
+  store: VectorStore,
+  sessionId: string,
+  threshold?: number,
+): number {
+  const sid = normalizeSessionId(sessionId);
+  const stateDir = store.stateDir;
+  const cfg = store.cfg;
+  const thr = threshold ?? cfg.SEMDEDUP_COSINE;
+  const cps = listCheckpoints(sid, stateDir).filter(
+    (c) => c.dedupStatus !== "removed",
+  );
+  let removed = 0;
+  for (let i = 0; i < cps.length; i++) {
+    for (let j = i + 1; j < cps.length; j++) {
+      const a = cps[i];
+      const b = cps[j];
+      if (a.dedupStatus === "removed" || b.dedupStatus === "removed") continue;
+      if (cosineSimilarity(a.embedding, b.embedding) > thr) {
+        const keep = a.tokenEstimate >= b.tokenEstimate ? a : b;
+        const drop = keep === a ? b : a;
+        setDedupStatus(drop.checkpointId, sid, "removed", stateDir);
+        drop.dedupStatus = "removed";
+        removed++;
+      }
+    }
+  }
+  return removed;
+}
+
+// ---------------------------------------------------------------------------
+// Dedup sentinel
+// ---------------------------------------------------------------------------
+
+/**
+ * Dedup sentinel check: has this region already been stored/represented?
+ * Consulted by both the persist path and the recall/inline path.
+ */
+export function vectorDedupe(
+  store: VectorStore,
+  sessionId: string,
+  regionHashOrText: string,
+  isText = false,
+): boolean {
+  const stateDir = store.stateDir;
+  const sid = normalizeSessionId(sessionId);
+  const hash = isText
+    ? computeRegionHash(regionHashOrText)
+    : regionHashOrText;
+  const state = loadSessionState(sid, stateDir);
+  if (state.storedRegionHashes.includes(hash)) return true;
+  return listCheckpoints(sid, stateDir).some(
+    (c) => c.regionHash === hash,
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Injection tracking
+// ---------------------------------------------------------------------------
+
+/** Mark a checkpoint as injected into the window (recall dedup). */
+export function vectorMarkInjected(store: VectorStore, sessionId: string, checkpointId: string): void {
+  const stateDir = store.stateDir;
+  const sid = normalizeSessionId(sessionId);
+  const state = loadSessionState(sid, stateDir);
+  if (!state.injectedCheckpointIds.includes(checkpointId)) {
+    state.injectedCheckpointIds.push(checkpointId);
+    saveSessionState(sid, state, stateDir);
+  }
+}
+
+/** True if this checkpoint was already injected this session. */
+export function vectorWasInjected(store: VectorStore, sessionId: string, checkpointId: string): boolean {
+  const stateDir = store.stateDir;
+  const state: SessionState = loadSessionState(
+    normalizeSessionId(sessionId),
+    stateDir,
+  );
+  return state.injectedCheckpointIds.includes(checkpointId);
+}
+
+// ---------------------------------------------------------------------------
+// List & TopSimilar
+// ---------------------------------------------------------------------------
+
+/** All checkpoints for a session (sorted by checkpointId). */
+export function vectorList(store: VectorStore, sessionId: string): StoredCheckpoint[] {
+  const stateDir = store.stateDir;
+  return listCheckpoints(normalizeSessionId(sessionId), stateDir);
+}
+
+/**
+ * Return the n most similar checkpoints to the current (most recent) checkpoint
+ * by cosine similarity. Returns fewer than n if the session has fewer checkpoints.
+ * The current checkpoint itself is excluded from results.
+ */
+export function vectorTopSimilar(store: VectorStore, sessionId: string, n: number): SearchHit[] {
+  const stateDir = store.stateDir;
+  const sid = normalizeSessionId(sessionId);
+  const checkpoints = listCheckpoints(sid, stateDir);
+  if (checkpoints.length <= 1) return [];
+
+  const ordered = [...checkpoints].sort((a, b) =>
+    a.checkpointId.localeCompare(b.checkpointId),
+  );
+  const current = ordered[ordered.length - 1];
+
+  const scored: SearchHit[] = ordered
+    .filter((cp) => cp.checkpointId !== current.checkpointId)
+    .map((cp) => ({
+      checkpoint: cp,
+      score: cosineSimilarity(current.embedding, cp.embedding),
+    }))
+    .sort((a, b) => b.score - a.score);
+
+  return scored.slice(0, n);
+}
+
+// ---------------------------------------------------------------------------
+// Stats
+// ---------------------------------------------------------------------------
+
+export interface VectorStats {
+  checkpointCount: number;
+  totalTokenEstimate: number;
+  lastCheckpointId: string | undefined;
+  lastSummary: string | undefined;
+  injectedCount: number;
+  dedupHitRate: number;
+  storageDedupRate: number;
+  tokensSaved: number;
+  originalTokens: number;
+  dedupAttempts: number;
+  dedupCollapsed: number;
+}
+
+/**
+ * Store statistics for status reporting / logging. Returns counts + the last
+ * (highest-numbered) checkpoint, or nulls when the session is empty.
+ */
+export function vectorStats(store: VectorStore, sessionId: string): VectorStats {
+  const stateDir = store.stateDir;
+  const sid = normalizeSessionId(sessionId);
+  const cps = listCheckpoints(sid, stateDir);
+  const state = loadSessionState(sid, stateDir);
+  const ordered = [...cps].sort((a, b) =>
+    a.checkpointId.localeCompare(b.checkpointId),
+  );
+  const last = ordered[ordered.length - 1];
+  const injected = state.injectedCheckpointIds.length;
+  const ds = getDedupStats(stateDir);
+  const sessionTok = cps.reduce((s, c) => s + (c.tokenEstimate ?? 0), 0);
+  const sessionOrig = cps.reduce((s, c) => s + (c.originalTokenEstimate ?? 0), 0);
+  const sessionSaved = cps.reduce(
+    (s, c) => s + Math.max(0, (c.originalTokenEstimate ?? 0) - (c.tokenEstimate ?? 0)),
+    0,
+  );
+  return {
+    checkpointCount: cps.length,
+    totalTokenEstimate: sessionTok,
+    lastCheckpointId: last?.checkpointId,
+    lastSummary: last?.summary,
+    injectedCount: injected,
+    dedupHitRate: cps.length === 0 ? 0 : injected / cps.length,
+    storageDedupRate: ds.attempts === 0 ? 0 : ds.deduped / ds.attempts,
+    tokensSaved: sessionSaved,
+    originalTokens: sessionOrig,
+    dedupAttempts: ds.attempts,
+    dedupCollapsed: ds.deduped,
+  };
+}
+
+/**
+ * Repo-wide stats — aggregates every session in this store (one per repo).
+ * Cumulative, resumable, cross-device.
+ */
+export function vectorRepoStats(store: VectorStore): ReturnType<typeof repoStatsFromStore> {
+  const stateDir = store.stateDir;
+  return repoStatsFromStore(stateDir);
+}
+
+/** Data-safety invariant: regions retained vs bytes permanently deleted. */
+export function vectorDataInvariant(store: VectorStore): ReturnType<typeof dataInvariantStats> {
+  const stateDir = store.stateDir;
+  return dataInvariantStats(stateDir);
+}