pi-mega-compact 0.11.5 → 0.11.6

package/src/memoryGraph.ts

@@ -0,0 +1,438 @@
+/**
+ * memoryGraph.ts — S46 visual memory map: build a graph of graphs.
+ *
+ * Nodes = checkpoints (with optional RAPTOR level annotations).
+ * Edges = temporal adjacency, semantic similarity (cosine above threshold), and
+ *         RAPTOR parent–child links.
+ *
+ * pi-agnostic (no pi runtime types). Zero network calls (PREVENT-PI-004).
+ * Synchronous read-only interface — non-fatal on store errors.
+ *
+ * Re-exports the public types as both MemoryGraph* and shorter aliases
+ * (GraphNode, GraphEdge, GraphMetadata) so dashboard api-contracts can import
+ * from the canonical source.
+ */
+import { getStateDir, normalizeSessionId } from "./store.js";
+import { openStore, decodeEmbedding } from "./store/sqlite/utils.js";
+import { listRaptorNodes } from "./store/sqlite/raptor.js";
+import { Logger } from "./log.js";
+const log = new Logger();
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** A node in the memory graph represents one checkpoint (potentially with
+ *  RAPTOR level information if it participates in a RAPTOR tree). */
+export interface MemoryGraphNode {
+  id: string;
+  sessionId: string;
+  label: string;
+  summaryTruncated: string;
+  tokenEstimate: number;
+  /** Unix-epoch ms (from checkpoint timestamp or created_at). */
+  timestamp: number;
+  dedupStatus: string | undefined;
+  /** 0 = no RAPTOR parent; 1+ = RAPTOR cluster level. */
+  raptorLevel: number;
+  topicSummary: string | undefined;
+  decisionCount: number;
+  /** First ~200 chars of the checkpoint summary/text for inline preview. */
+  textSnippet: string;
+}
+
+/** Edge types indicate the *reason* two nodes are connected. */
+export type MemoryGraphEdgeType = "temporal" | "semantic" | "raptor_parent";
+
+export interface MemoryGraphEdge {
+  source: string;
+  target: string;
+  weight: number;
+  type: MemoryGraphEdgeType;
+}
+
+export interface MemoryGraphMetadata {
+  sessionCount: number;
+  totalCheckpoints: number;
+  totalEdges: number;
+  semanticEdgeCount: number;
+  temporalEdgeCount: number;
+  raptorEdgeCount: number;
+  similarityThresholdUsed: number;
+  builtAt: number;
+}
+
+/** The complete memory graph returned by buildMemoryGraph. */
+export interface MemoryGraph {
+  nodes: MemoryGraphNode[];
+  edges: MemoryGraphEdge[];
+  metadata: MemoryGraphMetadata;
+}
+
+// ---------------------------------------------------------------------------
+// Short aliases for dashboard api-contracts import convenience
+// ---------------------------------------------------------------------------
+
+export type GraphNode = MemoryGraphNode;
+export type GraphEdge = MemoryGraphEdge;
+export type GraphMetadata = MemoryGraphMetadata;
+
+/** Options to configure graph construction. */
+export interface MemoryGraphOptions {
+  /** Similarity threshold for semantic edges (cosine). Default 0.7. */
+  similarityThreshold?: number;
+  /** Maximum number of semantic edges per node (avoids dense clique). Default 4. */
+  maxEdgesPerNode?: number;
+  /** Optional session filter — undefined = all sessions. */
+  sessionId?: string;
+  /** State directory (defaults to getStateDir()). */
+  stateDir?: string;
+  /** Include RAPTOR parent–child edges when available. Default true. */
+  includeRaptorEdges?: boolean;
+  /** Include temporal adjacency edges (consecutive checkpoints in same session). Default true. */
+  includeTemporalEdges?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+
+const DEFAULT_SIMILARITY_THRESHOLD = 0.7;
+const DEFAULT_MAX_EDGES_PER_NODE = 4;
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a memory graph from the store.
+ *
+ * Opens the SQLite store, reads all (or session-filtered) checkpoints,
+ * computes cosine similarity edges, temporal edges, and RAPTOR parent edges.
+ * Returns a MemoryGraph or an empty graph on failure (non-fatal).
+ */
+export function buildMemoryGraph(opts: MemoryGraphOptions = {}): MemoryGraph {
+  const {
+    similarityThreshold = DEFAULT_SIMILARITY_THRESHOLD,
+    maxEdgesPerNode = DEFAULT_MAX_EDGES_PER_NODE,
+    sessionId,
+    stateDir = getStateDir(),
+    includeRaptorEdges = true,
+    includeTemporalEdges = true,
+  } = opts;
+
+  const builtAt = Date.now();
+
+  try {
+    const nodes = loadNodes(sessionId, stateDir);
+    const nodeIndex = buildNodeIndex(nodes);
+    const edges: MemoryGraphEdge[] = [];
+
+    // Temporal edges: consecutive checkpoints within the same session
+    if (includeTemporalEdges) {
+      edges.push(...buildTemporalEdges(nodes));
+    }
+
+    // Semantic edges: cosine similarity above threshold
+    edges.push(...buildSemanticEdges(nodes, nodeIndex, similarityThreshold, maxEdgesPerNode));
+
+    // RAPTOR parent–child edges
+    if (includeRaptorEdges) {
+      edges.push(...buildRaptorEdges(nodes, sessionId, stateDir, nodeIndex));
+    }
+
+    // Deduplicate edges (keep highest weight for each source-target pair)
+    const dedupedEdges = deduplicateEdges(edges);
+
+    // Count sessions
+    const sessionSet = new Set<string>();
+    for (const n of nodes) {
+      sessionSet.add(n.sessionId);
+    }
+
+    const meta: MemoryGraphMetadata = {
+      sessionCount: sessionSet.size,
+      totalCheckpoints: nodes.length,
+      totalEdges: dedupedEdges.length,
+      semanticEdgeCount: dedupedEdges.filter((e) => e.type === "semantic").length,
+      temporalEdgeCount: dedupedEdges.filter((e) => e.type === "temporal").length,
+      raptorEdgeCount: dedupedEdges.filter((e) => e.type === "raptor_parent").length,
+      similarityThresholdUsed: similarityThreshold,
+      builtAt,
+    };
+
+    return { nodes, edges: dedupedEdges, metadata: meta };
+  } catch (err) {
+    log.warn("buildMemoryGraph failed", { error: String(err) });
+    return {
+      nodes: [],
+      edges: [],
+      metadata: {
+        sessionCount: 0,
+        totalCheckpoints: 0,
+        totalEdges: 0,
+        semanticEdgeCount: 0,
+        temporalEdgeCount: 0,
+        raptorEdgeCount: 0,
+        similarityThresholdUsed: similarityThreshold,
+        builtAt,
+      },
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Node loading
+// ---------------------------------------------------------------------------
+
+function loadNodes(sessionId: string | undefined, stateDir: string): MemoryGraphNode[] {
+  const db = openStore(stateDir);
+
+  const sql = sessionId
+    ? "SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC"
+    : "SELECT * FROM context_chunks ORDER BY session_id, id ASC";
+
+  const rows = sessionId
+    ? (db.prepare(sql).all(normalizeSessionId(sessionId)) as any[])
+    : (db.prepare(sql).all() as any[]);
+
+  return rows.map(rowToGraphNode);
+}
+
+function rowToGraphNode(row: any): MemoryGraphNode {
+  const summary = row.summary ?? "";
+  return {
+    id: row.id,
+    sessionId: row.session_id,
+    label: row.id,
+    summaryTruncated: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
+    tokenEstimate: row.token_estimate ?? 0,
+    timestamp: row.created_at ?? Date.now(),
+    dedupStatus: row.dedup_status ?? undefined,
+    raptorLevel: 0, // enriched below if RAPTOR data exists
+    topicSummary: row.topic_summary ?? undefined,
+    decisionCount: row.key_decisions
+      ? safeJsonCount(row.key_decisions)
+      : 0,
+    textSnippet: summary.length > 200 ? summary.slice(0, 197) + "..." : summary,
+  };
+}
+
+/** Parse a JSON array stored in a text column and return its length. */
+function safeJsonCount(raw: string | null | undefined): number {
+  if (!raw) return 0;
+  try {
+    const arr = JSON.parse(raw);
+    return Array.isArray(arr) ? arr.length : 0;
+  } catch {
+    return 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Node index building
+// ---------------------------------------------------------------------------
+
+function buildNodeIndex(nodes: MemoryGraphNode[]): Map<string, number> {
+  const idx = new Map<string, number>();
+  nodes.forEach((n, i) => idx.set(n.id, i));
+  return idx;
+}
+
+// ---------------------------------------------------------------------------
+// Edges: temporal
+// ---------------------------------------------------------------------------
+
+function buildTemporalEdges(nodes: MemoryGraphNode[]): MemoryGraphEdge[] {
+  const edges: MemoryGraphEdge[] = [];
+  if (nodes.length < 2) return edges;
+
+  let prev = nodes[0];
+  for (let i = 1; i < nodes.length; i++) {
+    const cur = nodes[i];
+    if (cur.sessionId === prev.sessionId) {
+      edges.push({
+        source: prev.id,
+        target: cur.id,
+        weight: 1.0,
+        type: "temporal",
+      });
+    }
+    prev = cur;
+  }
+  return edges;
+}
+
+// ---------------------------------------------------------------------------
+// Edges: semantic (cosine similarity)
+// ---------------------------------------------------------------------------
+
+function buildSemanticEdges(
+  nodes: MemoryGraphNode[],
+  _nodeIndex: Map<string, number>,
+  threshold: number,
+  maxPerNode: number,
+): MemoryGraphEdge[] {
+  if (nodes.length < 2) return [];
+  const db = openStore();
+
+  // Retrieve embeddings from DB for each node
+  const embeddings = new Map<string, number[]>();
+  const placeholders = nodes.map(() => "?").join(",");
+  const ids = nodes.map((n) => n.id);
+  const rows = db
+    .prepare(`SELECT id, embedding_blob FROM context_chunks WHERE id IN (${placeholders})`)
+    .all(...ids) as unknown as Array<{ id: string; embedding_blob: Uint8Array | null }>;
+
+  for (const row of rows) {
+    const emb = decodeEmbedding(row.embedding_blob as Uint8Array | null | undefined);
+    if (emb.length > 0) {
+      embeddings.set(row.id, emb);
+    }
+  }
+
+  type EdgeCandidate = { target: string; weight: number };
+  const edges: MemoryGraphEdge[] = [];
+
+  for (const node of nodes) {
+    const aId = node.id;
+    const aEmb = embeddings.get(aId);
+    if (!aEmb) continue;
+
+    const candidates: EdgeCandidate[] = [];
+
+    for (const other of nodes) {
+      if (other.id === aId) continue;
+      // Skip if already connected via temporal or raptor edge
+      const bEmb = embeddings.get(other.id);
+      if (!bEmb) continue;
+
+      const sim = cosineSimilarity(aEmb, bEmb);
+      if (sim >= threshold) {
+        candidates.push({ target: other.id, weight: sim });
+      }
+    }
+
+    // Keep top-K highest-weight semantic neighbors
+    candidates.sort((a, b) => b.weight - a.weight);
+    const top = candidates.slice(0, maxPerNode);
+    for (const c of top) {
+      edges.push({
+        source: aId,
+        target: c.target,
+        weight: c.weight,
+        type: "semantic",
+      });
+    }
+  }
+
+  return edges;
+}
+
+// ---------------------------------------------------------------------------
+// Edges: RAPTOR parent–child
+// ---------------------------------------------------------------------------
+
+function buildRaptorEdges(
+  nodes: MemoryGraphNode[],
+  sessionId: string | undefined,
+  stateDir: string,
+  nodeIndex: Map<string, number>,
+): MemoryGraphEdge[] {
+  const edges: MemoryGraphEdge[] = [];
+
+  try {
+    if (sessionId) {
+      enrichRaptorForSession(sessionId, stateDir, nodeIndex, nodes, edges);
+    } else {
+      // Load all sessions from DB
+      const db = openStore(stateDir);
+      const sessionRows = db
+        .prepare("SELECT DISTINCT session_id FROM raptor_nodes ORDER BY session_id")
+        .all() as any[];
+      for (const row of sessionRows) {
+        enrichRaptorForSession(row.session_id as string, stateDir, nodeIndex, nodes, edges);
+      }
+    }
+  } catch (err) {
+    log.warn("RAPTOR edge build failed (non-fatal)", { error: String(err) });
+  }
+
+  return edges;
+}
+
+function enrichRaptorForSession(
+  sessId: string,
+  stateDir: string,
+  nodeIndex: Map<string, number>,
+  nodes: MemoryGraphNode[],
+  edges: MemoryGraphEdge[],
+): void {
+  const raptorNodes = listRaptorNodes(sessId, stateDir);
+
+  for (const rn of raptorNodes) {
+    const idx = nodeIndex.get(rn.id);
+    if (idx !== undefined) {
+      // Annotate the graph node with the RAPTOR level
+      nodes[idx].raptorLevel = rn.level;
+    }
+
+    if (!rn.parentId) continue;
+    // Only create edge if both nodes are in the graph
+    if (nodeIndex.has(rn.id) && nodeIndex.has(rn.parentId)) {
+      edges.push({
+        source: rn.id,
+        target: rn.parentId,
+        weight: Math.max(0.1, 1.0 - rn.level * 0.1), // higher levels = weaker link
+        type: "raptor_parent",
+      });
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Edge deduplication
+// ---------------------------------------------------------------------------
+
+function deduplicateEdges(edges: MemoryGraphEdge[]): MemoryGraphEdge[] {
+  const seen = new Map<string, MemoryGraphEdge>();
+
+  for (const e of edges) {
+    const keyA = `${e.source}|${e.target}`;
+    const keyB = `${e.target}|${e.source}`;
+    const existing = seen.get(keyA) ?? seen.get(keyB);
+
+    if (!existing) {
+      seen.set(keyA, e);
+    } else if (e.weight > existing.weight) {
+      // Replace with higher-weight edge
+      seen.delete(keyA);
+      seen.delete(keyB);
+      seen.set(keyA, e);
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+// ---------------------------------------------------------------------------
+// Cosine similarity
+// ---------------------------------------------------------------------------
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  const len = Math.min(a.length, b.length);
+  if (len === 0) return 0;
+
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}

package/src/monitoring.ts

@@ -24,6 +24,10 @@ export interface DedupDecisionEvent {
   latencyMs: number;
   /** True when this dedup was later found to be a false positive. */
   falsePositive?: boolean;
+  /** Cosine similarity score (L2 semantic tier) or 1 for exact match (L0/L1). */
+  similarityScore?: number;
+  /** Checkpoint ID of the matched entry. */
+  matchedId?: string;
 }
 
 export interface DedupMetrics {
@@ -169,3 +173,34 @@ export function defaultMetricsPath(stateDir: string = STATE_DIR_DEFAULT): string
 export function defaultEventsPath(stateDir: string = STATE_DIR_DEFAULT): string {
   return join(stateDir, "events.log");
 }
+
+// ---------------------------------------------------------------------------
+// S45 CRAG quality metrics event
+// ---------------------------------------------------------------------------
+
+export interface RecallQualityEvent {
+  ts: number;
+  sessionId: string;
+  tier: "CRAG";
+  score: number;
+  breakdown: { relevance: number; coverage: number; diversity: number; specificity: number };
+  expanded: boolean;
+  reRetrieved: boolean;
+  uncalibrated: boolean;
+  weights: { relevance: number; coverage: number; diversity: number; specificity: number };
+  note?: string;
+  error?: string;
+}
+
+/**
+ * Append a RecallQualityEvent to events.log (best-effort, non-fatal).
+ * Follows the same append-one-JSON-line pattern as logDecision.
+ */
+export function logRecallQuality(path: string, ev: RecallQualityEvent): void {
+  try {
+    mkdirSync(dirname(path), { recursive: true });
+    appendFileSync(path, JSON.stringify(ev) + "\n", "utf-8");
+  } catch {
+    /* best-effort — never break the extension */
+  }
+}

package/src/queryExpansion.ts

@@ -0,0 +1,206 @@
+/**
+ * queryExpansion.ts — S45 CRAG query expansion via embedding neighbors.
+ *
+ * Expands a query with synonym/related-term variants by finding embedding-
+ * similar checkpoints and extracting their discriminative terms. Fully local:
+ * uses the TrigramEmbedder (shipped default, zero network) and in-process
+ * cosine scan — no LLM call, no external API. PREVENT-PI-004 compliant.
+ *
+ * Pi-agnostic: no pi runtime types or imports from `src/`.
+ */
+
+import type { Embedder } from "./embedder.js";
+import { cosineSimilarity, defaultEmbedder } from "./embedder.js";
+import type { VectorStore } from "./vectorStore.js";
+import { listCheckpoints } from "./store/sqlite.js";
+
+/**
+ * Options for query expansion.
+ */
+export interface QueryExpansionOptions {
+  /** Maximum number of expansion terms to return. Default 5. */
+  maxTerms?: number;
+  /** Minimum cosine similarity for a checkpoint to be considered. Default 0.3. */
+  minSimilarity?: number;
+  /** Embedder override (defaults to TrigramEmbedder, zero network). */
+  embedder?: Embedder;
+}
+
+const DEFAULT_MAX_TERMS = 5;
+const DEFAULT_MIN_SIM = 0.3;
+
+/**
+ * Tokenize text into lowercased alphanumeric terms, filtering stop words.
+ * Internal helper — no export needed outside this module.
+ */
+function tokenizeTerms(text: string): string[] {
+  const STOP_WORDS = new Set([
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "as", "is", "was", "are", "were", "be",
+    "been", "has", "have", "had", "do", "does", "did", "will", "would",
+    "could", "should", "may", "might", "shall", "can", "not", "no", "nor",
+    "so", "if", "then", "than", "that", "this", "these", "those", "it",
+    "its", "i", "me", "my", "we", "our", "you", "your", "he", "she",
+    "they", "them", "their", "what", "which", "who", "whom", "when",
+    "where", "why", "how", "all", "each", "every", "both", "few", "more",
+    "most", "other", "some", "such", "only", "own", "same", "so", "than",
+    "too", "very", "just", "about", "above", "after", "again", "against",
+    "because", "before", "between", "down", "during", "out", "over",
+    "through", "under", "up", "also", "into", "off", "onto", "upon",
+  ]);
+
+  // Split on non-alphanumeric, lowercase, filter empties and stop words.
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((t) => t.length >= 3 && !STOP_WORDS.has(t));
+}
+
+/**
+ * Compute TF-IDF-like term importance scores for a set of documents.
+ * Returns a Map from term to score (log-frequency within doc * IDF).
+ */
+function termScores(
+  docs: string[],
+): Map<string, number> {
+  const docCount = docs.length;
+  if (docCount === 0) return new Map();
+
+  // Document frequency: how many docs each term appears in.
+  const df = new Map<string, number>();
+  const docTerms = docs.map((d) => {
+    const terms = tokenizeTerms(d);
+    const unique = new Set(terms);
+    for (const t of unique) {
+      df.set(t, (df.get(t) ?? 0) + 1);
+    }
+    return terms;
+  });
+
+  // Compute TF-IDF score for each term across all docs:
+  //   score = sum over docs of (tf * idf)
+  //   tf    = term frequency within doc (log-smooth)
+  //   idf   = log(1 + docCount / (1 + docFreq))
+  const scores = new Map<string, number>();
+  for (let i = 0; i < docTerms.length; i++) {
+    const terms = docTerms[i];
+    const tf = new Map<string, number>();
+    for (const t of terms) tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const [t, count] of tf) {
+      const tfVal = 1 + Math.log10(count);
+      const docFreq = df.get(t) ?? 1;
+      const idf = Math.log10(1 + docCount / (1 + docFreq));
+      scores.set(t, (scores.get(t) ?? 0) + tfVal * idf);
+    }
+  }
+
+  return scores;
+}
+
+/**
+ * Expand a query with related terms discovered from embedding-neighbor
+ * checkpoints. Uses TF-IDF term extraction from the most similar checkpoints
+ * to find novel, discriminative terms not already in the query.
+ *
+ * @param query   The original search query.
+ * @param store   VectorStore instance (for checkpoint access + embedder).
+ * @returns       Array of expansion terms (lowercased, max `maxTerms`).
+ */
+export function expandQuery(
+  query: string,
+  store: VectorStore,
+  opts: QueryExpansionOptions = {},
+): string[] {
+  const maxTerms = opts.maxTerms ?? DEFAULT_MAX_TERMS;
+  const minSim = opts.minSimilarity ?? DEFAULT_MIN_SIM;
+  const embedder = opts.embedder ?? store.embedder ?? defaultEmbedder();
+
+  // Embed the query and scan all checkpoints for nearest neighbors.
+  const qv = embedder.embed(query);
+
+  // We need a valid session scope. Since VectorStore checkpoint access flows
+  // through the SQLite store, list checkpoints from the store's stateDir.
+  // We scan all sessions' checkpoints in the local store for broad coverage.
+  // Aggregate candidate documents from the top-N similar checkpoints.
+  interface Candidate {
+    summary: string;
+    score: number;
+  }
+  const candidates: Candidate[] = [];
+  const allSessions = collectSessionIds(store.stateDir);
+  const seen = new Set<string>();
+
+  for (const sid of allSessions) {
+    const cps = listCheckpoints(sid, store.stateDir);
+    for (const cp of cps) {
+      if (!cp.summary || seen.has(cp.checkpointId)) continue;
+      seen.add(cp.checkpointId);
+      const sim = cosineSimilarity(qv, cp.embedding);
+      if (sim >= minSim) {
+        candidates.push({ summary: cp.summary, score: sim });
+      }
+    }
+  }
+
+  if (candidates.length === 0) return [];
+
+  // Sort by similarity descending, take the top 20 for term extraction.
+  const topCandidates = candidates
+    .sort((a, b) => b.score - a.score)
+    .slice(0, 20);
+
+  // Compute TF-IDF scores across the candidate summaries.
+  const docTexts = topCandidates.map((c) => c.summary);
+  const scores = termScores(docTexts);
+
+  // Filter out query terms: we want NOVEL terms, not the ones already in the query.
+  const queryTerms = new Set(tokenizeTerms(query));
+
+  // Sort terms by score descending, filter query terms and short/noise terms.
+  const ranked = [...scores.entries()]
+    .filter(([term]) => !queryTerms.has(term))
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, maxTerms)
+    .map(([term]) => term);
+
+  return ranked;
+}
+
+/**
+ * Collect unique session IDs that have at least one checkpoint in the store.
+ * Scans the SQLite checkpoint table. Returns an empty array on any error
+ * (non-fatal — query expansion degrades gracefully).
+ */
+function collectSessionIds(stateDir: string): string[] {
+  try {
+    // We import listCheckpoints per session dynamically. A simpler approach:
+    // scan via a direct SQL query rather than reading every session.
+    // Reuse store/sqlite.ts functions — we can read the sessions table.
+    // Fallback: try listing checkpoints for all known session IDs via
+    // the sessions table.
+    const { db } = loadDb(stateDir);
+    const rows = db
+      .prepare("SELECT DISTINCT session_id FROM checkpoint_epochs ORDER BY session_id")
+      .all() as Array<{ session_id: string }>;
+    return rows.map((r) => r.session_id);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Lazy-load the SQLite database handle for a given stateDir.
+ * Cached per stateDir to avoid re-opening the file on every call.
+ */
+const _dbCache = new Map<string, { db: import("node:sqlite").DatabaseSync }>();
+
+function loadDb(stateDir: string): { db: import("node:sqlite").DatabaseSync } {
+  const cached = _dbCache.get(stateDir);
+  if (cached) return cached;
+  const { DatabaseSync } = require("node:sqlite") as typeof import("node:sqlite");
+  const { join } = require("node:path") as typeof import("node:path");
+  const db = new DatabaseSync(join(stateDir, "mega-compact.db"));
+  db.exec("PRAGMA journal_mode=WAL");
+  _dbCache.set(stateDir, { db });
+  return { db };
+}