pi-mega-compact 0.7.8 → 0.7.9

package/src/store/sqlite/memories.ts

@@ -0,0 +1,217 @@
+/**
+ * memories.ts — durable "save to memory" store (taken over from memory extensions).
+ *
+ * One SQLite store for user-saved memories, scoped by repo. Mirrors the
+ * lessons/sessions pattern: all state lives in SQLite from day one.
+ */
+import { getStateDir } from "../../store.js";
+import { openStore } from "./utils.js";
+
+// S24 storage hardening: keep each memory row bounded so the durable store can
+// never blow a downstream consumer's per-entry buffer (e.g. pi's native
+// file-backed memory caps a single entry at ~5k chars). We truncate content at
+// MEMORY_MAX_CHARS and evict the least-recently-referenced rows past
+// MEMORY_MAX_ROWS per repo via LRU. Both are SQLite-only (PREVENT-PI-004): no
+// file-backed memory is written anywhere. Defaults are overridable via env
+// (MEGACOMPACT_MEMORY_MAX_CHARS / MEGACOMPACT_MEMORY_MAX_ROWS).
+export const MEMORY_MAX_CHARS = 4000;
+export const MEMORY_MAX_ROWS = 500;
+
+/** Read an env override as a positive int, falling back to `fallback`. */
+function envInt(name: string, fallback: number): number {
+  const v = process.env[name];
+  if (v == null || v === "") return fallback;
+  const n = Number(v);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+
+/** Effective per-entry char cap (env-overridable, default MEMORY_MAX_CHARS). */
+export function memoryMaxChars(): number {
+  return envInt("MEGACOMPACT_MEMORY_MAX_CHARS", MEMORY_MAX_CHARS);
+}
+
+/** Effective per-repo row cap (env-overridable, default MEMORY_MAX_ROWS). */
+export function memoryMaxRows(): number {
+  return envInt("MEGACOMPACT_MEMORY_MAX_ROWS", MEMORY_MAX_ROWS);
+}
+
+/** Truncate memory content to the per-entry cap, preserving a trailing marker. */
+function capMemoryContent(content: string): string {
+  const cap = memoryMaxChars();
+  if (content.length <= cap) return content;
+  return content.slice(0, cap) + "…[truncated]";
+}
+
+/**
+ * Evict the least-recently-referenced rows for a repo past MEMORY_MAX_ROWS.
+ * LRU key = COALESCE(last_referenced, last_recalled_at, created_at) so a memory
+ * that is recalled/referenced survives over a stale one. Best-effort: any error
+ * is swallowed by the caller. Repo-scoped so one noisy repo can't evict another.
+ */
+function evictMemoryLru(repo: string | null, stateDir: string): void {
+  const db = openStore(stateDir);
+  const maxRows = memoryMaxRows();
+  // SQLite `= NULL` is never true, so the null-repo scope (memories are
+  // stateDir-scoped when repo is null — the applyMemoryOps path) needs `IS NULL`.
+  const where = repo == null ? "repo IS NULL" : "repo = ?";
+  const countRow = repo == null
+    ? db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get()
+    : db.prepare(`SELECT COUNT(*) AS n FROM memories WHERE ${where}`).get(repo);
+  const count = (countRow as { n: number }).n;
+  const over = count - maxRows;
+  if (over <= 0) return;
+  // Delete the `over` least-recently-used rows. ORDER BY the LRU key ASC, id ASC
+  // (id ASC breaks ties deterministically — oldest created first). The `where`
+  // clause is a code-controlled constant (never user input) → PREVENT-002 OK.
+  const sql =
+    `DELETE FROM memories WHERE ${where} AND id IN (
+       SELECT id FROM memories WHERE ${where}
+       ORDER BY COALESCE(last_referenced, last_recalled_at, created_at) ASC, id ASC
+       LIMIT ?
+     )`;
+  if (repo == null) db.prepare(sql).run(over);
+  else db.prepare(sql).run(repo, repo, over);
+}
+
+export interface MemoryRecord {
+  id: number;
+  repo: string | null;
+  kind: string;
+  content: string;
+  tags: string[];
+  createdAt: number;
+  lastRecalledAt: number | null;
+  category: string | null;
+  target: string | null;
+  lastReferenced: number | null;
+  sourceTurn: number | null;
+}
+
+/** Save a memory to the current repo's store. Returns the new row id.
+ *  S24 hardening: content is truncated to MEMORY_MAX_CHARS and, once the per-repo
+ *  row count exceeds MEMORY_MAX_ROWS, the least-recently-used rows are evicted
+ *  (LRU) so the store stays bounded. */
+export function addMemory(
+  memory: { kind?: string; content: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
+  repo: string | null,
+  stateDir: string = getStateDir(),
+): number {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db
+    .prepare(
+      `INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at, category, target, source_turn)
+       VALUES(?, ?, ?, ?, ?, NULL, ?, ?, ?)`,
+    )
+    .run(
+      repo ?? null,
+      memory.kind ?? "note",
+      capMemoryContent(memory.content),
+      JSON.stringify(memory.tags ?? []),
+      now,
+      memory.category ?? null,
+      memory.target ?? null,
+      memory.sourceTurn ?? null,
+    );
+  try {
+    evictMemoryLru(repo, stateDir);
+  } catch {
+    /* non-fatal: eviction must never fail an add */
+  }
+  return Number(res.lastInsertRowid);
+}
+
+/** List recent memories for a repo (or all repos when repo is null). */
+export function listMemories(repo: string | null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
+  const db = openStore(stateDir);
+  const rows = repo
+    ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
+    : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
+  return (rows as any[]).map(mapMemoryRow);
+}
+
+/** Substring search across content + tags. */
+export function searchMemories(query: string, repo: string | null = null, limit = 50, stateDir: string = getStateDir()): MemoryRecord[] {
+  const db = openStore(stateDir);
+  const like = `%${query}%`;
+  const rows = repo
+    ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
+    : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
+  return (rows as any[]).map(mapMemoryRow);
+}
+
+/** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
+export function recallMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
+  return res.changes > 0;
+}
+
+/** Mark a memory as referenced (updates last_referenced). Returns true if found. */
+export function referenceMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  const res = db.prepare("UPDATE memories SET last_referenced = ? WHERE id = ?").run(now, id);
+  return res.changes > 0;
+}
+
+/** Replace a memory's mutable fields by id. Returns true if a row was updated. */
+export function replaceMemory(
+  id: number,
+  patch: { kind?: string; content?: string; tags?: string[]; category?: string; target?: string; sourceTurn?: number },
+  stateDir: string = getStateDir(),
+): boolean {
+  const db = openStore(stateDir);
+  const res = db
+    .prepare(
+      `UPDATE memories
+       SET kind = COALESCE(?, kind),
+           content = COALESCE(?, content),
+           tags = COALESCE(?, tags),
+           category = COALESCE(?, category),
+           target = COALESCE(?, target),
+           source_turn = COALESCE(?, source_turn)
+       WHERE id = ?`,
+    )
+    .run(
+      patch.kind ?? null,
+      patch.content != null ? capMemoryContent(patch.content) : null,
+      patch.tags ? JSON.stringify(patch.tags) : null,
+      "category" in patch ? (patch.category ?? null) : null,
+      "target" in patch ? (patch.target ?? null) : null,
+      "sourceTurn" in patch ? (patch.sourceTurn ?? null) : null,
+      id,
+    );
+  return res.changes > 0;
+}
+
+/** Remove a memory by id. Returns true if a row was deleted. */
+export function removeMemory(id: number, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const res = db.prepare("DELETE FROM memories WHERE id = ?").run(id);
+  return res.changes > 0;
+}
+
+/** Look up a single memory by id (or undefined). */
+export function getMemory(id: number, stateDir: string = getStateDir()): MemoryRecord | undefined {
+  const db = openStore(stateDir);
+  const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
+  return row ? mapMemoryRow(row) : undefined;
+}
+
+function mapMemoryRow(row: any): MemoryRecord {
+  return {
+    id: row.id,
+    repo: row.repo ?? null,
+    kind: row.kind ?? "note",
+    content: row.content ?? "",
+    tags: row.tags ? JSON.parse(row.tags) : [],
+    createdAt: row.created_at ?? 0,
+    lastRecalledAt: row.last_recalled_at ?? null,
+    category: row.category ?? null,
+    target: row.target ?? null,
+    lastReferenced: row.last_referenced ?? null,
+    sourceTurn: row.source_turn ?? null,
+  };
+}

package/src/store/sqlite/meta.ts

@@ -0,0 +1,108 @@
+/**
+ * meta.ts — `meta` table key/value helpers + cumulative counters
+ * (tokens_saved, dedup stats, compact count, recall injected, cache-hit tokens).
+ */
+import { getStateDir } from "../../store.js";
+import { openStore } from "./utils.js";
+
+/** Read a string-valued meta key (or undefined). Used for cumulative counters. */
+export function getMeta(key: string, stateDir: string = getStateDir()): string | undefined {
+  const db = openStore(stateDir);
+  const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key) as
+    | { value: string }
+    | undefined;
+  return row?.value;
+}
+
+/**
+ * Cumulative "tokens saved" — the sum of stored checkpoint token estimates across
+ * all compactions in this store (one per repo). Persisted in the SQLite `meta`
+ * table so it survives session restarts and travels with the repo's state dir,
+ * mirroring how `storageDedupRate` is cumulative. Incremented in VectorStore.add()
+ * when a new (non-deduped) checkpoint is persisted.
+ */
+export function getTokensSaved(stateDir: string = getStateDir()): number {
+  const raw = getMeta("tokens_saved", stateDir);
+  const n = raw == null ? 0 : Number(raw);
+  return Number.isFinite(n) ? n : 0;
+}
+
+/** Add `delta` (>=0) to the cumulative tokens-saved counter. */
+export function addTokensSaved(delta: number, stateDir: string = getStateDir()): void {
+  if (!(delta > 0)) return;
+  const db = openStore(stateDir);
+  db.prepare(
+    `INSERT INTO meta(key, value) VALUES('tokens_saved', ?)
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
+  ).run(String(delta), delta);
+}
+
+/** Cumulative store-wide dedup accounting (Sprint 9+). Persisted in the SQLite
+ *  `meta` table so it survives session restarts and travels with the repo's
+ *  state dir — mirroring `tokens_saved`. Replaces the legacy JSON
+ *  `dedup-stats.json` file (all stats now live in the SQLite store). */
+export interface DedupStats {
+  /** Total add() calls (new checkpoints + deduped collapses). */
+  attempts: number;
+  /** add() calls that collapsed onto an existing checkpoint. */
+  deduped: number;
+}
+
+/** Read a store-wide integer counter from the meta table (0 if absent). */
+export function getMetaNumber(key: string, stateDir: string = getStateDir()): number {
+  const raw = getMeta(key, stateDir);
+  const n = raw == null ? 0 : Number(raw);
+  return Number.isFinite(n) ? n : 0;
+}
+
+/** Atomically add `delta` to an integer meta counter. */
+function incMeta(key: string, delta: number, stateDir: string = getStateDir()): void {
+  if (!(delta > 0)) return;
+  const db = openStore(stateDir);
+  db.prepare(
+    `INSERT INTO meta(key, value) VALUES(?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)`,
+  ).run(key, String(delta), delta);
+}
+
+/** Read the cumulative store-wide dedup counters. */
+export function getDedupStats(stateDir: string = getStateDir()): DedupStats {
+  return {
+    attempts: getMetaNumber("dedup_attempts", stateDir),
+    deduped: getMetaNumber("deduped", stateDir),
+  };
+}
+
+/** Increment the store-wide dedup counters for one add() call. */
+export function bumpDedupStats(deduped: boolean, stateDir: string = getStateDir()): void {
+  incMeta("dedup_attempts", 1, stateDir);
+  if (deduped) incMeta("deduped", 1, stateDir);
+}
+
+// --- Live dashboard counters (schemaless meta key/value — NO migration) -----
+// These reuse the private `incMeta` atomically-incrementing integer counter so
+// all cumulative tallies live in the same `meta` table as tokens_saved etc.
+
+export function incCompactCount(stateDir: string = getStateDir()): void {
+  incMeta("compact_count", 1, stateDir);
+}
+
+export function getCompactCount(stateDir: string = getStateDir()): number {
+  return getMetaNumber("compact_count", stateDir);
+}
+
+export function incRecallInjected(n: number, stateDir: string = getStateDir()): void {
+  if (n > 0) incMeta("recall_injected", n, stateDir);
+}
+
+export function getRecallInjected(stateDir: string = getStateDir()): number {
+  return getMetaNumber("recall_injected", stateDir);
+}
+
+export function incCacheHitTokens(delta: number, stateDir: string = getStateDir()): void {
+  if (delta > 0) incMeta("cache_hit_tokens_saved", delta, stateDir);
+}
+
+export function getCacheHitTokensSaved(stateDir: string = getStateDir()): number {
+  return getMetaNumber("cache_hit_tokens_saved", stateDir);
+}

package/src/store/sqlite/model-snapshots.ts

@@ -0,0 +1,83 @@
+/**
+ * model-snapshots.ts — `model_snapshots` table (active model/provider per repo).
+ */
+import { getStateDir } from "../../store.js";
+import { openStore } from "./utils.js";
+
+/** A captured model/provider snapshot (for cost estimation + dashboard). */
+export interface ModelSnapshot {
+  provider: string;
+  providerName: string | null;
+  modelId: string;
+  modelName: string | null;
+  inputRate: number; // USD per input token
+  outputRate: number; // USD per output token
+  contextWindow: number;
+  maxTokens: number;
+  reasoning: boolean;
+  capturedAt: number;
+}
+
+/** Persist the active model/provider for a repo (latest row wins per repo). */
+export function recordModelSnapshot(
+  repoRoot: string,
+  snap: Omit<ModelSnapshot, "capturedAt">,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  db.prepare(
+    `INSERT INTO model_snapshots
+       (repo_root, provider, provider_name, model_id, model_name, input_rate,
+        output_rate, context_window, max_tokens, reasoning, captured_at)
+     VALUES (@repo_root, @provider, @provider_name, @model_id, @model_name,
+             @input_rate, @output_rate, @context_window, @max_tokens, @reasoning, @captured_at)`,
+  ).run({
+    repo_root: repoRoot,
+    provider: snap.provider,
+    provider_name: snap.providerName,
+    model_id: snap.modelId,
+    model_name: snap.modelName,
+    input_rate: snap.inputRate,
+    output_rate: snap.outputRate,
+    context_window: snap.contextWindow,
+    max_tokens: snap.maxTokens,
+    reasoning: snap.reasoning ? 1 : 0,
+    captured_at: Date.now(),
+  });
+}
+
+/** Most recent model/provider snapshot for a repo, or undefined. */
+export function latestModelSnapshot(stateDir: string = getStateDir()): ModelSnapshot | undefined {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare(
+      `SELECT * FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`,
+    )
+    .get() as
+    | {
+        provider: string;
+        provider_name: string | null;
+        model_id: string;
+        model_name: string | null;
+        input_rate: number;
+        output_rate: number;
+        context_window: number;
+        max_tokens: number;
+        reasoning: number;
+        captured_at: number;
+      }
+    | undefined;
+  if (!row) return undefined;
+  return {
+    provider: row.provider,
+    providerName: row.provider_name,
+    modelId: row.model_id,
+    modelName: row.model_name,
+    inputRate: row.input_rate,
+    outputRate: row.output_rate,
+    contextWindow: row.context_window,
+    maxTokens: row.max_tokens,
+    reasoning: row.reasoning === 1,
+    capturedAt: row.captured_at,
+  };
+}

package/src/store/sqlite/raptor.ts

@@ -0,0 +1,107 @@
+/**
+ * raptor.ts — Sprint 13 RAPTOR node persistence.
+ */
+import { getStateDir, normalizeSessionId } from "../../store.js";
+import { openStore, jsonText, encodeEmbedding, decodeEmbedding } from "./utils.js";
+
+export interface StoredRaptorNode {
+  id: string;
+  sessionId: string;
+  level: number;
+  parentId: string | null;
+  children: string[];
+  summary: string;
+  embedding: number[];
+  qualityMarker: string;
+  tokenEstimate: number;
+  /** S25: epoch ms when the tree containing this node was built. */
+  builtAt: number;
+}
+
+/** Persist a single RAPTOR node (upsert by (session_id, id)). */
+export function upsertRaptorNode(node: StoredRaptorNode, stateDir: string = getStateDir()): void {
+  const db = openStore(stateDir);
+  db.prepare(
+    `INSERT INTO raptor_nodes(id, session_id, level, parent_id, children, summary, embedding_blob, quality_marker, token_estimate, built_at)
+     VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(session_id, id) DO UPDATE SET
+       level=excluded.level, parent_id=excluded.parent_id, children=excluded.children,
+       summary=excluded.summary, embedding_blob=excluded.embedding_blob,
+       quality_marker=excluded.quality_marker, token_estimate=excluded.token_estimate,
+       built_at=excluded.built_at`,
+  ).run(
+    node.id,
+    node.sessionId,
+    node.level,
+    node.parentId,
+    jsonText(node.children),
+    node.summary,
+    encodeEmbedding(node.embedding),
+    node.qualityMarker,
+    node.tokenEstimate,
+    node.builtAt,
+  );
+}
+
+/** Persist an entire built RAPTOR tree for a session (shadow or live). */
+export function saveRaptorTree(
+  sessionId: string,
+  tree: {
+    nodes: Map<string, {
+      id: string;
+      level: number;
+      parentId: string | null;
+      children: string[];
+      summary: string;
+      embedding: number[];
+      qualityMarker: string;
+      tokenEstimate: number;
+    }>
+  },
+  builtAt: number,
+  stateDir: string = getStateDir(),
+): void {
+  for (const node of tree.nodes.values()) {
+    upsertRaptorNode(
+      {
+        id: node.id,
+        sessionId,
+        level: node.level,
+        parentId: node.parentId,
+        children: node.children,
+        summary: node.summary,
+        embedding: node.embedding,
+        qualityMarker: node.qualityMarker,
+        tokenEstimate: node.tokenEstimate,
+        builtAt,
+      },
+      stateDir,
+    );
+  }
+}
+
+/** Load all RAPTOR nodes for a session. */
+export function listRaptorNodes(sessionId: string, stateDir: string = getStateDir()): StoredRaptorNode[] {
+  const db = openStore(stateDir);
+  const rows = db
+    .prepare("SELECT * FROM raptor_nodes WHERE session_id = ? ORDER BY level ASC, id ASC")
+    .all(normalizeSessionId(sessionId)) as any[];
+  return rows.map((row) => ({
+    id: row.id,
+    sessionId: row.session_id,
+    level: row.level,
+    parentId: row.parent_id ?? null,
+    children: row.children ? JSON.parse(row.children) : [],
+    summary: row.summary ?? "",
+    embedding: decodeEmbedding(row.embedding_blob),
+    qualityMarker: row.quality_marker ?? "low",
+    tokenEstimate: row.token_estimate ?? 0,
+    builtAt: Number(row.built_at ?? 0),
+  }));
+}
+
+/** Delete all RAPTOR nodes for a session (rollback/cleanup). */
+export function clearRaptorNodes(sessionId: string, stateDir: string = getStateDir()): void {
+  const db = openStore(stateDir);
+  db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
+}