pi-mega-compact 0.11.5 → 0.11.6

package/dist/src/store/sqlite/cache-stripes.js

@@ -0,0 +1,133 @@
+/**
+ * src/store/sqlite/cache-stripes.ts — Cache stripe distribution reader (A3, PLAN_V2 Phase 4).
+ *
+ * Reads from the cache_stripes table (schema.ts) to produce stripe distribution
+ * stats, stability aggregates, and a composite health score. Pi-agnostic.
+ *
+ * PREVENT-002: all queries use parameterized SQL.
+ */
+import { getStateDir } from "../../store.js";
+import { openStore } from "./utils.js";
+/** Human-readable labels for each stripe tier (0=permanent .. 4=volatile). */
+const STRIPE_LABELS = {
+    0: "Permanent",
+    1: "Epoch",
+    2: "Topic",
+    3: "Thread",
+    4: "Volatile",
+};
+/** Weight per stripe for health scoring (higher = more stable = better). */
+const STRIPE_HEALTH_WEIGHTS = {
+    0: 1.0,
+    1: 0.8,
+    2: 0.5,
+    3: 0.25,
+    4: 0.0,
+};
+/**
+ * Read cache stripe distribution from the database.
+ *
+ * Returns aggregated per-stripe buckets with stability stats and a composite
+ * health score. Returns empty buckets (all zeros) when the table is empty,
+ * so the dashboard always has a valid response.
+ */
+export function readCacheStripes(stateDir = getStateDir()) {
+    const db = openStore(stateDir);
+    // ── Fetch per-stripe aggregates ──
+    const rows = db
+        .prepare(`SELECT
+         stripe,
+         COUNT(*)                                         AS count,
+         ROUND(AVG(stability), 4)                         AS avgStability,
+         ROUND(MIN(stability), 4)                         AS minStability,
+         ROUND(MAX(stability), 4)                         AS maxStability
+       FROM cache_stripes
+       GROUP BY stripe
+       ORDER BY stripe ASC`)
+        .all();
+    // ── Latest epoch ──
+    const epochRow = db
+        .prepare(`SELECT epoch_id, MAX(assigned_at) AS last_refresh
+       FROM cache_stripes
+       WHERE epoch_id IS NOT NULL`)
+        .get();
+    // ── Total chunk count ──
+    const totalRow = db
+        .prepare("SELECT COUNT(*) AS cnt FROM cache_stripes")
+        .get();
+    const totalChunks = totalRow?.cnt ?? 0;
+    // ── Build bucket array (always 5 entries, stripe 0–4) ──
+    const rowMap = new Map();
+    for (const r of rows)
+        rowMap.set(r.stripe, r);
+    const buckets = [];
+    for (let s = 0; s <= 4; s++) {
+        const r = rowMap.get(s);
+        buckets.push({
+            stripe: s,
+            label: STRIPE_LABELS[s] ?? `Stripe ${s}`,
+            count: r?.count ?? 0,
+            avgStability: r?.avgStability ?? 0,
+            minStability: r?.minStability ?? 0,
+            maxStability: r?.maxStability ?? 0,
+        });
+    }
+    // ── Compute health score ──
+    const health = computeCacheHealth(buckets, totalChunks, stateDir);
+    return {
+        buckets,
+        health,
+        epochId: epochRow?.epoch_id ?? null,
+        lastRefreshAt: epochRow?.last_refresh ?? 0,
+        totalChunks,
+    };
+}
+/**
+ * Compute cache health from bucket distribution.
+ *
+ * Score = weighted sum of stripe fractions (normalised 0..1).
+ * Dominant tier = fraction of chunks in the top 2 stripes (0+1).
+ * Label: >= 0.8 = good, >= 0.6 = fair, >= 0.4 = degraded, else poor.
+ */
+function computeCacheHealth(buckets, total, stateDir) {
+    if (total === 0) {
+        return { score: 1, label: "good", dominantTier: 1, churnRate: 0 };
+    }
+    let weightedSum = 0;
+    let dominantCount = 0;
+    for (const b of buckets) {
+        const w = STRIPE_HEALTH_WEIGHTS[b.stripe] ?? 0;
+        weightedSum += (b.count / total) * w;
+        if (b.stripe <= 1)
+            dominantCount += b.count;
+    }
+    const dominantTier = dominantCount / total;
+    // ── Churn rate: open a new DB connection for the churn query ──
+    let churnRate = 0;
+    try {
+        const churnDb = openStore(stateDir);
+        const churnRow = churnDb
+            .prepare(`WITH
+          ranked AS (
+            SELECT chunk_id, epoch_id,
+                   ROW_NUMBER() OVER (PARTITION BY chunk_id ORDER BY assigned_at DESC) AS rn
+            FROM cache_stripes
+          ),
+          cur AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 1),
+          prev AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 2)
+        SELECT
+          COUNT(*) AS total_changed
+        FROM cur c
+        JOIN prev p ON c.chunk_id = p.chunk_id AND c.epoch_id != p.epoch_id`)
+            .get();
+        if (churnRow && total > 0) {
+            churnRate = Math.min(1, (churnRow.total_changed ?? 0) / total);
+        }
+    }
+    catch {
+        // churn unavailable — keep 0
+    }
+    const score = Math.round(weightedSum * 100) / 100;
+    const label = score >= 0.8 ? "good" : score >= 0.6 ? "fair" : score >= 0.4 ? "degraded" : "poor";
+    return { score, label, dominantTier, churnRate };
+}

package/dist/src/store/sqlite/fts5-search.js

@@ -0,0 +1,68 @@
+/**
+ * fts5-search.ts — FTS5 trigram search helper (L1 tier in the S44 tiered router).
+ *
+ * Runs BM25-ranked trigram search against the existing `context_chunks_trgm`
+ * virtual table (pg_trgm-equivalent, created in schema.ts). All queries are
+ * parameterized (PREVENT-002). Pi-agnostic — pure SQL over a node:sqlite
+ * DatabaseSync handle.
+ *
+ * The virtual table stores (id UNINDEXED, normalized_text) with tokenize='trigram'.
+ * FTS5 trigram queries tokenize the user query into overlapping 3-grams at query
+ * time automatically — no manual n-gram splitting needed. The BM25 rank from
+ * fts5 ranks results by trigram-overlap density.
+ */
+/**
+ * Search the context_chunks_trgm FTS5 table for checkpoints whose
+ * `normalized_text` matches `query` via trigram similarity.
+ *
+ * @param query  Raw search text (FTS5 trigram tokenizer handles n-gram splitting
+ *               automatically; input is NOT SQL-concatenated — bound as ? param).
+ * @param reader A sync-reader handle against the SQLite store.
+ * @param limit  Max results to return (default 10).
+ * @returns BM25-ranked hits, highest score first.
+ */
+export function fts5Search(query, reader, limit = 10) {
+    const sql = `
+    SELECT id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
+    FROM context_chunks_trgm
+    WHERE context_chunks_trgm MATCH ?
+    ORDER BY score
+    LIMIT ?
+  `;
+    const rows = reader
+        .prepare(sql)
+        .all(query, limit);
+    return rows.map((r) => ({ id: r.id, score: r.score }));
+}
+/**
+ * Search with an optional session_id filter for scoped FTS5 recall.
+ *
+ * JOINs the FTS5 virtual table against the real `context_chunks` table on `id`
+ * so we can filter by `session_id` while still using FTS5 MATCH. The vtab
+ * stores `id UNINDEXED` (it's a row-key passthrough, not searchable text), so
+ * the join is safe and efficient via the `idx_chunks_pk` index.
+ *
+ * @param query     Raw search text.
+ * @param reader    Sync-reader handle against the SQLite store.
+ * @param sessionId Scope to a specific session (omit for cross-session search).
+ * @param limit     Max results (default 10).
+ */
+export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
+    if (sessionId) {
+        const sql = `
+      SELECT t.id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
+      FROM context_chunks_trgm AS t
+      INNER JOIN context_chunks AS c ON c.id = t.id
+      WHERE t.context_chunks_trgm MATCH ?
+        AND c.session_id = ?
+      ORDER BY score
+      LIMIT ?
+    `;
+        const rows = reader
+            .prepare(sql)
+            .all(query, sessionId, limit);
+        return rows.map((r) => ({ id: r.id, score: r.score }));
+    }
+    // No session filter — plain FTS5 search across all sessions.
+    return fts5Search(query, reader, limit);
+}

package/dist/src/store/sqlite/perf-samples.js

@@ -28,6 +28,7 @@ export const PERF_KINDS = [
     "cpu_sys_ms",
     "db_recompute_ms",
     "disk_write_ms",
+    "cache_health",
 ];
 function isPerfKind(k) {
     return PERF_KINDS.includes(k);

package/dist/src/store/sqlite/schema.js

@@ -1,5 +1,5 @@
 import { ACHIEVEMENT_DEFS } from "../../game/scoring.js";
-const SCHEMA_VERSION = 2;
+const SCHEMA_VERSION = 4;
 /**
  * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
  * exist. Idempotent: checks PRAGMA table_info first, so it is safe to run on
@@ -353,6 +353,65 @@ export function initSchema(db) {
     );
     CREATE INDEX IF NOT EXISTS idx_perf_samples_ts ON perf_samples(ts);
     CREATE INDEX IF NOT EXISTS idx_perf_samples_kind_ts ON perf_samples(kind, ts);
+
+    -- A1 PLAN_V2 Phase 2: Message Separation tables (SCHEMA_VERSION 3).
+    -- Stable conversation thread: user + assistant turns ONLY. Tool results are
+    -- stored in tool_results to keep the conversation thread cache-predictable.
+    -- Parameterized queries only (PREVENT-002). CREATE TABLE IF NOT EXISTS is
+    -- a no-op on re-open — no migration needed.
+    CREATE TABLE IF NOT EXISTS conversation_thread (
+      id              INTEGER PRIMARY KEY AUTOINCREMENT,
+      conversation_id TEXT NOT NULL,
+      role            TEXT NOT NULL CHECK(role IN ('user','assistant')),
+      content         TEXT NOT NULL,
+      turn_index      INTEGER NOT NULL,
+      timestamp       INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_conv_thread_conv_turn
+      ON conversation_thread(conversation_id, turn_index);
+
+    -- Volatile tool results: appended at END so they never disrupt the
+    -- conversation thread prefix (the prompt-cache-friendly prefix).
+    CREATE TABLE IF NOT EXISTS tool_results (
+      id              INTEGER PRIMARY KEY AUTOINCREMENT,
+      conversation_id TEXT NOT NULL,
+      tool_call_id    TEXT NOT NULL,
+      tool_result     TEXT NOT NULL,
+      turn_index      INTEGER NOT NULL,
+      timestamp       INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_tool_results_conv_turn
+      ON tool_results(conversation_id, turn_index);
+
+    -- A2 PLAN_V2 Phase 3: Vector-Aware Cache Striping (SCHEMA_VERSION 4).
+    -- Cache stripe assignments: maps each context chunk to a cache stability
+    -- tier (stripe). Layer 0=permanent, 1=epoch, 2=topic, 3=thread, 4=volatile.
+    -- stability is the composite score (0.0-1.0) from computeStabilityScore.
+    -- Primary key is (chunk_id, epoch_id) so the same chunk can be reassigned
+    -- across epochs without conflict. See cache-stripe.ts for scoring details.
+    -- Parameterized queries only (PREVENT-002). IF NOT EXISTS is a no-op on re-open.
+    CREATE TABLE IF NOT EXISTS cache_stripes (
+      chunk_id      TEXT NOT NULL,
+      stripe        INTEGER NOT NULL,
+      stability     REAL NOT NULL,
+      assigned_at   INTEGER NOT NULL,
+      epoch_id      TEXT,
+      PRIMARY KEY (chunk_id, epoch_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_cs_stripe_stability
+      ON cache_stripes(stripe, stability DESC);
+
+    -- Embedding computation cache: avoids re-embedding content every time
+    -- stability scores are recomputed across epochs. Keyed by content_hash
+    -- (SHA-256 64-hex-char string). Embeddings are serialized via
+    -- Float64Array-to-Buffer to match the embedding_blob convention.
+    CREATE TABLE IF NOT EXISTS embedding_cache (
+      content_hash  TEXT PRIMARY KEY,
+      embedding     BLOB NOT NULL,
+      computed_at   INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_emb_cache_computed
+      ON embedding_cache(computed_at DESC);
   `);
     // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
     // pre-existing table, so new columns added to context_chunks after a store was

package/dist/src/store/sqlite.js

@@ -24,4 +24,5 @@ export * from "./sqlite/game-state.js";
 export * from "./sqlite/game-scores.js";
 export * from "./sqlite/game-achievements.js";
 export * from "./sqlite/perf-samples.js";
+export * from "./sqlite/cache-stripes.js";
 export * from "./sqlite/turns.js";

package/dist/src/store/vectorIndex.js

@@ -195,10 +195,15 @@ export async function upsertEmbedding(repoId, sessionId, checkpointId, embedding
         if (!pg)
             return;
         const lit = toVectorLiteral(embedding);
-        await pg.query(`INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
-       VALUES ($1, $2, $3, $4::vector)
-       ON CONFLICT (repo_id, session_id, checkpoint_id)
-       DO UPDATE SET embedding = EXCLUDED.embedding;`, [repoId, sessionId, checkpointId, lit]);
+        const upserted = await withOpenTimeout(pg.query(`INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
+         VALUES ($1, $2, $3, $4::vector)
+         ON CONFLICT (repo_id, session_id, checkpoint_id)
+         DO UPDATE SET embedding = EXCLUDED.embedding;`, [repoId, sessionId, checkpointId, lit]), (reason) => {
+            disabled = true;
+            logWarn(`upsert query timed out after ${queryTimeoutMs()}ms: ${reason}`);
+        }, queryTimeoutMs());
+        if (upserted === undefined)
+            return; // timed out — non-fatal
     }
     catch (err) {
         disabled = true;
@@ -227,7 +232,12 @@ export async function searchAsync(query, opts = {}) {
             params.push(repoId);
         }
         sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
-        const res = await pg.query(sql, params);
+        const res = await withOpenTimeout(pg.query(sql, params), (reason) => {
+            disabled = true;
+            logWarn(`search query timed out after ${queryTimeoutMs()}ms: ${reason}`);
+        }, queryTimeoutMs());
+        if (!res)
+            return []; // query timed out — degrade to empty (non-fatal)
         return res.rows.map((r) => ({
             repoId: r.repo_id,
             sessionId: r.session_id,
@@ -252,6 +262,20 @@ function closeTimeoutMs() {
     }
     return DEFAULT_CLOSE_TIMEOUT_MS;
 }
+/** Default cap for a PGlite search query. A WASM query that never resolves would
+ *  otherwise hang the calling turn (and node --test) forever — the catch below
+ *  only handles thrown errors, not a stalled promise. 0 disables the guard
+ *  (preserve original unbounded behavior). Mirrors the open/close timeouts. */
+export const DEFAULT_QUERY_TIMEOUT_MS = 10_000;
+function queryTimeoutMs() {
+    const raw = process.env.MEGACOMPACT_PGLITE_QUERY_TIMEOUT_MS;
+    if (raw !== undefined) {
+        const n = Number(raw);
+        if (Number.isFinite(n) && n >= 0)
+            return n;
+    }
+    return DEFAULT_QUERY_TIMEOUT_MS;
+}
 /**
  * Race a promise against a timeout. If the inner promise never settles, the
  * timer fires, resolves with `undefined`, and the caller proceeds. This mirrors

package/dist/src/tiered-router/types.js

@@ -0,0 +1,18 @@
+/**
+ * tiered-router/types.ts — S44 tiered router types, interfaces, and defaults.
+ *
+ * Shell file: pure data/type definitions. Zero runtime dependencies on store
+ * internals. Imported by tieredRouter.ts and any consumer that needs the
+ * RecallResult / TieredRouterMetrics shapes.
+ */
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+/** Default LRU cache size. */
+export const DEFAULT_CACHE_SIZE = 256;
+/** Default latency budgets per tier (ms). */
+export const DEFAULT_BUDGET_L0_MS = 1;
+export const DEFAULT_BUDGET_L1_MS = 50;
+export const DEFAULT_BUDGET_L2_MS = 500;
+/** Default cadence for tiered-metrics logging (every N queries). */
+export const DEFAULT_LOG_CADENCE = 100;