pi-mega-compact 0.11.5 → 0.11.7

package/src/store/sqlite/cache-stripes.ts

@@ -0,0 +1,196 @@
+/**
+ * src/store/sqlite/cache-stripes.ts — Cache stripe distribution reader (A3, PLAN_V2 Phase 4).
+ *
+ * Reads from the cache_stripes table (schema.ts) to produce stripe distribution
+ * stats, stability aggregates, and a composite health score. Pi-agnostic.
+ *
+ * PREVENT-002: all queries use parameterized SQL.
+ */
+import { getStateDir } from "../../store.js";
+import { openStore } from "./utils.js";
+
+// ---------------------------------------------------------------------------
+// Types (mirrors api-contracts/cache-stripes.ts but as a plain domain type).
+// ---------------------------------------------------------------------------
+
+export interface StripeBucket {
+  readonly stripe: number;
+  readonly label: string;
+  readonly count: number;
+  readonly avgStability: number;
+  readonly minStability: number;
+  readonly maxStability: number;
+}
+
+export interface CacheHealthScore {
+  readonly score: number;
+  readonly label: string;
+  readonly dominantTier: number;
+  readonly churnRate: number;
+}
+
+export interface CacheStripesResult {
+  readonly buckets: StripeBucket[];
+  readonly health: CacheHealthScore;
+  readonly epochId: string | null;
+  readonly lastRefreshAt: number;
+  readonly totalChunks: number;
+}
+
+/** Human-readable labels for each stripe tier (0=permanent .. 4=volatile). */
+const STRIPE_LABELS: Record<number, string> = {
+  0: "Permanent",
+  1: "Epoch",
+  2: "Topic",
+  3: "Thread",
+  4: "Volatile",
+};
+
+/** Weight per stripe for health scoring (higher = more stable = better). */
+const STRIPE_HEALTH_WEIGHTS: Record<number, number> = {
+  0: 1.0,
+  1: 0.8,
+  2: 0.5,
+  3: 0.25,
+  4: 0.0,
+};
+
+/**
+ * Read cache stripe distribution from the database.
+ *
+ * Returns aggregated per-stripe buckets with stability stats and a composite
+ * health score. Returns empty buckets (all zeros) when the table is empty,
+ * so the dashboard always has a valid response.
+ */
+export function readCacheStripes(
+  stateDir: string = getStateDir(),
+): CacheStripesResult {
+  const db = openStore(stateDir);
+
+  // ── Fetch per-stripe aggregates ──
+  const rows = db
+    .prepare(
+      `SELECT
+         stripe,
+         COUNT(*)                                         AS count,
+         ROUND(AVG(stability), 4)                         AS avgStability,
+         ROUND(MIN(stability), 4)                         AS minStability,
+         ROUND(MAX(stability), 4)                         AS maxStability
+       FROM cache_stripes
+       GROUP BY stripe
+       ORDER BY stripe ASC`,
+    )
+    .all() as Array<{
+    stripe: number;
+    count: number;
+    avgStability: number;
+    minStability: number;
+    maxStability: number;
+  }>;
+
+  // ── Latest epoch ──
+  const epochRow = db
+    .prepare(
+      `SELECT epoch_id, MAX(assigned_at) AS last_refresh
+       FROM cache_stripes
+       WHERE epoch_id IS NOT NULL`,
+    )
+    .get() as { epoch_id: string; last_refresh: number } | undefined;
+
+  // ── Total chunk count ──
+  const totalRow = db
+    .prepare("SELECT COUNT(*) AS cnt FROM cache_stripes")
+    .get() as { cnt: number };
+
+  const totalChunks = totalRow?.cnt ?? 0;
+
+  // ── Build bucket array (always 5 entries, stripe 0–4) ──
+  const rowMap = new Map<number, (typeof rows)[number]>();
+  for (const r of rows) rowMap.set(r.stripe, r);
+
+  const buckets: StripeBucket[] = [];
+  for (let s = 0; s <= 4; s++) {
+    const r = rowMap.get(s);
+    buckets.push({
+      stripe: s,
+      label: STRIPE_LABELS[s] ?? `Stripe ${s}`,
+      count: r?.count ?? 0,
+      avgStability: r?.avgStability ?? 0,
+      minStability: r?.minStability ?? 0,
+      maxStability: r?.maxStability ?? 0,
+    });
+  }
+
+  // ── Compute health score ──
+  const health = computeCacheHealth(buckets, totalChunks, stateDir);
+
+  return {
+    buckets,
+    health,
+    epochId: epochRow?.epoch_id ?? null,
+    lastRefreshAt: epochRow?.last_refresh ?? 0,
+    totalChunks,
+  };
+}
+
+/**
+ * Compute cache health from bucket distribution.
+ *
+ * Score = weighted sum of stripe fractions (normalised 0..1).
+ * Dominant tier = fraction of chunks in the top 2 stripes (0+1).
+ * Label: >= 0.8 = good, >= 0.6 = fair, >= 0.4 = degraded, else poor.
+ */
+function computeCacheHealth(
+  buckets: StripeBucket[],
+  total: number,
+  stateDir: string,
+): CacheHealthScore {
+  if (total === 0) {
+    return { score: 1, label: "good", dominantTier: 1, churnRate: 0 };
+  }
+
+  let weightedSum = 0;
+  let dominantCount = 0;
+
+  for (const b of buckets) {
+    const w = STRIPE_HEALTH_WEIGHTS[b.stripe] ?? 0;
+    weightedSum += (b.count / total) * w;
+    if (b.stripe <= 1) dominantCount += b.count;
+  }
+
+  const dominantTier = dominantCount / total;
+
+  // ── Churn rate: open a new DB connection for the churn query ──
+  let churnRate = 0;
+  try {
+    const churnDb = openStore(stateDir);
+    const churnRow = churnDb
+      .prepare(
+        `WITH
+          ranked AS (
+            SELECT chunk_id, epoch_id,
+                   ROW_NUMBER() OVER (PARTITION BY chunk_id ORDER BY assigned_at DESC) AS rn
+            FROM cache_stripes
+          ),
+          cur AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 1),
+          prev AS (SELECT chunk_id, epoch_id FROM ranked WHERE rn = 2)
+        SELECT
+          COUNT(*) AS total_changed
+        FROM cur c
+        JOIN prev p ON c.chunk_id = p.chunk_id AND c.epoch_id != p.epoch_id`,
+      )
+      .get() as { total_changed: number } | undefined;
+
+    if (churnRow && total > 0) {
+      churnRate = Math.min(1, (churnRow.total_changed ?? 0) / total);
+    }
+  } catch {
+    // churn unavailable — keep 0
+  }
+
+  const score = Math.round(weightedSum * 100) / 100;
+  const label =
+    score >= 0.8 ? "good" : score >= 0.6 ? "fair" : score >= 0.4 ? "degraded" : "poor";
+
+  return { score, label, dominantTier, churnRate };
+}

package/src/store/sqlite/fts5-search.ts

@@ -0,0 +1,90 @@
+/**
+ * fts5-search.ts — FTS5 trigram search helper (L1 tier in the S44 tiered router).
+ *
+ * Runs BM25-ranked trigram search against the existing `context_chunks_trgm`
+ * virtual table (pg_trgm-equivalent, created in schema.ts). All queries are
+ * parameterized (PREVENT-002). Pi-agnostic — pure SQL over a node:sqlite
+ * DatabaseSync handle.
+ *
+ * The virtual table stores (id UNINDEXED, normalized_text) with tokenize='trigram'.
+ * FTS5 trigram queries tokenize the user query into overlapping 3-grams at query
+ * time automatically — no manual n-gram splitting needed. The BM25 rank from
+ * fts5 ranks results by trigram-overlap density.
+ */
+
+import type { DatabaseSync } from "node:sqlite";
+
+export interface Fts5Hit {
+	/** checkpoint id (chkpt_001 etc.) from the context_chunks_trgm row. */
+	id: string;
+	/** BM25 relevance score from FTS5 (higher = better match). */
+	score: number;
+}
+
+/**
+ * Search the context_chunks_trgm FTS5 table for checkpoints whose
+ * `normalized_text` matches `query` via trigram similarity.
+ *
+ * @param query  Raw search text (FTS5 trigram tokenizer handles n-gram splitting
+ *               automatically; input is NOT SQL-concatenated — bound as ? param).
+ * @param reader A sync-reader handle against the SQLite store.
+ * @param limit  Max results to return (default 10).
+ * @returns BM25-ranked hits, highest score first.
+ */
+export function fts5Search(
+	query: string,
+	reader: DatabaseSync,
+	limit = 10,
+): Fts5Hit[] {
+	const sql = `
+    SELECT id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
+    FROM context_chunks_trgm
+    WHERE context_chunks_trgm MATCH ?
+    ORDER BY score
+    LIMIT ?
+  `;
+	const rows = reader
+		.prepare(sql)
+		.all(query, limit) as Array<{ id: string; score: number }>;
+
+	return rows.map((r) => ({ id: r.id, score: r.score }));
+}
+
+/**
+ * Search with an optional session_id filter for scoped FTS5 recall.
+ *
+ * JOINs the FTS5 virtual table against the real `context_chunks` table on `id`
+ * so we can filter by `session_id` while still using FTS5 MATCH. The vtab
+ * stores `id UNINDEXED` (it's a row-key passthrough, not searchable text), so
+ * the join is safe and efficient via the `idx_chunks_pk` index.
+ *
+ * @param query     Raw search text.
+ * @param reader    Sync-reader handle against the SQLite store.
+ * @param sessionId Scope to a specific session (omit for cross-session search).
+ * @param limit     Max results (default 10).
+ */
+export function fts5SearchScoped(
+	query: string,
+	reader: DatabaseSync,
+	sessionId: string | undefined,
+	limit = 10,
+): Fts5Hit[] {
+	if (sessionId) {
+		const sql = `
+      SELECT t.id, bm25(context_chunks_trgm, 0.0, 1.0) AS score
+      FROM context_chunks_trgm AS t
+      INNER JOIN context_chunks AS c ON c.id = t.id
+      WHERE t.context_chunks_trgm MATCH ?
+        AND c.session_id = ?
+      ORDER BY score
+      LIMIT ?
+    `;
+		const rows = reader
+			.prepare(sql)
+			.all(query, sessionId, limit) as Array<{ id: string; score: number }>;
+		return rows.map((r) => ({ id: r.id, score: r.score }));
+	}
+
+	// No session filter — plain FTS5 search across all sessions.
+	return fts5Search(query, reader, limit);
+}

package/src/store/sqlite/perf-samples.ts

@@ -29,7 +29,8 @@ export type PerfKind =
 	| "cpu_user_ms"
 	| "cpu_sys_ms"
 	| "db_recompute_ms"
-	| "disk_write_ms";
+	| "disk_write_ms"
+	| "cache_health";
 
 /** Allow-list of valid perf sample kinds (mirrors the table's domain). */
 export const PERF_KINDS: readonly PerfKind[] = [
@@ -44,6 +45,7 @@ export const PERF_KINDS: readonly PerfKind[] = [
 	"cpu_sys_ms",
 	"db_recompute_ms",
 	"disk_write_ms",
+	"cache_health",
 ];
 
 /** A single perf sample row (as stored + returned). */

package/src/store/sqlite/schema.ts

@@ -4,7 +4,7 @@
 import type { DatabaseSync } from "node:sqlite";
 import { ACHIEVEMENT_DEFS } from "../../game/scoring.js";
 
-const SCHEMA_VERSION = 2;
+const SCHEMA_VERSION = 4;
 
 /**
  * Add `column` (with `decl`, e.g. "INTEGER") to `table` if it does not already
@@ -366,6 +366,65 @@ export function initSchema(db: DatabaseSync): void {
     );
     CREATE INDEX IF NOT EXISTS idx_perf_samples_ts ON perf_samples(ts);
     CREATE INDEX IF NOT EXISTS idx_perf_samples_kind_ts ON perf_samples(kind, ts);
+
+    -- A1 PLAN_V2 Phase 2: Message Separation tables (SCHEMA_VERSION 3).
+    -- Stable conversation thread: user + assistant turns ONLY. Tool results are
+    -- stored in tool_results to keep the conversation thread cache-predictable.
+    -- Parameterized queries only (PREVENT-002). CREATE TABLE IF NOT EXISTS is
+    -- a no-op on re-open — no migration needed.
+    CREATE TABLE IF NOT EXISTS conversation_thread (
+      id              INTEGER PRIMARY KEY AUTOINCREMENT,
+      conversation_id TEXT NOT NULL,
+      role            TEXT NOT NULL CHECK(role IN ('user','assistant')),
+      content         TEXT NOT NULL,
+      turn_index      INTEGER NOT NULL,
+      timestamp       INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_conv_thread_conv_turn
+      ON conversation_thread(conversation_id, turn_index);
+
+    -- Volatile tool results: appended at END so they never disrupt the
+    -- conversation thread prefix (the prompt-cache-friendly prefix).
+    CREATE TABLE IF NOT EXISTS tool_results (
+      id              INTEGER PRIMARY KEY AUTOINCREMENT,
+      conversation_id TEXT NOT NULL,
+      tool_call_id    TEXT NOT NULL,
+      tool_result     TEXT NOT NULL,
+      turn_index      INTEGER NOT NULL,
+      timestamp       INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_tool_results_conv_turn
+      ON tool_results(conversation_id, turn_index);
+
+    -- A2 PLAN_V2 Phase 3: Vector-Aware Cache Striping (SCHEMA_VERSION 4).
+    -- Cache stripe assignments: maps each context chunk to a cache stability
+    -- tier (stripe). Layer 0=permanent, 1=epoch, 2=topic, 3=thread, 4=volatile.
+    -- stability is the composite score (0.0-1.0) from computeStabilityScore.
+    -- Primary key is (chunk_id, epoch_id) so the same chunk can be reassigned
+    -- across epochs without conflict. See cache-stripe.ts for scoring details.
+    -- Parameterized queries only (PREVENT-002). IF NOT EXISTS is a no-op on re-open.
+    CREATE TABLE IF NOT EXISTS cache_stripes (
+      chunk_id      TEXT NOT NULL,
+      stripe        INTEGER NOT NULL,
+      stability     REAL NOT NULL,
+      assigned_at   INTEGER NOT NULL,
+      epoch_id      TEXT,
+      PRIMARY KEY (chunk_id, epoch_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_cs_stripe_stability
+      ON cache_stripes(stripe, stability DESC);
+
+    -- Embedding computation cache: avoids re-embedding content every time
+    -- stability scores are recomputed across epochs. Keyed by content_hash
+    -- (SHA-256 64-hex-char string). Embeddings are serialized via
+    -- Float64Array-to-Buffer to match the embedding_blob convention.
+    CREATE TABLE IF NOT EXISTS embedding_cache (
+      content_hash  TEXT PRIMARY KEY,
+      embedding     BLOB NOT NULL,
+      computed_at   INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_emb_cache_computed
+      ON embedding_cache(computed_at DESC);
   `);
 	// Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
 	// pre-existing table, so new columns added to context_chunks after a store was

package/src/store/sqlite.ts

@@ -24,4 +24,5 @@ export * from "./sqlite/game-state.js";
 export * from "./sqlite/game-scores.js";
 export * from "./sqlite/game-achievements.js";
 export * from "./sqlite/perf-samples.js";
+export * from "./sqlite/cache-stripes.js";
 export * from "./sqlite/turns.js";

package/src/store/vectorIndex.ts

@@ -230,13 +230,21 @@ export async function upsertEmbedding(
     const pg = await initVectorIndex();
     if (!pg) return;
     const lit = toVectorLiteral(embedding);
-    await pg.query(
-      `INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
-       VALUES ($1, $2, $3, $4::vector)
-       ON CONFLICT (repo_id, session_id, checkpoint_id)
-       DO UPDATE SET embedding = EXCLUDED.embedding;`,
-      [repoId, sessionId, checkpointId, lit],
+    const upserted = await withOpenTimeout(
+      pg.query(
+        `INSERT INTO vector_index (repo_id, session_id, checkpoint_id, embedding)
+         VALUES ($1, $2, $3, $4::vector)
+         ON CONFLICT (repo_id, session_id, checkpoint_id)
+         DO UPDATE SET embedding = EXCLUDED.embedding;`,
+        [repoId, sessionId, checkpointId, lit],
+      ),
+      (reason) => {
+        disabled = true;
+        logWarn(`upsert query timed out after ${queryTimeoutMs()}ms: ${reason}`);
+      },
+      queryTimeoutMs(),
     );
+    if (upserted === undefined) return; // timed out — non-fatal
   } catch (err) {
     disabled = true;
     logWarn(`upsert failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -274,7 +282,15 @@ export async function searchAsync(
       params.push(repoId);
     }
     sql += " ORDER BY embedding <=> $1::vector LIMIT $2";
-    const res = await pg.query(sql, params);
+    const res = await withOpenTimeout(
+      pg.query(sql, params),
+      (reason) => {
+        disabled = true;
+        logWarn(`search query timed out after ${queryTimeoutMs()}ms: ${reason}`);
+      },
+      queryTimeoutMs(),
+    );
+    if (!res) return []; // query timed out — degrade to empty (non-fatal)
     return res.rows.map((r: any) => ({
       repoId: r.repo_id as string,
       sessionId: r.session_id as string,
@@ -300,6 +316,21 @@ function closeTimeoutMs(): number {
   return DEFAULT_CLOSE_TIMEOUT_MS;
 }
 
+/** Default cap for a PGlite search query. A WASM query that never resolves would
+ *  otherwise hang the calling turn (and node --test) forever — the catch below
+ *  only handles thrown errors, not a stalled promise. 0 disables the guard
+ *  (preserve original unbounded behavior). Mirrors the open/close timeouts. */
+export const DEFAULT_QUERY_TIMEOUT_MS = 10_000;
+
+function queryTimeoutMs(): number {
+  const raw = process.env.MEGACOMPACT_PGLITE_QUERY_TIMEOUT_MS;
+  if (raw !== undefined) {
+    const n = Number(raw);
+    if (Number.isFinite(n) && n >= 0) return n;
+  }
+  return DEFAULT_QUERY_TIMEOUT_MS;
+}
+
 /**
  * Race a promise against a timeout. If the inner promise never settles, the
  * timer fires, resolves with `undefined`, and the caller proceeds. This mirrors

package/src/tiered-router/types.ts

@@ -0,0 +1,64 @@
+/**
+ * tiered-router/types.ts — S44 tiered router types, interfaces, and defaults.
+ *
+ * Shell file: pure data/type definitions. Zero runtime dependencies on store
+ * internals. Imported by tieredRouter.ts and any consumer that needs the
+ * RecallResult / TieredRouterMetrics shapes.
+ */
+
+import type { SearchHit } from "../vectorStore.js";
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface TieredRecallOpts {
+  /** Session ID to scope search. When absent, searches cross-session. */
+  sessionId?: string;
+  /** Max number of results to return (default 10). */
+  k?: number;
+  /** Skip LRU-cache lookup (force L1 hit). */
+  bypassCache?: boolean;
+  /**
+   * FTS5 max BM25 score for the L1 confidence gate.
+   * Read from calibration.json at construction; overridable per-call.
+   */
+  overrideFts5MaxBm25?: number;
+}
+
+export interface RecallResult {
+  /** Checkpoint search hits, ranked by relevance. */
+  hits: SearchHit[];
+  /** Which tier produced the result. */
+  tier: "L0" | "L1" | "L2";
+  /** Per-tier latency deltas in ms (contains only tiers that ran). */
+  latencyMs: { l0?: number; l1?: number; l2?: number };
+}
+
+export interface TieredRouterMetrics {
+  l0Hits: number;
+  l0Misses: number;
+  l1Hits: number;
+  l1Misses: number;
+  l2Hits: number;
+  l2Misses: number;
+  totalQueries: number;
+  avgLatencyMs: { l0: number; l1: number; l2: number };
+  fts5MaxBm25: number | null;
+  uncalibrated: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+
+/** Default LRU cache size. */
+export const DEFAULT_CACHE_SIZE = 256;
+
+/** Default latency budgets per tier (ms). */
+export const DEFAULT_BUDGET_L0_MS = 1;
+export const DEFAULT_BUDGET_L1_MS = 50;
+export const DEFAULT_BUDGET_L2_MS = 500;
+
+/** Default cadence for tiered-metrics logging (every N queries). */
+export const DEFAULT_LOG_CADENCE = 100;