pi-mega-compact 0.11.3 → 0.11.5

package/src/store/sqlite/perf-samples.ts

@@ -15,6 +15,7 @@
  */
 import { getStateDir } from "../../store.js";
 import { openStore } from "./utils.js";
+import type { PrefixBreak } from "../../prefix-break.js";
 
 /** Sample kinds recorded into perf_samples. */
 export type PerfKind =
@@ -22,6 +23,7 @@ export type PerfKind =
 	| "provider_latency_ms"
 	| "tps"
 	| "cache_hit_pct"
+	| "prefix_break"
 	| "rss_mb"
 	| "heap_mb"
 	| "cpu_user_ms"
@@ -35,6 +37,7 @@ export const PERF_KINDS: readonly PerfKind[] = [
 	"provider_latency_ms",
 	"tps",
 	"cache_hit_pct",
+	"prefix_break",
 	"rss_mb",
 	"heap_mb",
 	"cpu_user_ms",
@@ -137,30 +140,69 @@ export interface ProviderCacheLifetime {
 	firstSampleAt: string | null;
 	/** ISO timestamp of the most recent sample, or null if no samples. */
 	latestSampleAt: string | null;
+	/**
+	 * Per-model breakdown of aggregates (F4). Samples without a model tag are
+	 * omitted from this array (they remain in the flat totals above).
+	 */
+	byModel: ProviderCacheModelAgg[];
+}
+
+/** Per-model provider cache aggregate (F4). */
+export interface ProviderCacheModelAgg {
+	/** Model label (modelName || modelId from meta, or null for untagged). */
+	model: string;
+	/** Average hit rate for this model (0-100). */
+	hitPct: number;
+	/** Sum of cache-read tokens for this model. */
+	totalCacheRead: number;
+	/** Sum of cache-write tokens for this model. */
+	totalCacheWrite: number;
+	/** Number of samples for this model. */
+	sampleCount: number;
 }
 
 /**
- * Read lifetime provider prompt cache aggregates from `perf_samples`.
- *
- * Aggregates `cache_creation_input_tokens` / `cache_read_input_tokens` from
- * the `meta` JSON column of `cache_hit_pct` samples (`json_extract`,
- * available in node:sqlite ≥22.13 / SQLite ≥3.38). Returns a zeroed
- * `ProviderCacheLifetime` when no samples exist (never undefined).
+ * Extract a model label from parsed meta (modelName or modelId), or null.
  */
-export function readProviderCacheLifetime(
-	stateDir: string = getStateDir(),
+function extractModelLabel(
+	meta: Record<string, unknown>,
+): string | null {
+	if (typeof meta.modelName === "string" && meta.modelName.length > 0)
+		return meta.modelName;
+	if (typeof meta.modelId === "string" && meta.modelId.length > 0)
+		return meta.modelId;
+	return null;
+}
+
+/**
+ * Build sorted byModel array from the model aggregation map.
+ */
+function byModelFromMap(
+	map: Map<string, { sumCr: number; sumCw: number; sumInp: number; sumHitPct: number; hitCount: number; samples: number }>,
+): ProviderCacheModelAgg[] {
+	const out: ProviderCacheModelAgg[] = [];
+	for (const [model, m] of map) {
+		out.push({
+			model,
+			hitPct: m.hitCount > 0 ? m.sumHitPct / m.hitCount : 0,
+			totalCacheRead: m.sumCr,
+			totalCacheWrite: m.sumCw,
+			sampleCount: m.samples,
+		});
+	}
+	out.sort((a, b) => b.sampleCount - a.sampleCount);
+	return out;
+}
+
+/**
+ * Aggregate `cache_hit_pct` rows (ts + parsed meta) into a
+ * `ProviderCacheLifetime`. Shared by the lifetime and window readers so both
+ * use the identical aggregation formula. `meta` is parsed defensively
+ * (PREVENT-001 null-safe). Rows must be ascending by ts.
+ */
+function aggregateCacheRows(
+	rows: Array<{ ts: number; meta: string | null }>,
 ): ProviderCacheLifetime {
-	const db = openStore(stateDir);
-	const rows = db
-		.prepare(
-			`SELECT ts, meta FROM perf_samples
-       WHERE kind = ?
-       ORDER BY ts ASC`,
-		)
-		.all("cache_hit_pct") as Array<{
-		ts: number;
-		meta: string | null;
-	}>;
 	if (rows.length === 0) {
 		return {
 			sampleCount: 0,
@@ -170,6 +212,7 @@ export function readProviderCacheLifetime(
 			totalInput: 0,
 			firstSampleAt: null,
 			latestSampleAt: null,
+			byModel: [],
 		};
 	}
 	let totalCacheRead = 0;
@@ -177,6 +220,17 @@ export function readProviderCacheLifetime(
 	let totalInput = 0;
 	let sumHitPct = 0;
 	let hitPctCount = 0;
+	const modelMap = new Map<
+		string,
+		{
+			sumCr: number;
+			sumCw: number;
+			sumInp: number;
+			sumHitPct: number;
+			hitCount: number;
+			samples: number;
+		}
+	>();
 	for (const r of rows) {
 		if (r.meta != null) {
 			let meta: Record<string, unknown> | null = null;
@@ -208,6 +262,31 @@ export function readProviderCacheLifetime(
 					sumHitPct += (cr / denom) * 100;
 					hitPctCount++;
 				}
+				// Per-model grouping (F4)
+				const modelLabel = extractModelLabel(meta);
+				if (modelLabel != null) {
+					let m = modelMap.get(modelLabel);
+					if (!m) {
+						m = {
+							sumCr: 0,
+							sumCw: 0,
+							sumInp: 0,
+							sumHitPct: 0,
+							hitCount: 0,
+							samples: 0,
+						};
+						modelMap.set(modelLabel, m);
+					}
+					m.sumCr += cr;
+					m.sumCw += cw;
+					m.sumInp += inp;
+					m.samples++;
+					if (denom > 0) {
+						m.sumHitPct += (cr / denom) * 100;
+						m.hitCount++;
+					}
+				}
+				// Untagged samples are omitted from byModel
 			}
 		}
 	}
@@ -219,9 +298,62 @@ export function readProviderCacheLifetime(
 		totalInput,
 		firstSampleAt: new Date(rows[0].ts).toISOString(),
 		latestSampleAt: new Date(rows[rows.length - 1].ts).toISOString(),
+		byModel: byModelFromMap(modelMap),
 	};
 }
 
+/**
+ * Read lifetime provider prompt cache aggregates from `perf_samples`.
+ *
+ * Aggregates `cache_creation_input_tokens` / `cache_read_input_tokens` from
+ * the `meta` JSON column of `cache_hit_pct` samples (`json_extract`,
+ * available in node:sqlite >=22.13 / SQLite >=3.38). Returns a zeroed
+ * `ProviderCacheLifetime` when no samples exist (never undefined).
+ */
+export function readProviderCacheLifetime(
+	stateDir: string = getStateDir(),
+): ProviderCacheLifetime {
+	const db = openStore(stateDir);
+	const rows = db
+		.prepare(
+			`SELECT ts, meta FROM perf_samples
+	       WHERE kind = ?
+	       ORDER BY ts ASC`,
+		)
+		.all("cache_hit_pct") as Array<{
+		ts: number;
+		meta: string | null;
+	}>;
+	return aggregateCacheRows(rows);
+}
+
+/**
+ * Read provider prompt cache aggregates over a trailing window of `minutes`
+ * (samples with ts >= now - minutes*60_000). Same aggregate shape as
+ * `readProviderCacheLifetime`; the existing lifetime reader is untouched.
+ * Returns a zeroed `ProviderCacheLifetime` when no samples fall in the window.
+ */
+export function readProviderCacheWindow(
+	stateDir: string = getStateDir(),
+	minutes: number,
+): ProviderCacheLifetime {
+	const db = openStore(stateDir);
+	const sinceTs = Date.now() - minutes * 60_000;
+	const rows = db
+		.prepare(
+			`SELECT ts, meta FROM perf_samples
+	       WHERE kind = ? AND ts >= ?
+	       ORDER BY ts ASC`,
+		)
+		.all("cache_hit_pct", sinceTs) as Array<{
+		ts: number;
+		meta: string | null;
+	}>;
+	return aggregateCacheRows(rows);
+}
+
+// ... keep every function below unchanged ...
+
 /**
  * Read the most recent `cache_hit_pct` value from `perf_samples`.
  *
@@ -236,9 +368,9 @@ export function readLatestCacheHitPct(
 	const row = db
 		.prepare(
 			`SELECT value FROM perf_samples
-       WHERE kind = ?
-       ORDER BY ts DESC, id DESC
-       LIMIT 1`,
+	       WHERE kind = ?
+	       ORDER BY ts DESC, id DESC
+	       LIMIT 1`,
 		)
 		.get("cache_hit_pct") as { value: number } | undefined;
 	if (!row || typeof row.value !== "number" || !Number.isFinite(row.value))
@@ -259,3 +391,64 @@ export function readProviderCacheForRepo(
 ): ProviderCacheLifetime {
 	return readProviderCacheLifetime(stateDir);
 }
+
+// ─── Prefix-break reader (S53A) ───────────────────────────────────────────────
+
+/** Re-exported for consumers that import via perf-samples. */
+export type { PrefixBreak } from "../../prefix-break.js";
+
+/**
+ * Read `prefix_break` samples in a time window.
+ *
+ * @param stateDir  State directory for this repo.
+ * @param sinceTs   Lower bound (epoch ms). Pass 0 for no lower bound.
+ * @param untilTs   Upper bound (epoch ms). Pass 0 for no upper bound.
+ *                  Rows with ts > untilTs are excluded.
+ * @returns         Sorted ascending array of PrefixBreak rows.
+ *
+ * SQL is fully parameterized (PREVENT-002). `meta` is parsed defensively
+ * (PREVENT-001 null-safe). When no rows exist the return is an empty array.
+ */
+export function readPrefixBreaks(
+	stateDir: string = getStateDir(),
+	sinceTs: number = 0,
+	untilTs: number = 0,
+): PrefixBreak[] {
+	const db = openStore(stateDir);
+	let rows: Array<{
+		id: number;
+		ts: number;
+		value: number;
+		meta: string | null;
+	}>;
+
+	if (untilTs > 0) {
+		const sql = `SELECT id, ts, value, meta FROM perf_samples
+		   WHERE kind = ? AND ts >= ? AND ts <= ? ORDER BY ts ASC`;
+		rows = db.prepare(sql).all("prefix_break", sinceTs, untilTs) as typeof rows;
+	} else {
+		const sql = `SELECT id, ts, value, meta FROM perf_samples
+		   WHERE kind = ? AND ts >= ? ORDER BY ts ASC`;
+		rows = db.prepare(sql).all("prefix_break", sinceTs) as typeof rows;
+	}
+
+	const out: PrefixBreak[] = [];
+	for (const r of rows) {
+		let meta: unknown = null;
+		if (r.meta != null) {
+			try {
+				meta = JSON.parse(r.meta);
+			} catch {
+				meta = null;
+			}
+		}
+		out.push({
+			id: r.id,
+			ts: r.ts,
+			kind: "prefix_break",
+			value: r.value,
+			meta: meta as PrefixBreak["meta"],
+		});
+	}
+	return out;
+}

package/src/store/turns/memory-store.ts

@@ -8,7 +8,7 @@
  */
 
 import { randomBytes } from "node:crypto";
-import type {
+import { DuplicateTurnError, type
 	TurnStore,
 	TurnReader,
 	TurnWriter,
@@ -198,6 +198,15 @@ export class InMemoryTurnStore implements TurnStore {
 	// ── TurnWriter ──────────────────────────────────────────────
 
 	appendTurn(entry: TurnEntry): TurnId {
+		// Check for duplicate (conversationId, turnIndex)
+		for (const [, row] of this.turns) {
+			if (
+				row.entry.conversationId === entry.conversationId &&
+				row.entry.turnIndex === entry.turnIndex
+			) {
+				throw new DuplicateTurnError(entry.conversationId, entry.turnIndex);
+			}
+		}
 		const id = allocId();
 		const sid = normalizeSessionId(entry.sessionId);
 		const normalized = { ...entry, sessionId: sid };

package/src/store/turns/sqlite-store.ts

@@ -33,6 +33,7 @@ import type {
 	ConversationStats,
 	TurnStoreOptions,
 } from "./types.js";
+import { DuplicateTurnError } from "./types.js";
 
 // ─── Helpers ──────────────────────────────────────────────────────
 
@@ -237,23 +238,34 @@ export class SqliteTurnStore implements TurnStore {
 	appendTurn(entry: TurnEntry): TurnId {
 		const sid = normalizeSessionId(entry.sessionId);
 		this.persistSessionConv(sid, entry.conversationId);
-		this.db
-			.prepare(
-				`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
+		try {
+			this.db
+				.prepare(
+					`INSERT INTO turns (conversation_id, session_id, turn_index, role, ended_at,
                                    ctx_tokens, ctx_percent, pressure_band, model)
          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-			)
-			.run(
-				entry.conversationId,
-				sid,
-				entry.turnIndex,
-				entry.role,
-				entry.endedAt,
-				entry.ctxTokens ?? null,
-				entry.ctxPercent ?? null,
-				entry.pressureBand ?? null,
-				entry.model ?? null,
-			);
+				)
+				.run(
+					entry.conversationId,
+					sid,
+					entry.turnIndex,
+					entry.role,
+					entry.endedAt,
+					entry.ctxTokens ?? null,
+					entry.ctxPercent ?? null,
+					entry.pressureBand ?? null,
+					entry.model ?? null,
+				);
+		} catch (e) {
+			if (
+				e instanceof Error &&
+				e.message.includes("UNIQUE constraint") &&
+				(e as { code?: string }).code === "ERR_SQLITE_ERROR"
+			) {
+				throw new DuplicateTurnError(entry.conversationId, entry.turnIndex);
+			}
+			throw e;
+		}
 		const row = this.db
 			.prepare(
 				"SELECT id FROM turns WHERE conversation_id = ? AND turn_index = ?",

package/src/store/turns/types.ts

@@ -16,6 +16,27 @@
 
 // ─── Domain types ───────────────────────────────────────────────────
 
+/**
+ * Thrown when appendTurn is called with a (conversationId, turnIndex) pair
+ * that already exists. Both backends must throw this in lieu of a raw
+ * SQLite error (SqliteTurnStore) or silent success (InMemoryTurnStore).
+ */
+export class DuplicateTurnError extends Error {
+	/** The conversation that already has this turn index. */
+	readonly conversationId: ConversationId;
+	/** The duplicate turn index. */
+	readonly turnIndex: number;
+
+	constructor(conversationId: ConversationId, turnIndex: number) {
+		super(
+			`Duplicate turn: conversation "${conversationId}" already has turnIndex ${turnIndex}`,
+		);
+		this.name = "DuplicateTurnError";
+		this.conversationId = conversationId;
+		this.turnIndex = turnIndex;
+	}
+}
+
 /** Unique turn identifier (string to stay backend-agnostic). */
 export type TurnId = string;
 

package/src/store/vectorIndex.ts

@@ -288,13 +288,42 @@ export async function searchAsync(
   }
 }
 
+/** Default cap for the WASM close (PGlite can stall ~40 min on teardown). */
+export const DEFAULT_CLOSE_TIMEOUT_MS = 5_000;
+
+function closeTimeoutMs(): number {
+  const raw = process.env.MEGACOMPACT_PGLITE_CLOSE_TIMEOUT_MS;
+  if (raw) {
+    const n = Number(raw);
+    if (Number.isFinite(n) && n > 0) return n;
+  }
+  return DEFAULT_CLOSE_TIMEOUT_MS;
+}
+
+/**
+ * Race a promise against a timeout. If the inner promise never settles, the
+ * timer fires, resolves with `undefined`, and the caller proceeds. This mirrors
+ * `withOpenTimeout` (pgOpenGuard.ts) but is local to the close path.
+ */
+function withCloseTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+): Promise<T | undefined> {
+  return Promise.race([
+    promise,
+    new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), timeoutMs)),
+  ]);
+}
+
 /** Close the index (test teardown / shutdown). Safe to call when unopened. */
 export async function closeVectorIndex(): Promise<void> {
   if (db) {
-    try {
-      await db.close();
-    } catch {
-      /* ignore */
+    const closePromise = db.close().catch(() => {});
+    const result = await withCloseTimeout(closePromise, closeTimeoutMs());
+    if (result === undefined) {
+      logWarn(
+        `close timed out after ${closeTimeoutMs()}ms (WASM teardown stalled; OS will reclaim)`,
+      );
     }
   }
   db = undefined;

package/dist/extensions/conflict-scan.test.js

@@ -1,115 +0,0 @@
-/**
- * conflict-scan.test.ts — unit tests for the extension-conflict scanner.
- *
- * Fixture trees are written under a temp dir and scanned via
- * MEGACOMPACT_EXT_SCAN_DIR (which makes collectScanRoots() return that
- * single root). This covers the S24 follow-up fix:
- *
- *   1. node_modules-style code extensions (package.json + pi.extensions) are
- *      still detected by source-marker grep (regression).
- *   2. USER-LEVEL extensions installed outside npm (e.g. pi-hermes-memory)
- *      now get scanned too — previously only `node_modules` was walked, so a
- *      data-only memory store (MEMORY.md + sessions.db, no package.json)
- *      was never flagged (the 5000-char file-buffer error slipped through).
- *   3. The data-only memory-store signature is detected even with no source.
- *   4. pi-mega-compact (selfName) is always skipped.
- */
-import { test, after } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { detectConflicts, collectScanRoots } from "./conflict-scan.js";
-const base = mkdtempSync(join(tmpdir(), "mc-scan-"));
-let n = 0;
-/** Make a fixture root containing one or more fake extensions, return its path. */
-function fixture(build) {
-    const root = join(base, `case-${n++}`);
-    mkdirSync(root, { recursive: true });
-    build(root);
-    return root;
-}
-after(() => {
-    rmSync(base, { recursive: true, force: true });
-});
-test("scans a user-level, data-only memory store (no package.json)", () => {
-    const root = fixture((r) => {
-        const ext = join(r, "pi-hermes-memory");
-        mkdirSync(ext, { recursive: true });
-        // No package.json, no source — just pi's memory-store signature.
-        writeFileSync(join(ext, "MEMORY.md"), "# memory\n");
-        writeFileSync(join(ext, "sessions.db"), "");
-    });
-    process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
-    try {
-        const { conflicts } = detectConflicts();
-        assert.ok(conflicts.length >= 1, "expected a memory conflict");
-        const hit = conflicts.find((c) => c.kind === "memory");
-        assert.ok(hit, "expected a memory-kind conflict");
-        assert.equal(hit.severity, "high");
-        assert.ok(hit.evidence.includes("MEMORY.md") ||
-            hit.evidence.includes("sessions.db"), "evidence should name the on-disk memory signature");
-    }
-    finally {
-        delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
-    }
-});
-test("still detects a code extension by source marker (regression)", () => {
-    const root = fixture((r) => {
-        // A code extension is a DIRECT child of the scan root (mirrors the
-        // node_modules layout: packages live one level under the root).
-        const ext = join(r, "some-memory-ext");
-        mkdirSync(ext, { recursive: true });
-        writeFileSync(join(ext, "package.json"), JSON.stringify({ name: "some-memory-ext", pi: { extensions: ["x.ts"] } }));
-        writeFileSync(join(ext, "index.ts"), "export const MEMORY_TOOL = true;");
-    });
-    process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
-    try {
-        const { conflicts } = detectConflicts();
-        const hit = conflicts.find((c) => c.package === "some-memory-ext");
-        assert.ok(hit, "expected some-memory-ext to be flagged");
-        assert.equal(hit.kind, "memory");
-    }
-    finally {
-        delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
-    }
-});
-test("skips pi-mega-compact (selfName) and non-extension dirs", () => {
-    const root = fixture((r) => {
-        // selfName dir with a memory signature — must be ignored.
-        const me = join(r, "node_modules", "pi-mega-compact");
-        mkdirSync(me, { recursive: true });
-        writeFileSync(join(me, "sessions.db"), "");
-        // unrelated dir with no pi.extensions and no memory signature.
-        mkdirSync(join(r, "node_modules", "totally-fine"), { recursive: true });
-    });
-    process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
-    try {
-        const { scanned, conflicts } = detectConflicts();
-        assert.equal(conflicts.length, 0, "no conflicts expected");
-        assert.ok(!scanned.some((s) => s.includes("pi-mega-compact")), "selfName should not appear in scanned");
-    }
-    finally {
-        delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
-    }
-});
-test("collectScanRoots honors MEGACOMPACT_EXT_SCAN_DIR override", () => {
-    const root = fixture(() => { });
-    process.env.MEGACOMPACT_EXT_SCAN_DIR = root;
-    try {
-        const roots = collectScanRoots();
-        assert.deepEqual(roots, [root], "override replaces the whole root list");
-    }
-    finally {
-        delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
-    }
-});
-test("collectScanRoots falls back to node_modules + user dir when no override", () => {
-    delete process.env.MEGACOMPACT_EXT_SCAN_DIR;
-    delete process.env.MEGACOMPACT_EXT_USER_DIR;
-    // No override set and this test file lives under extensions/, so node_modules
-    // resolution walks up from here; the user dir (~/.pi/agent) may or may
-    // not exist in CI. We only assert the call returns a non-throwing array.
-    const roots = collectScanRoots();
-    assert.ok(Array.isArray(roots), "collectScanRoots must return an array");
-});