pi-mega-compact 0.9.2 → 0.11.0

package/dist/src/intent.test.js

@@ -0,0 +1,78 @@
+/**
+ * intent.test.ts — S52A rewind-intent queue tests. No network; temp dirs.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { openTurnStore, closeAllTurnDbs } from "./store/turns/connection.js";
+import { openIntentQueue } from "./intent.js";
+let tmpDir;
+let counter = 0;
+beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "mc-intent-"));
+});
+afterEach(() => {
+    closeAllTurnDbs();
+    rmSync(tmpDir, { recursive: true, force: true });
+});
+function stateDir() {
+    return join(tmpDir, `run-${counter++}`);
+}
+test("postIntent creates a pending intent", () => {
+    const dir = stateDir();
+    const db = openTurnStore(dir);
+    const q = openIntentQueue(db);
+    const intent = q.postIntent({ conversationId: "conv_a", targetTurnIndex: 3 });
+    assert.equal(intent.conversationId, "conv_a");
+    assert.equal(intent.targetTurnIndex, 3);
+    assert.equal(intent.status, "pending");
+    assert.ok(intent.id.length > 0);
+    assert.ok(intent.createdAt > 0);
+});
+test("pendingIntents lists only unconsumed", () => {
+    const dir = stateDir();
+    const db = openTurnStore(dir);
+    const q = openIntentQueue(db);
+    q.postIntent({ conversationId: "c1", targetTurnIndex: 1 });
+    const i2 = q.postIntent({ conversationId: "c2", targetTurnIndex: 2 });
+    assert.equal(q.pendingIntents().length, 2);
+    q.consumeIntent(i2.id);
+    assert.equal(q.pendingIntents().length, 1);
+    assert.equal(q.pendingIntents()[0].conversationId, "c1");
+});
+test("consumeIntent marks status consumed (not deleted)", () => {
+    const dir = stateDir();
+    const db = openTurnStore(dir);
+    const q = openIntentQueue(db);
+    const intent = q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
+    q.consumeIntent(intent.id);
+    assert.equal(q.pendingIntents().length, 0);
+    const all = q.allIntents();
+    assert.equal(all.length, 1);
+    assert.equal(all[0].status, "consumed");
+    assert.equal(all[0].id, intent.id);
+});
+test("abandonIntent deletes the intent", () => {
+    const dir = stateDir();
+    const db = openTurnStore(dir);
+    const q = openIntentQueue(db);
+    const intent = q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
+    q.abandonIntent(intent.id);
+    assert.equal(q.pendingIntents().length, 0);
+    assert.equal(q.allIntents().length, 0);
+});
+test("allIntents returns newest-first + respects limit", () => {
+    const dir = stateDir();
+    const db = openTurnStore(dir);
+    const q = openIntentQueue(db);
+    q.postIntent({ conversationId: "a", targetTurnIndex: 0 });
+    q.postIntent({ conversationId: "b", targetTurnIndex: 0 });
+    q.postIntent({ conversationId: "c", targetTurnIndex: 0 });
+    const all = q.allIntents();
+    assert.equal(all.length, 3);
+    // newest-first: c was posted last
+    assert.equal(all[0].conversationId, "c");
+    assert.equal(q.allIntents(2).length, 2);
+});

package/dist/src/metrics/index.js

@@ -0,0 +1,4 @@
+/**
+ * index.ts — S50 metrics barrel. Re-export only.
+ */
+export { turnMetrics, conversationMetrics, } from "./turns.js";

package/dist/src/metrics/turns.js

@@ -0,0 +1,118 @@
+/** Approximate byte length of a text (utf8). */
+function byteLen(s) {
+    return Buffer.byteLength(s, "utf8");
+}
+/**
+ * Per-turn metrics for a conversation. Combines turns.db (turn spine + recall)
+ * with main-db raw_transcript (dedup) + checkpoint_epochs (compression).
+ *
+ * `mainDb` may be a store without raw_transcript/checkpoint_epochs (e.g. a
+ * reuse host that only mirrors turns); in that case dedup/compression = 0 and
+ * rawMessageCount = 0. Detected defensively per-query.
+ */
+export function turnMetrics(store, mainDb, conversationId) {
+    const turns = store.query({ conversationId, limit: 10000 });
+    if (turns.length === 0)
+        return [];
+    const hasRaw = tableExists(mainDb, "raw_transcript");
+    const hasEpoch = tableExists(mainDb, "checkpoint_epochs");
+    // Aggregate raw_transcript per turn_index for this session (raw rows + unique
+    // content_hash). Grouped by turn_index; sessions may share turn_index values,
+    // so we scope by session_id below per turn.
+    const rawAggBySessionTurn = new Map();
+    if (hasRaw) {
+        const sessionIds = [...new Set(turns.map((t) => t.sessionId))];
+        const placeholders = sessionIds.map(() => "?").join(",");
+        const rows = mainDb
+            .prepare(`SELECT session_id, turn_index,
+                COUNT(*) AS raw_count,
+                COUNT(DISTINCT content_hash) AS unique_count
+         FROM raw_transcript
+         WHERE session_id IN (${placeholders}) AND turn_index IS NOT NULL
+         GROUP BY session_id, turn_index`)
+            .all(...sessionIds);
+        for (const r of rows) {
+            rawAggBySessionTurn.set(`${r.session_id}::${r.turn_index}`, r);
+        }
+    }
+    // Compression ratio per epoch: committed raw bytes → summary bytes. Precomputed
+    // per epoch referenced by any turn.
+    const compressionByEpoch = new Map();
+    if (hasEpoch) {
+        const epochIds = turns
+            .map((t) => t.epochId)
+            .filter((e) => e != null);
+        for (const eid of new Set(epochIds)) {
+            const ratio = epochCompressionRatio(mainDb, eid);
+            if (ratio != null)
+                compressionByEpoch.set(eid, ratio);
+        }
+    }
+    return turns.map((t) => {
+        const recall = store.listRecallByIndex(conversationId, t.turnIndex);
+        const agg = rawAggBySessionTurn.get(`${t.sessionId}::${t.turnIndex}`);
+        const rawCount = agg?.raw_count ?? 0;
+        const uniqueCount = agg?.unique_count ?? 0;
+        return {
+            turnId: t.turnIndex,
+            conversationId: t.conversationId,
+            sessionId: t.sessionId,
+            turnIndex: t.turnIndex,
+            epochId: t.epochId ?? null,
+            ctxTokens: t.ctxTokens ?? null,
+            ctxPercent: t.ctxPercent ?? null,
+            recallCount: recall.length,
+            rawMessageCount: rawCount,
+            dedupUniqueRatio: rawCount > 0 ? uniqueCount / rawCount : 0,
+            compressionRatio: t.epochId
+                ? (compressionByEpoch.get(t.epochId) ?? 0)
+                : 0,
+        };
+    });
+}
+/** Aggregate a conversation's per-turn metrics. */
+export function conversationMetrics(store, mainDb, conversationId) {
+    const perTurn = turnMetrics(store, mainDb, conversationId);
+    const turnCount = perTurn.length;
+    const totalRecall = perTurn.reduce((s, t) => s + t.recallCount, 0);
+    const totalRaw = perTurn.reduce((s, t) => s + t.rawMessageCount, 0);
+    const dedupSum = perTurn.reduce((s, t) => s + t.dedupUniqueRatio, 0);
+    const compSum = perTurn.reduce((s, t) => s + t.compressionRatio, 0);
+    const epochs = new Set(perTurn.map((t) => t.epochId).filter(Boolean));
+    return {
+        conversationId,
+        turnCount,
+        totalRecall,
+        totalRawMessages: totalRaw,
+        avgDedupUniqueRatio: turnCount > 0 ? dedupSum / turnCount : 0,
+        avgCompressionRatio: turnCount > 0 ? compSum / turnCount : 0,
+        epochCount: epochs.size,
+    };
+}
+function tableExists(db, name) {
+    return (db
+        .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?")
+        .get(name) !== undefined);
+}
+/**
+ * Compression ratio for an epoch: summaryMessageText bytes ÷ raw bytes of the
+ * committed range. Returns null when the epoch or its raw rows are absent.
+ * Ratio < 1 means compression shrank the content (lower = more compression).
+ */
+function epochCompressionRatio(mainDb, epochId) {
+    const epoch = mainDb
+        .prepare("SELECT session_id, committed_seq, summary_message_text FROM checkpoint_epochs WHERE epoch_id = ?")
+        .get(epochId);
+    if (!epoch)
+        return null;
+    const raw = mainDb
+        .prepare(`SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS bytes
+       FROM raw_transcript
+       WHERE session_id = ? AND seq <= ?`)
+        .get(epoch.session_id, epoch.committed_seq);
+    const rawBytes = raw?.bytes ?? 0;
+    if (rawBytes <= 0)
+        return null;
+    const summaryBytes = byteLen(epoch.summary_message_text ?? "");
+    return summaryBytes / rawBytes;
+}

package/dist/src/metrics/turns.test.js

@@ -0,0 +1,152 @@
+/**
+ * turns.test.ts — S50C per-turn + per-conversation metrics tests. No network.
+ * Seeds a turns.db (S49 contract store) and a main db (raw_transcript +
+ * checkpoint_epochs) and asserts the rollups match seeded truth.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { createTurnStore, closeAllTurnDbs, } from "../store/turns/index.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { appendRawTranscript } from "../store/sqlite/raw-transcript.js";
+import { writeCheckpointEpoch } from "../store/sqlite/raw-transcript.js";
+import { turnMetrics, conversationMetrics } from "./turns.js";
+let tmpDir;
+let counter = 0;
+beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "mc-metrics-"));
+});
+afterEach(() => {
+    closeAllTurnDbs();
+    rmSync(tmpDir, { recursive: true, force: true });
+});
+function stateDir() {
+    return join(tmpDir, `run-${counter++}`);
+}
+/** Seed one conversation: 2 turns, recall on turn 0, raw messages, one epoch. */
+function seed(dir) {
+    const store = createTurnStore({ stateDir: dir });
+    const conv = store.ensureConversationId("sess_m");
+    const t0 = store.appendTurn({
+        conversationId: conv,
+        sessionId: "sess_m",
+        turnIndex: 0,
+        role: "assistant",
+        endedAt: Date.now(),
+        ctxTokens: 800,
+        ctxPercent: 0.4,
+    });
+    store.appendRecall({
+        turnId: t0,
+        checkpointId: "cp_1",
+        score: 0.9,
+        source: "checkpoint",
+    });
+    store.appendRecall({
+        turnId: t0,
+        checkpointId: "cp_2",
+        score: 0.8,
+        source: "checkpoint",
+    });
+    store.appendTurn({
+        conversationId: conv,
+        sessionId: "sess_m",
+        turnIndex: 1,
+        role: "assistant",
+        endedAt: Date.now(),
+        ctxTokens: 1200,
+    });
+    store.asAdmin().stampTurnsEpoch("sess_m", "ep_1");
+    const main = openStore(dir);
+    // 5 raw messages in the committed range; ep_1's summary is much smaller
+    // (compression ratio < 1). turn_index is set on all of them.
+    const msgs = [
+        { hash: "h0", turn: 0, bytes: "aaaaaaaa" },
+        { hash: "h1", turn: 0, bytes: "bbbbbbbb" },
+        { hash: "h2", turn: 1, bytes: "cccccccc" },
+        { hash: "h3", turn: 1, bytes: "dddddddd" },
+        { hash: "h4", turn: 1, bytes: "eeeeeeee" },
+    ];
+    for (const m of msgs) {
+        appendRawTranscript(main, {
+            contentHash: m.hash,
+            sessionId: "sess_m",
+            seq: 0,
+            role: "user",
+            contentBytes: m.bytes,
+            toolName: null,
+            messageTimestamp: null,
+            checkpointEpoch: "ep_1",
+            turnIndex: m.turn,
+        });
+    }
+    writeCheckpointEpoch(main, {
+        epochId: "ep_1",
+        sessionId: "sess_m",
+        startedSeq: 0,
+        committedSeq: 3,
+        summaryMessageText: "sum",
+        cutIndex: 3,
+        checkpointId: "cp_x",
+        createdAt: Date.now(),
+    });
+    return { store, conv };
+}
+test("turnMetrics returns per-turn recall + dedup + compression", () => {
+    const dir = stateDir();
+    const { store, conv } = seed(dir);
+    const main = openStore(dir);
+    const rows = turnMetrics(store, main, conv);
+    assert.equal(rows.length, 2);
+    const t0 = rows.find((r) => r.turnIndex === 0);
+    assert.equal(t0.recallCount, 2);
+    assert.equal(t0.rawMessageCount, 2);
+    assert.equal(t0.dedupUniqueRatio, 1); // PK(content_hash,session_id) → all stored rows unique
+    const t1 = rows.find((r) => r.turnIndex === 1);
+    assert.equal(t1.rawMessageCount, 3);
+    assert.equal(t1.epochId, "ep_1");
+    assert.ok(t0.compressionRatio > 0 && t0.compressionRatio < 1); // summary smaller than raw
+    store.close();
+});
+test("conversationMetrics aggregates per-turn metrics", () => {
+    const dir = stateDir();
+    const { store, conv } = seed(dir);
+    const main = openStore(dir);
+    const m = conversationMetrics(store, main, conv);
+    assert.equal(m.turnCount, 2);
+    assert.equal(m.totalRecall, 2);
+    assert.equal(m.totalRawMessages, 5);
+    assert.equal(m.epochCount, 1);
+    assert.ok(m.avgDedupUniqueRatio > 0);
+    store.close();
+});
+test("metrics tolerate a main db without raw_transcript (reuse host)", () => {
+    const dir = stateDir();
+    const store = createTurnStore({ stateDir: dir });
+    const conv = store.ensureConversationId("sess_n");
+    const t = store.appendTurn({
+        conversationId: conv,
+        sessionId: "sess_n",
+        turnIndex: 0,
+        role: "assistant",
+        endedAt: Date.now(),
+    });
+    store.appendRecall({
+        turnId: t,
+        checkpointId: "cp",
+        score: 0.5,
+        source: "checkpoint",
+    });
+    // Bare in-memory db with NO raw_transcript / checkpoint_epochs tables.
+    const bare = new DatabaseSync(":memory:");
+    const rows = turnMetrics(store, bare, conv);
+    assert.equal(rows.length, 1);
+    assert.equal(rows[0].recallCount, 1);
+    assert.equal(rows[0].rawMessageCount, 0);
+    assert.equal(rows[0].dedupUniqueRatio, 0);
+    assert.equal(rows[0].compressionRatio, 0);
+    store.close();
+});

package/dist/src/store/sqlite/raw-transcript.js

@@ -9,6 +9,7 @@ function rowToRawTranscript(row) {
         toolName: row.tool_name ?? null,
         messageTimestamp: row.message_timestamp == null ? null : Number(row.message_timestamp),
         checkpointEpoch: row.checkpoint_epoch,
+        turnIndex: row.turn_index == null ? null : Number(row.turn_index),
     };
 }
 function rowToCheckpointEpoch(row) {
@@ -34,11 +35,11 @@ function rowToCheckpointEpoch(row) {
 export function appendRawTranscript(db, row) {
     withTx(db, () => {
         db.prepare(`INSERT OR IGNORE INTO raw_transcript
-        (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch)
+        (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch, turn_index)
        VALUES (
          @content_hash, @session_id,
          COALESCE((SELECT MAX(seq) FROM raw_transcript WHERE session_id = @session_id), 0) + 1,
-         @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch
+         @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch, @turn_index
        )`).run({
             "@content_hash": row.contentHash,
             "@session_id": row.sessionId,
@@ -47,6 +48,7 @@ export function appendRawTranscript(db, row) {
             "@tool_name": row.toolName,
             "@message_timestamp": row.messageTimestamp,
             "@checkpoint_epoch": row.checkpointEpoch,
+            "@turn_index": row.turnIndex ?? null,
         });
     });
 }
@@ -56,7 +58,7 @@ export function appendRawTranscript(db, row) {
  */
 export function listRawTranscriptRange(db, sessionId, fromSeq, toSeq) {
     const rows = db
-        .prepare(`SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch
+        .prepare(`SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch, turn_index
        FROM raw_transcript
        WHERE session_id = @session_id AND seq >= @from_seq AND seq <= @to_seq
        ORDER BY seq ASC`)

package/dist/src/store/sqlite.dbmirror.test.js

@@ -82,6 +82,15 @@ describe("DB mirror", () => {
             assert.equal(rows[0].toolName, "bash");
             assert.equal(rows[0].messageTimestamp, 1234567890);
         });
+        it("S50A: stores turn_index when provided; null when omitted (back-compat)", () => {
+            appendRawTranscript(db, makeRow({ contentHash: "with-turn", turnIndex: 7 }));
+            appendRawTranscript(db, makeRow({ contentHash: "no-turn" }));
+            const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);
+            const withTurn = rows.find((r) => r.contentHash === "with-turn");
+            const noTurn = rows.find((r) => r.contentHash === "no-turn");
+            assert.equal(withTurn?.turnIndex, 7);
+            assert.equal(noTurn?.turnIndex, null);
+        });
         it("stores checkpoint_epoch", () => {
             appendRawTranscript(db, makeRow({ checkpointEpoch: "ep-42" }));
             const rows = listRawTranscriptRange(db, "sess-1", 0, 999999);

package/dist/src/store/turns/connection.js

@@ -0,0 +1,123 @@
+/**
+ * connection.ts — S49 reconciled turns.db connection manager.
+ *
+ * RECONCILIATION (s49-turn-db ∪ master): exposes master's contract-facing
+ * `openTurnDb` / `closeTurnDb` / `closeAllTurnDbs` (consumed by SqliteTurnStore)
+ * AND this branch's value-adds: closed-handle eviction, the MEGACOMPACT_TURNS_DB_PATH
+ * env override, and the one-time main-db → turns.db migration (S49B) run after
+ * schema init. The unified `initTurnSchema` (schema.ts) creates both the
+ * contract tables and the S51/S52 additive shells.
+ *
+ * Separate cache from the main memory store (src/store/sqlite/utils.ts) so a
+ * turn-DB failure can never touch the authoritative memory DB (program §2).
+ * Pi-agnostic (PREVENT-PI-004: node:sqlite in-process, no network).
+ */
+import { existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { TurnsConfig } from "../../config/turns.js";
+import { initTurnSchema } from "./schema.js";
+import { migrateTurnTablesIfNeeded } from "./migrations.js";
+/** Default turns.db filename within a state dir. */
+export const TURNS_DB_FILE = "turns.db";
+// Module-level cache, SEPARATE from the memory store's cache. Keyed by the
+// resolved db path (or `:memory:<stateDir>` for in-memory).
+const cache = new Map();
+/** Resolve the turns.db path (env override wins — tests/DR). */
+export function turnDbPath(stateDir) {
+    const override = process.env.MEGACOMPACT_TURNS_DB_PATH;
+    if (override && override.trim() !== "")
+        return override;
+    return join(stateDir, TURNS_DB_FILE);
+}
+/**
+ * Open (or reuse) the isolated turn database connection. Contract-facing
+ * shape consumed by `SqliteTurnStore`. Accepts `dbPath` / `inMemory` overrides
+ * for tests. Runs the unified schema init + the one-time main-db migration.
+ */
+export function openTurnDb(stateDir, options) {
+    const resolvedPath = options?.inMemory === true
+        ? ":memory:"
+        : (options?.dbPath ?? turnDbPath(stateDir));
+    const cacheKey = options?.inMemory ? `:memory:${stateDir}` : resolvedPath;
+    const cached = cache.get(cacheKey);
+    if (cached) {
+        // A closed handle in the cache (test called db.close() directly) would
+        // surface as "database is not open" on reuse. Detect + evict.
+        try {
+            cached.exec("SELECT 1");
+            return cached;
+        }
+        catch {
+            cache.delete(cacheKey);
+        }
+    }
+    if (options?.inMemory !== true && !existsSync(stateDir)) {
+        mkdirSync(stateDir, { recursive: true });
+    }
+    const db = new DatabaseSync(resolvedPath);
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
+    db.exec("PRAGMA busy_timeout = 5000");
+    initTurnSchema(db);
+    // S49B: one-time move of legacy main-db turn tables into turns.db (idempotent,
+    // non-fatal). File-backed only; never runs for the in-memory test backend.
+    if (options?.inMemory !== true && TurnsConfig.TURNS_DB_ENABLED) {
+        migrateTurnTablesIfNeeded(db, stateDir);
+    }
+    cache.set(cacheKey, db);
+    return db;
+}
+/**
+ * Back-compat alias for callers (S51 topics store, dashboard routes) that open
+ * a raw DatabaseSync against turns.db to run their own read queries. Equivalent
+ * to `openTurnDb(stateDir)` (file-backed, default path).
+ */
+export function openTurnStore(stateDir) {
+    return openTurnDb(stateDir);
+}
+/** Back-compat alias: close the cached file-backed connection for a state dir. */
+export function closeTurnStore(stateDir) {
+    closeTurnDb(stateDir);
+}
+/** Close and evict a cached connection. Idempotent. */
+export function closeTurnDb(stateDir, options) {
+    const resolvedPath = options?.inMemory === true
+        ? ":memory:"
+        : (options?.dbPath ?? turnDbPath(stateDir));
+    const cacheKey = options?.inMemory ? `:memory:${stateDir}` : resolvedPath;
+    const db = cache.get(cacheKey);
+    if (!db)
+        return;
+    db.close();
+    cache.delete(cacheKey);
+}
+/** Close all cached connections (test teardown / graceful shutdown). */
+export function closeAllTurnDbs() {
+    for (const db of cache.values()) {
+        try {
+            db.close();
+        }
+        catch {
+            // best-effort
+        }
+    }
+    cache.clear();
+}
+/**
+ * Run `fn` atomically. SAVEPOINT (not BEGIN) so it nests safely under an outer
+ * transaction. Self-contained (not imported from memory utils) to keep the
+ * reuse seam clean.
+ */
+export function withTx(db, fn) {
+    db.exec("SAVEPOINT turns_tx");
+    try {
+        fn();
+        db.exec("RELEASE turns_tx");
+    }
+    catch (e) {
+        db.exec("ROLLBACK TO turns_tx");
+        db.exec("RELEASE turns_tx");
+        throw e;
+    }
+}