pi-mega-compact 0.7.2 → 0.7.4

package/src/store/sqlite.ts

@@ -25,7 +25,7 @@ import { getStateDir } from "../store.js";
 import type { StoredCheckpoint, SessionState } from "../store.js";
 import { normalizeSessionId } from "../store.js";
 
-const SCHEMA_VERSION = 1;
+const SCHEMA_VERSION = 2;
 
 /** Encode a float vector as a little-endian Float32 BLOB for cosine scanning. */
 function encodeEmbedding(v: number[]): Buffer {
@@ -543,6 +543,54 @@ function initSchema(db: DatabaseSync): void {
       normalized_text,
       tokenize='trigram'
     );
+
+    -- S27: durable raw-transcript mirror (MEGACOMPACT_DB_MIRROR). Appended
+    -- RAW message bytes per session so a compacted window can be rehydrated
+    -- from the local store instead of the pi runtime transcript (which is
+    -- trimmed). PK is (content_hash, session_id) — NOT content_hash alone —
+    -- so identical content in different sessions never collides. Additive:
+    -- CREATE TABLE IF NOT EXISTS leaves existing DBs untouched on open until
+    -- the S27 mirror flag is flipped on. All queries parameterized (PREVENT-002).
+    CREATE TABLE IF NOT EXISTS raw_transcript (
+      content_hash      TEXT NOT NULL,
+      session_id        TEXT NOT NULL,
+      seq               INTEGER NOT NULL,
+      role              TEXT NOT NULL,
+      content_bytes     TEXT NOT NULL,
+      tool_name         TEXT,
+      message_timestamp INTEGER,          -- ORIGINAL msg ts at append, NOT served
+      checkpoint_epoch  TEXT NOT NULL,
+      PRIMARY KEY (content_hash, session_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_rt_session_seq ON raw_transcript(session_id, seq);
+    CREATE INDEX IF NOT EXISTS idx_rt_epoch ON raw_transcript(checkpoint_epoch);
+
+    -- S27: checkpoint-epoch registry. One row per compaction epoch; the
+    -- summary_message_text is the verbatim system message that replaced the
+    -- trimmed prefix. Informational bookkeeping (the raw_transcript rows are
+    -- authoritative); refresh-safe via ON CONFLICT(epoch_id) DO UPDATE.
+    CREATE TABLE IF NOT EXISTS checkpoint_epochs (
+      epoch_id             TEXT PRIMARY KEY,
+      session_id           TEXT NOT NULL,
+      started_seq          INTEGER NOT NULL,
+      committed_seq        INTEGER NOT NULL,
+      summary_message_text TEXT NOT NULL,
+      cut_index            INTEGER NOT NULL,
+      checkpoint_id        TEXT NOT NULL,
+      created_at           INTEGER NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_epoch_session ON checkpoint_epochs(session_id, created_at DESC);
+
+    -- S27 Task 6: dedup_mirror for space-efficient deduplicated storage.
+    -- Each unique content_hash stores its bytes ONCE; raw_transcript rows
+    -- reference this table via content_ref instead of storing duplicate content_bytes inline.
+    CREATE TABLE IF NOT EXISTS dedup_mirror (
+      content_hash    TEXT PRIMARY KEY,
+      content_bytes   TEXT NOT NULL,
+      ref_count       INTEGER NOT NULL DEFAULT 1,
+      first_seen_seq  INTEGER NOT NULL,
+      created_at      INTEGER NOT NULL
+    );
   `);
   // Idempotent column migrations. `CREATE TABLE IF NOT EXISTS` is a no-op on a
   // pre-existing table, so new columns added to context_chunks after a store was
@@ -550,6 +598,8 @@ function initSchema(db: DatabaseSync): void {
   // databases created by an older version — otherwise repoStats()/upsert crash
   // with "no such column" and the extension fails to load. Additive only.
   ensureColumn(db, "context_chunks", "original_token_estimate", "INTEGER");
+  // S27 Task 6: content_ref column in raw_transcript for dedup_mirror references.
+  ensureColumn(db, "raw_transcript", "content_ref", "TEXT");
   // S20 memory-RAG extension: additive columns for auto-review ops. Idempotent —
   // only alters DBs created by an older version that lack these columns.
   ensureColumn(db, "memories", "category", "TEXT");
@@ -1509,3 +1559,330 @@ export function clearRaptorNodes(sessionId: string, stateDir: string = getStateD
   const db = openStore(stateDir);
   db.prepare("DELETE FROM raptor_nodes WHERE session_id = ?").run(normalizeSessionId(sessionId));
 }
+
+// --- S27: durable raw-transcript mirror + checkpoint-epoch registry ---------
+// ADDITIVE + flag-default-OFF (MEGACOMPACT_DB_MIRROR). No behavior change
+// until the flag is flipped on; the tables above are created IF NOT EXISTS on
+// open so existing stores are untouched. These helpers are storage primitives
+// only — the runtime hook (Task 5) wires them into the compaction flow. All
+// queries use @-parameterized placeholders (PREVENT-002); no `any` types
+// (PREVENT-011).
+
+/** One appended raw-message row in the durable mirror. */
+export interface RawTranscriptRow {
+  contentHash: string;
+  sessionId: string;
+  seq: number;
+  role: string;
+  contentBytes: string;
+  toolName: string | null;
+  /** ORIGINAL message timestamp captured at append time (NOT a served ts). */
+  messageTimestamp: number | null;
+  checkpointEpoch: string;
+}
+
+/** One checkpoint-epoch bookkeeping row (informational registry). */
+export interface CheckpointEpoch {
+  epochId: string;
+  sessionId: string;
+  startedSeq: number;
+  committedSeq: number;
+  summaryMessageText: string;
+  cutIndex: number;
+  checkpointId: string;
+  createdAt: number;
+}
+
+/** DB row shape for raw_transcript (snake_case column names). */
+interface RawTranscriptDBRow {
+  content_hash: string;
+  session_id: string;
+  seq: number;
+  role: string;
+  content_bytes: string;
+  tool_name: string | null;
+  message_timestamp: number | null;
+  checkpoint_epoch: string;
+}
+
+/** DB row shape for checkpoint_epochs (snake_case column names). */
+interface CheckpointEpochDBRow {
+  epoch_id: string;
+  session_id: string;
+  started_seq: number;
+  committed_seq: number;
+  summary_message_text: string;
+  cut_index: number;
+  checkpoint_id: string;
+  created_at: number;
+}
+
+function rowToRawTranscript(row: RawTranscriptDBRow): RawTranscriptRow {
+  return {
+    contentHash: row.content_hash,
+    sessionId: row.session_id,
+    seq: Number(row.seq),
+    role: row.role,
+    contentBytes: row.content_bytes,
+    toolName: row.tool_name ?? null,
+    messageTimestamp:
+      row.message_timestamp == null ? null : Number(row.message_timestamp),
+    checkpointEpoch: row.checkpoint_epoch,
+  };
+}
+
+function rowToCheckpointEpoch(row: CheckpointEpochDBRow): CheckpointEpoch {
+  return {
+    epochId: row.epoch_id,
+    sessionId: row.session_id,
+    startedSeq: Number(row.started_seq),
+    committedSeq: Number(row.committed_seq),
+    summaryMessageText: row.summary_message_text,
+    cutIndex: Number(row.cut_index),
+    checkpointId: row.checkpoint_id,
+    createdAt: Number(row.created_at),
+  };
+}
+
+/**
+ * Append one raw-message row to the durable mirror. Idempotent by
+ * (content_hash, session_id) via INSERT OR IGNORE — re-appending the same
+ * content for the same session is a no-op. seq is assigned server-side as
+ * COALESCE(MAX(seq),0)+1 within the session, so callers never need to compute
+ * it. Pass an open store handle (openStore) — matches the other DatabaseSync
+ * helpers. Parameterized (PREVENT-002).
+ */
+export function appendRawTranscript(db: DatabaseSync, row: RawTranscriptRow): void {
+  withTx(db, () => {
+    db.prepare(
+      `INSERT OR IGNORE INTO raw_transcript
+        (content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch)
+       VALUES (
+         @content_hash, @session_id,
+         COALESCE((SELECT MAX(seq) FROM raw_transcript WHERE session_id = @session_id), 0) + 1,
+         @role, @content_bytes, @tool_name, @message_timestamp, @checkpoint_epoch
+       )`,
+    ).run({
+      "@content_hash": row.contentHash,
+      "@session_id": row.sessionId,
+      "@role": row.role,
+      "@content_bytes": row.contentBytes,
+      "@tool_name": row.toolName,
+      "@message_timestamp": row.messageTimestamp,
+      "@checkpoint_epoch": row.checkpointEpoch,
+    });
+  });
+}
+
+/**
+ * List raw-transcript rows for a session in [fromSeq, toSeq], ordered by seq
+ * ascending. Returns camel-cased RawTranscriptRow[]. Parameterized.
+ */
+export function listRawTranscriptRange(
+  db: DatabaseSync,
+  sessionId: string,
+  fromSeq: number,
+  toSeq: number,
+): RawTranscriptRow[] {
+  const rows = db
+    .prepare(
+      `SELECT content_hash, session_id, seq, role, content_bytes, tool_name, message_timestamp, checkpoint_epoch
+       FROM raw_transcript
+       WHERE session_id = @session_id AND seq >= @from_seq AND seq <= @to_seq
+       ORDER BY seq ASC`,
+    )
+    .all({
+      "@session_id": sessionId,
+      "@from_seq": fromSeq,
+      "@to_seq": toSeq,
+    }) as unknown as RawTranscriptDBRow[];
+  return rows.map(rowToRawTranscript);
+}
+
+/**
+ * Insert (or refresh) a checkpoint-epoch row. ON CONFLICT(epoch_id) DO UPDATE
+ * so re-running the same compaction epoch is idempotent / refresh-safe.
+ * Parameterized (PREVENT-002).
+ */
+export function writeCheckpointEpoch(db: DatabaseSync, epoch: CheckpointEpoch): void {
+  withTx(db, () => {
+    db.prepare(
+      `INSERT INTO checkpoint_epochs
+        (epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at)
+       VALUES (@epoch_id, @session_id, @started_seq, @committed_seq, @summary_message_text, @cut_index, @checkpoint_id, @created_at)
+       ON CONFLICT(epoch_id) DO UPDATE SET
+         session_id = excluded.session_id,
+         started_seq = excluded.started_seq,
+         committed_seq = excluded.committed_seq,
+         summary_message_text = excluded.summary_message_text,
+         cut_index = excluded.cut_index,
+         checkpoint_id = excluded.checkpoint_id,
+         created_at = excluded.created_at`,
+    ).run({
+      "@epoch_id": epoch.epochId,
+      "@session_id": epoch.sessionId,
+      "@started_seq": epoch.startedSeq,
+      "@committed_seq": epoch.committedSeq,
+      "@summary_message_text": epoch.summaryMessageText,
+      "@cut_index": epoch.cutIndex,
+      "@checkpoint_id": epoch.checkpointId,
+      "@created_at": epoch.createdAt,
+    });
+  });
+}
+
+/** Read one checkpoint-epoch row by id (or null if absent). Parameterized. */
+export function readCheckpointEpoch(db: DatabaseSync, epochId: string): CheckpointEpoch | null {
+  const row = db
+    .prepare(
+      `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
+       FROM checkpoint_epochs WHERE epoch_id = @epoch_id`,
+    )
+    .get({ "@epoch_id": epochId }) as unknown as CheckpointEpochDBRow | undefined;
+  return row ? rowToCheckpointEpoch(row) : null;
+}
+
+/**
+ * Latest checkpoint-epoch row for a session (highest created_at), or null if
+ * none. Parameterized (PREVENT-002).
+ */
+export function getActiveEpochForSession(db: DatabaseSync, sessionId: string): CheckpointEpoch | null {
+  const row = db
+    .prepare(
+      `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
+       FROM checkpoint_epochs
+       WHERE session_id = @session_id
+       ORDER BY created_at DESC
+       LIMIT 1`,
+    )
+    .get({ "@session_id": sessionId }) as unknown as CheckpointEpochDBRow | undefined;
+  return row ? rowToCheckpointEpoch(row) : null;
+}
+
+/** List all checkpoint epochs (diagnostic / test helper). */
+export function listCheckpointEpochs(db: DatabaseSync): CheckpointEpoch[] {
+  const rows = db
+    .prepare(
+      `SELECT epoch_id, session_id, started_seq, committed_seq, summary_message_text, cut_index, checkpoint_id, created_at
+       FROM checkpoint_epochs
+       ORDER BY created_at DESC`,
+    )
+    .all() as unknown as CheckpointEpochDBRow[];
+  return rows.map(rowToCheckpointEpoch);
+}
+
+/** Count raw transcript rows (diagnostic / test helper). */
+export function countRawTranscript(db: DatabaseSync): number {
+  const row = db.prepare(`SELECT COUNT(*) AS cnt FROM raw_transcript`).get() as { cnt: number };
+  return row.cnt;
+}
+
+// ────────────────────────────────────────────────────────────────────────
+// S27 Task 6: dedup_mirror functions
+// ────────────────────────────────────────────────────────────────────────
+
+/**
+ * Dedup mirror row (DB representation).
+ */
+export interface DedupMirrorRowDB {
+  content_hash: string;
+  content_bytes: string;
+  ref_count: number;
+  first_seen_seq: number;
+  created_at: number;
+}
+
+/**
+ * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
+ * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
+ */
+export function upsertDedupMirror(
+  db: DatabaseSync,
+  contentHash: string,
+  contentBytes: string,
+  seq: number,
+): boolean {
+  const now = Date.now();
+  const existing = db
+    .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
+    .get({ "@hash": contentHash }) as { content_hash: string } | undefined;
+  if (existing) {
+    db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
+      "@hash": contentHash,
+    });
+    return false;
+  }
+  db.prepare(
+    `INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
+     VALUES (@hash, @bytes, 1, @seq, @now)`,
+  ).run({
+    "@hash": contentHash,
+    "@bytes": contentBytes,
+    "@seq": seq,
+    "@now": now,
+  });
+  return true;
+}
+
+/**
+ * Get dedup ratio for a session: total bytes vs unique bytes.
+ */
+export function getDedupRatio(
+  db: DatabaseSync,
+  sessionId: string,
+): { totalBytes: number; uniqueBytes: number; ratio: number } {
+  const totalRow = db
+    .prepare(
+      `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS total
+       FROM raw_transcript
+       WHERE session_id = @session_id`,
+    )
+    .get({ "@session_id": sessionId }) as { total: number };
+  const uniqueRow = db
+    .prepare(
+      `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
+       FROM dedup_mirror`,
+    )
+    .get() as { unique_bytes: number };
+  const totalBytes = totalRow.total;
+  const uniqueBytes = uniqueRow.unique_bytes;
+  const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
+  return { totalBytes, uniqueBytes, ratio };
+}
+
+/**
+ * Get dedup mirror stats (diagnostic / test helper).
+ */
+export function getDedupMirrorStats(db: DatabaseSync): {
+  rowCount: number;
+  totalBytes: number;
+  avgRefCount: number;
+} {
+  const row = db
+    .prepare(
+      `SELECT COUNT(*) AS cnt,
+              COALESCE(SUM(LENGTH(content_bytes)), 0) AS total_bytes,
+              COALESCE(AVG(ref_count), 0) AS avg_ref
+       FROM dedup_mirror`,
+    )
+    .get() as { cnt: number; total_bytes: number; avg_ref: number };
+  return { rowCount: row.cnt, totalBytes: row.total_bytes, avgRefCount: row.avg_ref };
+}
+
+/**
+ * Update raw_transcript.content_ref to point to dedup_mirror.
+ */
+export function updateRawTranscriptRef(
+  db: DatabaseSync,
+  sessionId: string,
+  seq: number,
+  contentHash: string,
+): void {
+  db.prepare(
+    `UPDATE raw_transcript SET content_ref = @ref WHERE session_id = @sid AND seq = @seq`,
+  ).run({
+    "@ref": contentHash,
+    "@sid": sessionId,
+    "@seq": seq,
+  });
+}