pi-mega-compact 0.7.8 → 0.7.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.7.8",
+  "version": "0.7.9",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-2-Clause",
@@ -41,15 +41,16 @@
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "lint": "tsc --noEmit && node scripts/guardrails-scan.mjs",
+    "lint": "tsc --noEmit && node scripts/guardrails-scan.mjs && node scripts/semantic-scan.mjs",
     "test": "npm run build && node scripts/run-tests.mjs",
-    "guardrails": "python3 scripts/regression_check.py --all || node scripts/guardrails-scan.mjs",
+    "guardrails": "python3 scripts/regression_check.py --all || node scripts/guardrails-scan.mjs || node scripts/semantic-scan.mjs",
     "precommit": "bash .claude/hooks/pre-commit.sh",
     "prepublishOnly": "npm run build",
     "postinstall": "npm rebuild @mongodb-js/zstd || true"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": ">=0.80",
     "openclaw": ">=0.1.0"
   },
   "devDependencies": {

package/src/store/sqlite/checkpoints.ts

@@ -0,0 +1,204 @@
+/**
+ * checkpoints.ts — `context_chunks` checkpoint CRUD + MinHash/LSH dedup helpers.
+ */
+import type { StoredCheckpoint } from "../../store.js";
+import { getStateDir, normalizeSessionId } from "../../store.js";
+import {
+  openStore,
+  withTx,
+  jsonText,
+  encodeEmbedding,
+  rowToCheckpoint,
+} from "./utils.js";
+
+/** Insert or replace a checkpoint (idempotent by id). */
+export function upsertCheckpoint(cp: StoredCheckpoint, stateDir: string = getStateDir()): void {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(cp.sessionId);
+  withTx(db, () => {
+    db.prepare(
+      `INSERT INTO context_chunks
+        (id, session_id, region_hash, content_hash, content_hash2, content_hash_version,
+         normalized_text, summary, topic_summary, summary_hash,
+         key_decisions, next_steps, files_modified, embedding_blob,
+         token_estimate, original_token_estimate, timestamp, dedup_status, compressed_original)
+       VALUES (@id, @sid, @region_hash, @content_hash, @content_hash2, @content_hash_version,
+               @normalized_text, @summary, @topic_summary, @summary_hash,
+               @key_decisions, @next_steps, @files_modified, @embedding_blob,
+               @token_estimate, @original_token_estimate, @timestamp, @dedup_status, @compressed_original)
+       ON CONFLICT(session_id, id) DO UPDATE SET
+         summary=excluded.summary,
+         topic_summary=excluded.topic_summary,
+         summary_hash=excluded.summary_hash,
+         key_decisions=excluded.key_decisions,
+         next_steps=excluded.next_steps,
+         files_modified=excluded.files_modified,
+         embedding_blob=excluded.embedding_blob,
+         token_estimate=excluded.token_estimate,
+         original_token_estimate=excluded.original_token_estimate,
+         timestamp=excluded.timestamp,
+         dedup_status=excluded.dedup_status,
+         compressed_original=excluded.compressed_original`,
+    ).run({
+      "@id": cp.checkpointId,
+      "@sid": sid,
+      "@region_hash": cp.regionHash ?? null,
+      "@content_hash": cp.contentHash ?? null,
+      "@content_hash2": cp.contentHash2 ?? null,
+      "@content_hash_version": cp.contentHashVersion ?? null,
+      "@normalized_text": cp.normalizedText ?? null,
+      "@summary": cp.summary ?? "",
+      "@topic_summary": cp.topicSummary ?? null,
+      "@summary_hash": cp.summaryHash ?? null,
+      "@key_decisions": jsonText(cp.keyDecisions),
+      "@next_steps": jsonText(cp.nextSteps),
+      "@files_modified": jsonText(cp.filesModified),
+      "@embedding_blob": encodeEmbedding(cp.embedding ?? []),
+      "@token_estimate": cp.tokenEstimate ?? 0,
+      "@original_token_estimate": cp.originalTokenEstimate ?? null,
+      "@timestamp": cp.timestamp ?? 0,
+      "@dedup_status": "active",
+      "@compressed_original": cp.compressedOriginal ?? null,
+    });
+
+    // FTS5 virtual tables don't support UPSERT — delete any prior row, reinsert.
+    // Store normalized_text (the L1 verify key); fall back to summary for rows
+    // that predate normalized_text population.
+    db.prepare("DELETE FROM context_chunks_trgm WHERE id = ?").run(cp.checkpointId);
+    db.prepare(
+      "INSERT INTO context_chunks_trgm(id, normalized_text) VALUES(?, ?)",
+    ).run(cp.checkpointId, cp.normalizedText ?? cp.summary ?? "");
+  });
+}
+
+// --- Sprint 11: MinHash signatures + LSH buckets --------------------------
+
+/** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
+export function upsertMinhashSignature(
+  chunkId: string,
+  sessionId: string,
+  signatureVersion: number,
+  signatures: number[],
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  db.prepare(
+    `INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
+     VALUES(?, ?, ?, ?)
+     ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
+       session_id=excluded.session_id, signatures=excluded.signatures`,
+  ).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
+}
+
+/** Persist LSH bucket memberships for a chunk (one row per bucket key). */
+export function insertLshBuckets(
+  chunkId: string,
+  sessionId: string,
+  signatureVersion: number,
+  bucketKeys: string[],
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
+  const ins = db.prepare(
+    "INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)",
+  );
+  withTx(db, () => {
+    del.run(chunkId);
+    for (const key of bucketKeys) ins.run(key, chunkId, sid, signatureVersion);
+  });
+}
+
+/**
+ * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
+ * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
+ * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
+ */
+export function lshCandidateChunks(
+  bucketKeys: string[],
+  sessionId: string,
+  excludeChunkId: string,
+  stateDir: string = getStateDir(),
+  limit = 100,
+): string[] {
+  if (bucketKeys.length === 0) return [];
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const placeholders = bucketKeys.map(() => "?").join(",");
+  const rows = db
+    .prepare(
+      `SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
+       WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
+       LIMIT ?`,
+    )
+    .all(...bucketKeys, sid, excludeChunkId, limit) as { chunk_id: string }[];
+  return rows.map((r) => r.chunk_id);
+}
+
+/** All checkpoints for a session, sorted by id. */
+export function listCheckpoints(sessionId: string, stateDir: string = getStateDir()): StoredCheckpoint[] {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const rows = db
+    .prepare("SELECT * FROM context_chunks WHERE session_id = ? ORDER BY id ASC")
+    .all(sid) as any[];
+  return rows.map(rowToCheckpoint);
+}
+
+/** S25: the newest checkpoint timestamp for a session, or 0 when none. Used by
+ *  the RAPTOR freshness guard to reject a tree older than the live checkpoints. */
+export function maxCheckpointTimestamp(sessionId: string, stateDir: string = getStateDir()): number {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare("SELECT MAX(timestamp) AS mx FROM context_chunks WHERE session_id = ?")
+    .get(normalizeSessionId(sessionId)) as { mx: number | null } | undefined;
+  return Number(row?.mx ?? 0);
+}
+
+/** Next sequential checkpoint id (chkpt_001 …) for a session. */
+export function nextCheckpointId(sessionId: string, stateDir: string = getStateDir()): string {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const row = db
+    .prepare("SELECT MAX(CAST(SUBSTR(id, 7) AS INTEGER)) AS n FROM context_chunks WHERE session_id = ?")
+    .get(sid) as { n: number | null };
+  const next = (row.n ?? 0) + 1;
+  return `chkpt_${String(next).padStart(3, "0")}`;
+}
+
+/** True if a checkpoint id already exists for a session. */
+export function hasCheckpoint(sessionId: string, checkpointId: string, stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare("SELECT 1 FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+    .get(normalizeSessionId(sessionId), checkpointId);
+  return row !== undefined;
+}
+
+/** Fetch a single checkpoint by (session, id), or undefined if absent. */
+export function getCheckpoint(
+  sessionId: string,
+  checkpointId: string,
+  stateDir: string = getStateDir(),
+): StoredCheckpoint | undefined {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare("SELECT * FROM context_chunks WHERE session_id = ? AND id = ? LIMIT 1")
+    .get(normalizeSessionId(sessionId), checkpointId) as any;
+  return row ? rowToCheckpoint(row) : undefined;
+}
+
+/** Mark a checkpoint's dedup_status (e.g. 'removed' by SemDeDup). */
+export function setDedupStatus(
+  checkpointId: string,
+  sessionId: string,
+  status: string,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  db.prepare(
+    "UPDATE context_chunks SET dedup_status = ? WHERE id = ? AND session_id = ?",
+  ).run(status, checkpointId, normalizeSessionId(sessionId));
+}

package/src/store/sqlite/dedup-mirror.ts

@@ -0,0 +1,114 @@
+/**
+ * dedup-mirror.ts — S27 Task 6 dedup_mirror CRUD.
+ *
+ * Space-efficient deduplicated storage: each unique content_hash stores its
+ * bytes ONCE; raw_transcript rows reference this table via content_ref
+ * instead of storing duplicate content_bytes inline.
+ */
+import type { DatabaseSync } from "node:sqlite";
+
+/**
+ * Dedup mirror row (DB representation).
+ */
+export interface DedupMirrorRowDB {
+  content_hash: string;
+  content_bytes: string;
+  ref_count: number;
+  first_seen_seq: number;
+  created_at: number;
+}
+
+/**
+ * Upsert a row into dedup_mirror. If the hash already exists, increment ref_count.
+ * Returns true if this was a NEW unique content (first insert), false if it was a duplicate.
+ */
+export function upsertDedupMirror(
+  db: DatabaseSync,
+  contentHash: string,
+  contentBytes: string,
+  seq: number,
+): boolean {
+  const now = Date.now();
+  const existing = db
+    .prepare(`SELECT content_hash FROM dedup_mirror WHERE content_hash = @hash`)
+    .get({ "@hash": contentHash }) as { content_hash: string } | undefined;
+  if (existing) {
+    db.prepare(`UPDATE dedup_mirror SET ref_count = ref_count + 1 WHERE content_hash = @hash`).run({
+      "@hash": contentHash,
+    });
+    return false;
+  }
+  db.prepare(
+    `INSERT INTO dedup_mirror (content_hash, content_bytes, ref_count, first_seen_seq, created_at)
+     VALUES (@hash, @bytes, 1, @seq, @now)`,
+  ).run({
+    "@hash": contentHash,
+    "@bytes": contentBytes,
+    "@seq": seq,
+    "@now": now,
+  });
+  return true;
+}
+
+/**
+ * Get dedup ratio for a session: total bytes vs unique bytes.
+ */
+export function getDedupRatio(
+  db: DatabaseSync,
+  sessionId: string,
+): { totalBytes: number; uniqueBytes: number; ratio: number } {
+  const totalRow = db
+    .prepare(
+      `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS total
+       FROM raw_transcript
+       WHERE session_id = @session_id`,
+    )
+    .get({ "@session_id": sessionId }) as { total: number };
+  const uniqueRow = db
+    .prepare(
+      `SELECT COALESCE(SUM(LENGTH(content_bytes)), 0) AS unique_bytes
+       FROM dedup_mirror`,
+    )
+    .get() as { unique_bytes: number };
+  const totalBytes = totalRow.total;
+  const uniqueBytes = uniqueRow.unique_bytes;
+  const ratio = uniqueBytes > 0 ? totalBytes / uniqueBytes : 1;
+  return { totalBytes, uniqueBytes, ratio };
+}
+
+/**
+ * Get dedup mirror stats (diagnostic / test helper).
+ */
+export function getDedupMirrorStats(db: DatabaseSync): {
+  rowCount: number;
+  totalBytes: number;
+  avgRefCount: number;
+} {
+  const row = db
+    .prepare(
+      `SELECT COUNT(*) AS cnt,
+              COALESCE(SUM(LENGTH(content_bytes)), 0) AS total_bytes,
+              COALESCE(AVG(ref_count), 0) AS avg_ref
+       FROM dedup_mirror`,
+    )
+    .get() as { cnt: number; total_bytes: number; avg_ref: number };
+  return { rowCount: row.cnt, totalBytes: row.total_bytes, avgRefCount: row.avg_ref };
+}
+
+/**
+ * Update raw_transcript.content_ref to point to dedup_mirror.
+ */
+export function updateRawTranscriptRef(
+  db: DatabaseSync,
+  sessionId: string,
+  seq: number,
+  contentHash: string,
+): void {
+  db.prepare(
+    `UPDATE raw_transcript SET content_ref = @ref WHERE session_id = @sid AND seq = @seq`,
+  ).run({
+    "@ref": contentHash,
+    "@sid": sessionId,
+    "@seq": seq,
+  });
+}

package/src/store/sqlite/foundation.ts

@@ -0,0 +1,63 @@
+/**
+ * foundation.ts — future-feature foundation (resume sessions / daily log / lessons).
+ *
+ * Scaffolded tables + minimal helpers so all store data lives in SQLite from
+ * day one. Full UI/recall for these lands in later sprints.
+ */
+import { getStateDir, normalizeSessionId } from "../../store.js";
+import { openStore } from "./utils.js";
+
+/** Upsert a `sessions` row (resume + per-repo session history). */
+export function touchSession(
+  sessionId: string,
+  repo: string | undefined,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const sid = normalizeSessionId(sessionId);
+  const existing = db
+    .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
+    .get(sid) as { started_at: number | null } | undefined;
+  const now = Math.floor(Date.now() / 1000);
+  if (!existing) {
+    db.prepare(
+      `INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
+       VALUES(?, ?, ?, ?, 'active')`,
+    ).run(sid, repo ?? null, now, now);
+  } else {
+    db.prepare(
+      "UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?",
+    ).run(now, repo ?? null, sid);
+  }
+}
+
+/** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
+export function logDaily(
+  sessionId: string,
+  event: string,
+  detail: string | undefined,
+  tokensSaved: number,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const day = new Date().toISOString().slice(0, 10);
+  const now = Math.floor(Date.now() / 1000);
+  db.prepare(
+    `INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
+     VALUES(?, ?, ?, ?, ?, ?)`,
+  ).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
+}
+
+/** Append a `lessons` entry (future lessons-learned browse/recall). */
+export function addLesson(
+  sessionId: string,
+  repo: string | undefined,
+  lesson: string,
+  stateDir: string = getStateDir(),
+): void {
+  const db = openStore(stateDir);
+  const now = Math.floor(Date.now() / 1000);
+  db.prepare(
+    `INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`,
+  ).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
+}