pi-mega-compact 0.4.0

package/src/store/backfill.ts

@@ -0,0 +1,263 @@
+/**
+ * backfill.ts — resumable / idempotent hash backfill (Sprint 10).
+ *
+ * Purpose: populate `content_hash` / `content_hash2` / `content_hash_version` /
+ * `normalized_text` for any rows left with null hashes (e.g. pre-Sprint-9 data
+ * or rows that degraded to "store without dedup" under the QA #13 timeout).
+ * Structured so Sprint 11 can plug in its own MinHash/LSH phase after the
+ * content hashes land.
+ *
+ * Properties (QA #1 / QA #14):
+ *   - Resumable: progress stored in a `backfill_progress` table (last processed id).
+ *   - Idempotent: ON CONFLICT DO NOTHING + partial UNIQUE on (session_id, content_hash)
+ *     make a second run a no-op where it safely can.
+ *   - Batched: 1000 rows/commit to bound lock time; throttle between batches.
+ *
+ * SQLite is the source of truth; this touches no network (PREVENT-PI-004).
+ */
+
+import type { Database } from "better-sqlite3";
+import { openStore } from "./sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
+import { lshBands } from "../dedup/l1-lsh.js";
+import { upsertMinhashSignature, insertLshBuckets, listCheckpoints, saveRaptorTree } from "./sqlite.js";
+import { buildRaptorTree, type Leaf } from "../dedup/raptor/tree.js";
+import type { Embedder } from "../embedder.js";
+import { defaultEmbedder } from "../embedder.js";
+import { getStateDir } from "../store.js";
+
+const BATCH = 1000;
+const THROTTLE_MS = 0; // synchronous backfill; no cross-process yield needed
+
+/** Backfill phases, in order (Sprint 14 full-pipeline wiring). */
+export type BackfillPhase = "L0" | "L1" | "L2" | "RAPTOR";
+
+interface BackfillResult {
+  processed: number;
+  updated: number;
+  duplicatesResolved: number;
+}
+
+interface PhaseProgressRow {
+  last_session_id: string | null;
+  last_id: string | null;
+  processed: number;
+}
+
+function ensureProgressTable(db: Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS backfill_progress (
+      name TEXT PRIMARY KEY,
+      last_session_id TEXT, -- session of the highest (session_id, id) scanned
+      last_id TEXT,         -- highest context_chunks.id scanned within that session
+      updated INTEGER,
+      duplicates_resolved INTEGER
+    );
+  `);
+}
+
+function progress(db: Database): { lastSid: string | null; lastId: string | null; updated: number; dups: number } {
+  const row = db
+    .prepare("SELECT last_session_id, last_id, updated, duplicates_resolved FROM backfill_progress WHERE name='content_hashes'")
+    .get() as { last_session_id: string | null; last_id: string | null; updated: number; duplicates_resolved: number } | undefined;
+  return { lastSid: row?.last_session_id ?? null, lastId: row?.last_id ?? null, updated: row?.updated ?? 0, dups: row?.duplicates_resolved ?? 0 };
+}
+
+/**
+ * Backfill content hashes for all rows missing them, from the last scanned
+ * (session_id, id) forward. Returns counts; fully idempotent and resumable.
+ */
+export function backfillContentHashes(stateDir: string = getStateDir()): BackfillResult {
+  const db = openStore(stateDir);
+  ensureProgressTable(db);
+  const start = progress(db);
+
+  // Rows needing hashing: null content_hash, ordered by (session_id, id) for a
+  // stable, resumable cursor (ids are only unique per session).
+  const pending = db
+    .prepare(
+      `SELECT id, session_id, summary FROM context_chunks
+       WHERE content_hash IS NULL
+         AND (session_id > COALESCE(?, '')
+              OR (session_id = COALESCE(?, '') AND id > COALESCE(?, '')))
+       ORDER BY session_id ASC, id ASC LIMIT ?`,
+    )
+    .all(start.lastSid, start.lastSid, start.lastId, BATCH) as { id: string; session_id: string; summary: string }[];
+
+  let updated = start.updated;
+  let duplicatesResolved = start.dups;
+  let processed = 0;
+  let lastSid = start.lastSid;
+  let lastId = start.lastId;
+
+  const tx = db.transaction((rows: { id: string; session_id: string; summary: string }[]) => {
+    const lookup = db.prepare(
+      "SELECT id FROM context_chunks WHERE session_id = ? AND content_hash = ? AND content_hash2 = ? AND id != ? LIMIT 1",
+    );
+    const update = db.prepare(
+      `UPDATE context_chunks
+       SET content_hash=?, content_hash2=?, content_hash_version=?, normalized_text=?,
+           dedup_status='active'
+       WHERE id=?`,
+    );
+    for (const row of rows) {
+      const digest = computeContentDigest(row.summary ?? "");
+      // Keep the oldest row on a collision (partial UNIQUE would reject the
+      // newer insert); mark the newer one as superseded-without-store.
+      const clash = lookup.get(row.session_id, digest.contentHash, digest.contentHash2, row.id);
+      if (clash) {
+        db.prepare("UPDATE context_chunks SET dedup_status='dup-resolved' WHERE id=?").run(row.id);
+        duplicatesResolved++;
+      } else {
+        update.run(
+          digest.contentHash,
+          digest.contentHash2,
+          digest.contentHashVersion,
+          digest.normalizedText,
+          row.id,
+        );
+        updated++;
+      }
+      lastSid = row.session_id;
+      lastId = row.id;
+      processed++;
+    }
+  });
+
+  if (pending.length > 0) {
+    tx(pending);
+    db.prepare(
+      "INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved) VALUES('content_hashes',?,?,?,?) ON CONFLICT(name) DO UPDATE SET last_session_id=excluded.last_session_id, last_id=excluded.last_id, updated=excluded.updated, duplicates_resolved=excluded.duplicates_resolved",
+    ).run(lastSid, lastId, updated, duplicatesResolved);
+  }
+
+  if (THROTTLE_MS > 0) {
+    // No-op in this synchronous build; placeholder for future streaming backfill.
+  }
+
+  return { processed, updated, duplicatesResolved };
+}
+
+/** True when no rows remain pending (backfill complete for this state dir). */
+export function isBackfillComplete(stateDir: string = getStateDir()): boolean {
+  const db = openStore(stateDir);
+  const row = db
+    .prepare("SELECT COUNT(*) AS c FROM context_chunks WHERE content_hash IS NULL")
+    .get() as { c: number };
+  return row.c === 0;
+}
+
+// ---- Sprint 14: L1 / L2 / RAPTOR phase backfill (resumable) ---------------
+
+function phaseCursor(db: Database, phase: BackfillPhase): { lastId: string | null; processed: number } {
+  ensureProgressTable(db);
+  const row = db
+    .prepare("SELECT last_id, updated AS processed FROM backfill_progress WHERE name = ?")
+    .get(`phase_${phase}`) as PhaseProgressRow | undefined;
+  return { lastId: row?.last_id ?? null, processed: row?.processed ?? 0 };
+}
+
+function savePhaseCursor(db: Database, phase: BackfillPhase, lastId: string | null, processed: number): void {
+  db.prepare(
+    `INSERT INTO backfill_progress(name, last_session_id, last_id, updated, duplicates_resolved)
+     VALUES(?, NULL, ?, ?, 0)
+     ON CONFLICT(name) DO UPDATE SET last_id=excluded.last_id, updated=excluded.updated`,
+  ).run(`phase_${phase}`, lastId, processed);
+}
+
+export interface PhaseBackfillResult {
+  phase: BackfillPhase;
+  processed: number;
+  batches: number;
+  interrupted: boolean;
+  /** Last processed checkpoint id (the resume cursor). */
+  cursor: string | undefined;
+}
+
+/**
+ * Backfill L1 (MinHash sigs + LSH buckets) or L2 (MinHash sigs only) derived
+ * data for a session, in batches, resumable from the persisted cursor. Pass
+ * `interruptAfterBatches` to simulate a crash for resume testing.
+ */
+export function backfillPhase(
+  phase: "L1" | "L2",
+  sessionId: string,
+  stateDir: string,
+  opts: { batchSize?: number; interruptAfterBatches?: number } = {},
+): PhaseBackfillResult {
+  const db = openStore(stateDir);
+  const batchSize = opts.batchSize ?? BATCH;
+  const all = listCheckpoints(sessionId, stateDir).sort((a, b) =>
+    a.checkpointId.localeCompare(b.checkpointId),
+  );
+  const { lastId } = phaseCursor(db, phase);
+  let { processed } = phaseCursor(db, phase);
+  const startIndex = lastId ? all.findIndex((c) => c.checkpointId === lastId) + 1 : 0;
+
+  let batches = 0;
+  let interrupted = false;
+  let cursor: string | undefined = lastId ?? undefined;
+
+  for (let i = Math.max(0, startIndex); i < all.length; i += batchSize) {
+    const batch = all.slice(i, i + batchSize);
+    const tx = db.transaction(() => {
+      for (const cp of batch) {
+        const sig = minhashSignature(cp.normalizedText ?? cp.summary ?? "");
+        if (sig.length === NUM_HASHES) {
+          upsertMinhashSignature(cp.checkpointId, sessionId, SIGNATURE_VERSION, sig, stateDir);
+          if (phase === "L1") {
+            insertLshBuckets(
+              cp.checkpointId, sessionId, SIGNATURE_VERSION,
+              lshBands(sig, sessionId, SIGNATURE_VERSION), stateDir,
+            );
+          }
+        }
+        cursor = cp.checkpointId;
+        processed++;
+      }
+    });
+    tx();
+    savePhaseCursor(db, phase, cursor ?? null, processed);
+    batches++;
+    if (THROTTLE_MS > 0) { const end = Date.now() + THROTTLE_MS; while (Date.now() < end) { /* throttle */ } }
+    if (opts.interruptAfterBatches && batches >= opts.interruptAfterBatches) {
+      interrupted = true;
+      break;
+    }
+  }
+
+  return { phase, processed, batches, interrupted, cursor };
+}
+
+/**
+ * Backfill the RAPTOR tree for a session (single pass over all leaves). Builds
+ * + persists raptor_nodes. Not batched — the builder has its own budget cap.
+ */
+export function backfillRaptor(
+  sessionId: string,
+  stateDir: string,
+  embedder: Embedder = defaultEmbedder(),
+): PhaseBackfillResult {
+  const all = listCheckpoints(sessionId, stateDir).sort((a, b) =>
+    a.checkpointId.localeCompare(b.checkpointId),
+  );
+  const leaves: Leaf[] = all.map((cp) => {
+    const text = cp.normalizedText ?? cp.summary ?? "";
+    return {
+      id: cp.checkpointId,
+      messages: [{ role: "user", text }],
+      sourceText: text,
+      embedding: embedder.embed(text),
+    };
+  });
+  if (leaves.length === 0) {
+    return { phase: "RAPTOR", processed: 0, batches: 0, interrupted: false, cursor: undefined };
+  }
+  const tree = buildRaptorTree(leaves, { embedder });
+  saveRaptorTree(sessionId, tree, stateDir);
+  const db = openStore(stateDir);
+  ensureProgressTable(db);
+  savePhaseCursor(db, "RAPTOR", leaves[leaves.length - 1].id, leaves.length);
+  return { phase: "RAPTOR", processed: leaves.length, batches: 1, interrupted: false, cursor: leaves[leaves.length - 1].id };
+}

package/src/store/bloom.ts

@@ -0,0 +1,122 @@
+/**
+ * bloom.ts — local bloom-filter accelerator for the L0 content-hash dedup tier
+ * (Sprint 10).
+ *
+ * ACCELERATOR ONLY (QA #2 spirit, re-mapped locally): a bloom filter has zero
+ * false negatives — a MISS truly means "this content_hash is not present", so we
+ * can skip the full SQLite scan on the happy path. A HIT is only a candidate and
+ * MUST be confirmed by a SELECT against SQLite, which remains the source of truth
+ * (PREVENT-PI-004: in-process, no network; SQLite owns durability).
+ *
+ * The filter is an in-memory `bloom-filters` Map persisted to
+ * `STATE_DIR/bloom.json.gz` so a fresh VectorStore over the same dir reuses the
+ * warm filter instead of rebuilding from a scan.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { getStateDir } from "../store.js";
+import { compressSmart, decompressSmart } from "../store.js";
+
+// Fixed bit-array size + hash count sized for a 1K-checkpoint fixture at <1% FP
+// (m ≈ -n*ln(p)/ln(2)^2). 8 KiB bits → ~8192 bits, k=7 → well under 1% at 1K.
+const BITS = 8192;
+const HASHES = 7;
+const STORAGE_MARK = 0x42; // 'B' — marks a persisted bloom blob (not versioned)
+
+function fnv1a(data: Buffer, seed: number): number {
+  let h = 0x811c9dc5 ^ seed;
+  for (let i = 0; i < data.length; i++) {
+    h ^= data[i];
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+export class BloomFilter {
+  private bits: Uint8Array;
+
+  constructor(bits?: Uint8Array) {
+    this.bits = bits ?? new Uint8Array(BITS);
+  }
+
+  private indices(key: string): number[] {
+    const data = Buffer.from(key, "utf-8");
+    const idx: number[] = [];
+    for (let i = 0; i < HASHES; i++) {
+      // Double-hashing (Kirsch–Mitzenmacher) to derive k independent positions.
+      const h1 = fnv1a(data, 0x9e3779b1 * i);
+      const h2 = fnv1a(data, 0x85ebca77 * (i + 1));
+      idx.push((h1 + i * h2) % BITS);
+    }
+    return idx;
+  }
+
+  add(key: string): void {
+    for (const i of this.indices(key)) this.bits[i >> 3] |= 1 << (i & 7);
+  }
+
+  /** A miss is definitive (zero false negatives): false ⇒ definitely absent. */
+  maybeHas(key: string): boolean {
+    for (const i of this.indices(key)) {
+      if ((this.bits[i >> 3] & (1 << (i & 7))) === 0) return false;
+    }
+    return true;
+  }
+
+  toBuffer(): Buffer {
+    return Buffer.concat([Buffer.from([STORAGE_MARK]), Buffer.from(this.bits)]);
+  }
+
+  /** Raw bit array (for persistence). */
+  bytes(): Uint8Array {
+    return this.bits;
+  }
+
+  static fromBuffer(buf: Buffer): BloomFilter {
+    if (buf.length >= 1 && buf[0] === STORAGE_MARK) {
+      return new BloomFilter(Uint8Array.from(buf.subarray(1)));
+    }
+    // Legacy/compressed form: best-effort decompress.
+    try {
+      const raw = decompressSmart(buf);
+      return new BloomFilter(Uint8Array.from(raw));
+    } catch {
+      return new BloomFilter();
+    }
+  }
+}
+
+const cache = new Map<string, BloomFilter>();
+
+/** Load (or lazily create + cache) the bloom filter for a state dir. */
+export function openBloom(stateDir: string = getStateDir()): BloomFilter {
+  const existing = cache.get(stateDir);
+  if (existing) return existing;
+  const path = join(stateDir, "bloom.json.gz");
+  let filter = new BloomFilter();
+  if (existsSync(path)) {
+    try {
+      filter = BloomFilter.fromBuffer(readFileSync(path));
+    } catch {
+      filter = new BloomFilter();
+    }
+  }
+  cache.set(stateDir, filter);
+  return filter;
+}
+
+/** Persist the bloom filter to disk (additive — does not clear the cache). */
+export function saveBloom(stateDir: string = getStateDir()): void {
+  const filter = cache.get(stateDir);
+  if (!filter) return;
+  if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true });
+  // Compress the raw bit array for a smaller, versioned-on-disk footprint.
+  const blob = compressSmart(Buffer.from(filter.bytes()));
+  writeFileSync(join(stateDir, "bloom.json.gz"), blob);
+}
+
+/** Evict the cached filter (test teardown only). */
+export function closeBloom(stateDir: string): void {
+  cache.delete(stateDir);
+}

package/src/store/compression.test.ts

@@ -0,0 +1,83 @@
+/**
+ * compression.test.ts — versioned compression tiers + backward compatibility.
+ *
+ * Proves Sprint 8's root-cause fix: the 0x03 tag collision is impossible because
+ * new blobs carry a 2-byte version magic, and legacy blobs (untagged gzip, legacy
+ * single-tag incl. the old 0x03=brotli) still decompress.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { gzipSync, brotliCompressSync } from "node:zlib";
+import {
+  compressSmart,
+  decompressSmart,
+  compressZstd,
+  decompressZstd,
+  isVersioned,
+  detectFormat,
+  decompressSyncAuto,
+} from "./compression.js";
+
+const buf = (s: string, n: number) => Buffer.from(s.repeat(n));
+
+test("versioned format: all size tiers roundtrip and are versioned", () => {
+  const cases = [
+    { label: "tiny", data: buf("x", 100), expectTag: 0x00 },
+    { label: "small", data: buf("a", 600), expectTag: 0x01 },
+    { label: "medium", data: buf("b", 8000), expectTag: 0x02 },
+    { label: "large", data: buf("c", 40000), expectTag: 0x05 },
+  ];
+  for (const { label, data, expectTag } of cases) {
+    const c = compressSmart(data);
+    // New 2-byte magic header present.
+    assert.ok(isVersioned(c), `${label}: should be versioned (0xEC 0x01)`);
+    assert.equal(c[0], 0xec, `${label}: magic hi`);
+    assert.equal(c[1], 0x01, `${label}: magic lo (version)`);
+    assert.equal(c[2], 0x01, `${label}: format version 1`);
+    assert.equal(c[3], expectTag, `${label}: tier tag ${expectTag.toString(16)}`);
+    // Roundtrips exactly.
+    assert.deepEqual(decompressSmart(c), data, `${label}: roundtrip`);
+  }
+});
+
+test("large tier actually compresses better than raw (brotli 0x05)", () => {
+  const data = buf("this is a long summary of a coding session. ", 900);
+  const c = compressSmart(data);
+  assert.ok(c.length < data.length, "compressed smaller than raw");
+  assert.equal(c[3], 0x05, "tag is brotli-4");
+  assert.deepEqual(decompressSmart(c), data);
+});
+
+test("legacy untagged gzip (0x1f magic) still decompresses", () => {
+  const data = Buffer.from(JSON.stringify({ legacy: true }));
+  const legacyGzip = gzipSync(data); // no tag byte, starts with 0x1f
+  assert.equal(legacyGzip[0], 0x1f, "gzip magic present");
+  assert.equal(detectFormat(legacyGzip), "legacy-gzip");
+  assert.deepEqual(JSON.parse(decompressSmart(legacyGzip).toString()), { legacy: true });
+});
+
+test("legacy single-tag 0x03=brotli (the collision case) still decompresses", () => {
+  const data = buf("legacy brotli payload ", 200);
+  // Reconstruct the EXACT v0.1.0 legacy brotli frame: tag 0x03 + brotli payload.
+  const legacy = Buffer.concat([Buffer.from([0x03]), brotliCompressSync(data)]);
+  assert.equal(detectFormat(legacy), "legacy-tag");
+  assert.deepEqual(decompressSmart(legacy), data, "legacy 0x03 brotli roundtrips");
+});
+
+test("detectFormat classifies all eras", () => {
+  assert.equal(detectFormat(compressSmart(buf("q", 700))), "versioned");
+  assert.equal(detectFormat(gzipSync(buf("q", 10))), "legacy-gzip");
+  assert.equal(detectFormat(Buffer.from([0x00, 1, 2, 3])), "legacy-tag");
+  assert.equal(detectFormat(Buffer.from([0x99, 0x88])), "unknown");
+});
+
+test("zstd helper roundtrips (async) and is not sync-decoded", async () => {
+  const data = buf("zstd dr export payload ", 1500);
+  const c = await compressZstd(data);
+  assert.ok(c.length < data.length, "zstd compresses");
+  // decompressSyncAuto reports zstd without throwing (caller awaits decompressZstd).
+  const auto = decompressSyncAuto(c);
+  assert.equal(auto.isZstd, true, "flagged as zstd");
+  assert.deepEqual(await decompressZstd(c), data, "zstd roundtrip");
+});

package/src/store/compression.ts

@@ -0,0 +1,203 @@
+/**
+ * compression.ts — versioned, size-adaptive compression for checkpoint blobs.
+ *
+ * Extracted from store.ts (Sprint 8). Two coordinated compressors:
+ *
+ *  1. `compressSmart` / `decompressSmart` — SYNCHRONOUS, zlib-based. Used by the
+ *     VectorStore write path (which must stay synchronous — see Sprint 8 plan:
+ *     better-sqlite3 replaced PGlite precisely to avoid an async cascade).
+ *
+ *  2. `compressZstd` / `decompressZstd` — ASYNCHRONOUS, via @mongodb-js/zstd.
+ *     Optional, used for DR-export / large-blob paths where an await is fine.
+ *
+ * FORMAT-VERSION PROBLEM (root cause of Sprint 8):
+ *   store.ts shipped `0x03` = brotli (legacy single-tag format). PLAN.md reassigns
+ *   `0x03` → zstd, which would corrupt every existing checkpoint file. We fix this
+ *   with a 2-byte magic header on the NEW format so the tag byte is namespaced and
+ *   can never collide with legacy payloads:
+ *
+ *     NEW (versioned):  0xEC 0x01 [TIER_TAG] [payload]
+ *     LEGACY single-tag: [TIER_TAG] [payload]   (tags 0x00..0x03)
+ *     LEGACY untagged:   0x1f ...               (plain gzip magic)
+ *
+ * `0xEC` is chosen because it collides with no zlib output: gzip magic is 0x1f,
+ * brotli streams start 0xCE/0xCF, zlib/deflate streams start 0x78/0x05/0x03.
+ * decompressSmart detects the magic first, so all three eras roundtrip together.
+ */
+
+import {
+  gzipSync,
+  gunzipSync,
+  brotliCompressSync,
+  brotliDecompressSync,
+  constants as zlibConstants,
+} from "node:zlib";
+import zstd from "@mongodb-js/zstd";
+
+// --- Versioned format markers ----------------------------------------------
+const MAGIC_HI = 0xec;
+const MAGIC_LO = 0x01; // format version 1
+
+// Tier tags (only meaningful inside the 0xEC 0x01 versioned frame).
+const TAG_RAW = 0x00; // no compression (< 512 bytes)
+const TAG_GZIP_1 = 0x01; // gzip level 1 (fast, 512B–4KB)
+const TAG_GZIP_6 = 0x02; // gzip level 6 (default, 4KB–32KB)
+const TAG_BROTLI_4 = 0x05; // brotli level 4 (> 32KB, best text ratio, sync)
+
+// Reserved for the async zstd helper (see compressZstd). Not used by the sync path.
+const TAG_ZSTD_3 = 0x03;
+const TAG_ZSTD_9 = 0x04;
+
+/** Gzip magic byte — used to detect legacy untagged files. */
+const GZIP_MAGIC = 0x1f;
+
+const SIZE_TINY = 512;
+const SIZE_SMALL = 4096;
+const SIZE_MEDIUM = 32768;
+
+function header(ver: number, tag: number): Buffer {
+  return Buffer.from([MAGIC_HI, MAGIC_LO, ver, tag]);
+}
+
+/**
+ * Compress synchronously using the best zlib tier for the payload size.
+ *
+ * Tiers (all synchronous — no network, no async, PREVENT-PI-004):
+ *   < 512 B  → raw         (tag 0x00)
+ *   512B–4KB → gzip level 1 (tag 0x01)
+ *   4KB–32KB → gzip level 6 (tag 0x02)
+ *   > 32 KB  → brotli 4     (tag 0x05)
+ *
+ * Writes the versioned header so readers disambiguate from legacy blobs.
+ */
+export function compressSmart(data: Buffer): Buffer {
+  const len = data.length;
+  if (len < SIZE_TINY) {
+    return Buffer.concat([header(1, TAG_RAW), data]);
+  }
+  if (len < SIZE_SMALL) {
+    return Buffer.concat([header(1, TAG_GZIP_1), gzipSync(data, { level: 1 })]);
+  }
+  if (len < SIZE_MEDIUM) {
+    return Buffer.concat([header(1, TAG_GZIP_6), gzipSync(data, { level: 6 })]);
+  }
+  const compressed = brotliCompressSync(data, {
+    params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+  });
+  return Buffer.concat([header(1, TAG_BROTLI_4), compressed]);
+}
+
+/** True when `buf` is a versioned-format blob (0xEC 0x01 …). */
+export function isVersioned(buf: Buffer): boolean {
+  return buf.length >= 2 && buf[0] === MAGIC_HI && buf[1] === MAGIC_LO;
+}
+
+/** Detect which format era a buffer belongs to (for tests/telemetry). */
+export type CompressedFormat = "versioned" | "legacy-tag" | "legacy-gzip" | "unknown";
+export function detectFormat(buf: Buffer): CompressedFormat {
+  if (isVersioned(buf)) return "versioned";
+  if (buf[0] === GZIP_MAGIC) return "legacy-gzip";
+  // Legacy single-tag: first byte is a known legacy tag.
+  if (buf[0] === 0x00 || buf[0] === 0x01 || buf[0] === 0x02 || buf[0] === 0x03) {
+    return "legacy-tag";
+  }
+  return "unknown";
+}
+
+/**
+ * Decompress a buffer written by `compressSmart` (versioned) OR any legacy
+ * format still on disk (legacy single-tag, legacy untagged gzip). SYNCHRONOUS.
+ *
+ * Throws on zstd blobs — those must go through the async `decompressZstd`,
+ * because zstd decompression cannot be awaited inside this sync path.
+ */
+export function decompressSmart(buf: Buffer): Buffer {
+  if (buf.length === 0) return buf;
+
+  // New versioned format — dispatch on the namespaced tier tag.
+  if (isVersioned(buf)) {
+    const tag = buf[3];
+    const payload = buf.subarray(4);
+    switch (tag) {
+      case TAG_RAW:
+        return payload;
+      case TAG_GZIP_1:
+      case TAG_GZIP_6:
+        return gunzipSync(payload);
+      case TAG_BROTLI_4:
+        return brotliDecompressSync(payload);
+      case TAG_ZSTD_3:
+      case TAG_ZSTD_9:
+        throw new Error(
+          "decompressSmart cannot read zstd blobs (async only) — use decompressZstd",
+        );
+      default:
+        throw new Error(`decompressSmart: unknown versioned tier tag 0x${tag.toString(16)}`);
+    }
+  }
+
+  // Legacy untagged gzip file (old writeGzJson with no tag byte).
+  if (buf[0] === GZIP_MAGIC) {
+    return gunzipSync(buf);
+  }
+
+  // Legacy single-tag format (store.ts v0.1.0): tags 0x00..0x03.
+  const tag = buf[0];
+  const payload = buf.subarray(1);
+  switch (tag) {
+    case 0x00: // TAG_RAW (legacy)
+      return payload;
+    case 0x01: // TAG_GZIP_1 (legacy)
+    case 0x02: // TAG_GZIP_6 (legacy)
+      return gunzipSync(payload);
+    case 0x03: // TAG_BROTLI (legacy) — the very collision this format fixes
+      return brotliDecompressSync(payload);
+    default:
+      // Unknown legacy tag — last-ditch try plain gzip.
+      return gunzipSync(buf);
+  }
+}
+
+// --- Optional async zstd path (DR export / large blobs) --------------------
+// Self-describing: own 2-byte marker so it never routes through decompressSmart.
+const ZSTD_MAGIC_HI = 0x5a; // 'Z'
+const ZSTD_MAGIC_LO = 0x53; // 'S'
+
+async function compressZstdWithLevel(data: Buffer, level: number): Promise<Buffer> {
+  const compressed = await zstd.compress(data, level);
+  return Buffer.concat([Buffer.from([ZSTD_MAGIC_HI, ZSTD_MAGIC_LO]), compressed]);
+}
+
+/** Compress with zstd level 3 (fast, balanced). Async. */
+export function compressZstd(data: Buffer): Promise<Buffer> {
+  return compressZstdWithLevel(data, 3);
+}
+
+/** Compress with zstd level 9 (max ratio). Async. */
+export function compressZstdMax(data: Buffer): Promise<Buffer> {
+  return compressZstdWithLevel(data, 9);
+}
+
+/** True when a buffer is a zstd-compressed blob from this helper. */
+export function isZstd(buf: Buffer): boolean {
+  return buf.length >= 2 && buf[0] === ZSTD_MAGIC_HI && buf[1] === ZSTD_MAGIC_LO;
+}
+
+/** Decompress a zstd blob produced by compressZstd/compressZstdMax. Async. */
+export async function decompressZstd(buf: Buffer): Promise<Buffer> {
+  if (buf.length === 0) return buf;
+  if (!isZstd(buf)) {
+    throw new Error("decompressZstd: buffer is not a zstd blob (missing ZS marker)");
+  }
+  return zstd.decompress(buf.subarray(2));
+}
+
+/**
+ * Decompress anything we can WITHOUT awaiting: versioned + legacy zlib formats.
+ * zstd blobs are detected and reported (not thrown blindly) so callers can
+ * decide whether to await decompressZstd.
+ */
+export function decompressSyncAuto(buf: Buffer): { data: Buffer; isZstd: boolean } {
+  if (isZstd(buf)) return { data: buf, isZstd: true };
+  return { data: decompressSmart(buf), isZstd: false };
+}