pi-mega-compact 0.7.9 → 0.8.0

package/dist/src/store/sqlite/index-store.js

@@ -1,167 +0,0 @@
-/**
- * Global machine-wide index store (Phase 5b).
- *
- * A single SQLite DB, separate from every per-repo store, that aggregates one
- * row per repo this machine has run on. The multi-repo dashboard (Summary /
- * All-repos tabs) reads it so ONE dashboard can show every repo's checkpoints,
- * tokens saved, and active model — instead of a per-repo dashboard that only
- * ever sees the repo it was launched from.
- *
- * Written by every pi process on repo-switch (bindRepo) + model capture; read by
- * the dashboard server. Concurrency across 10+ pi processes is handled by WAL +
- * infrequent idempotent upserts (ON CONFLICT). Fully local (PREVENT-PI-004).
- */
-import { DatabaseSync } from "node:sqlite";
-import { existsSync, mkdirSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { join } from "node:path";
-/** Resolve the machine-wide index directory (env-overridable). */
-export function getIndexDir() {
-    const override = process.env.MEGACOMPACT_INDEX_DIR;
-    if (override && override.trim() !== "")
-        return override;
-    // homedir() can throw in exotic sandboxes; fall back to tmpdir.
-    try {
-        return join(homedir(), ".mega-compact-index");
-    }
-    catch {
-        return join(tmpdir(), ".mega-compact-index");
-    }
-}
-let indexCache;
-let indexCacheDir;
-/** Open (or reuse) the machine-wide index DB. WAL for concurrent writers. */
-export function openIndexStore(indexDir = getIndexDir()) {
-    if (indexCache && indexCacheDir === indexDir)
-        return indexCache;
-    if (!existsSync(indexDir))
-        mkdirSync(indexDir, { recursive: true });
-    const iddb = new DatabaseSync(join(indexDir, "index.sqlite"));
-    iddb.exec("PRAGMA journal_mode = WAL");
-    iddb.exec("PRAGMA busy_timeout = 3000"); // tolerate brief cross-process write contention
-    iddb.exec(`
-    CREATE TABLE IF NOT EXISTS repo_registry (
-      repo_root                 TEXT PRIMARY KEY,
-      display_name              TEXT,
-      state_dir                 TEXT NOT NULL,
-      first_seen                INTEGER,
-      last_seen                 INTEGER,
-      last_compacted_at         INTEGER,
-      checkpoint_count          INTEGER DEFAULT 0,
-      tokens_saved              INTEGER DEFAULT 0,
-      compressed_original_bytes INTEGER DEFAULT 0,
-      provider                  TEXT,
-      provider_name             TEXT,
-      model_name                TEXT,
-      input_rate                REAL,
-      output_rate               REAL,
-      model_captured_at         INTEGER
-    );
-    CREATE INDEX IF NOT EXISTS idx_registry_last_seen ON repo_registry(last_seen DESC);
-  `);
-    indexCache = iddb;
-    indexCacheDir = indexDir;
-    return iddb;
-}
-/**
- * Upsert a repo's aggregate stats into the global index. Called on repo-switch
- * (infrequent). Preserves first_seen + the model columns on update (model is
- * written separately by recordRepoModel so we never clobber it here with nulls).
- */
-export function upsertRepoRegistry(row, indexDir = getIndexDir()) {
-    const db = openIndexStore(indexDir);
-    const now = Date.now();
-    db.prepare(`INSERT INTO repo_registry
-       (repo_root, display_name, state_dir, first_seen, last_seen, last_compacted_at,
-        checkpoint_count, tokens_saved, compressed_original_bytes)
-     VALUES (@repo_root, @display_name, @state_dir, @now, @now, @last_compacted_at,
-             @checkpoint_count, @tokens_saved, @compressed_original_bytes)
-     ON CONFLICT(repo_root) DO UPDATE SET
-       display_name = excluded.display_name,
-       state_dir = excluded.state_dir,
-       last_seen = excluded.last_seen,
-       last_compacted_at = COALESCE(excluded.last_compacted_at, repo_registry.last_compacted_at),
-       checkpoint_count = excluded.checkpoint_count,
-       tokens_saved = excluded.tokens_saved,
-       compressed_original_bytes = excluded.compressed_original_bytes`).run({
-        repo_root: row.repoRoot,
-        display_name: row.displayName,
-        state_dir: row.stateDir,
-        now,
-        last_compacted_at: row.lastCompactedAt ?? null,
-        checkpoint_count: row.checkpointCount,
-        tokens_saved: row.tokensSaved,
-        compressed_original_bytes: row.compressedOriginalBytes,
-    });
-}
-/**
- * Record the active model/provider for a repo in the global index (denormalized
- * so the All-repos table shows model without opening each repo's DB). Upserts a
- * bare registry row if the repo isn't registered yet.
- */
-export function recordRepoModel(repoRoot, model, indexDir = getIndexDir()) {
-    const db = openIndexStore(indexDir);
-    const now = Date.now();
-    db.prepare(`INSERT INTO repo_registry
-       (repo_root, display_name, state_dir, first_seen, last_seen,
-        provider, provider_name, model_name, input_rate, output_rate, model_captured_at)
-     VALUES (@repo_root, @display_name, @state_dir, @now, @now,
-             @provider, @provider_name, @model_name, @input_rate, @output_rate, @now)
-     ON CONFLICT(repo_root) DO UPDATE SET
-       last_seen = excluded.last_seen,
-       provider = excluded.provider,
-       provider_name = excluded.provider_name,
-       model_name = excluded.model_name,
-       input_rate = excluded.input_rate,
-       output_rate = excluded.output_rate,
-       model_captured_at = excluded.model_captured_at`).run({
-        repo_root: repoRoot,
-        display_name: model.displayName,
-        state_dir: model.stateDir,
-        now,
-        provider: model.provider,
-        provider_name: model.providerName,
-        model_name: model.modelName,
-        input_rate: model.inputRate,
-        output_rate: model.outputRate,
-    });
-}
-function mapRegistryRow(row) {
-    return {
-        repoRoot: row.repo_root,
-        displayName: row.display_name ?? "",
-        stateDir: row.state_dir,
-        firstSeen: row.first_seen ?? 0,
-        lastSeen: row.last_seen ?? 0,
-        lastCompactedAt: row.last_compacted_at ?? null,
-        checkpointCount: row.checkpoint_count ?? 0,
-        tokensSaved: row.tokens_saved ?? 0,
-        compressedOriginalBytes: row.compressed_original_bytes ?? 0,
-        provider: row.provider ?? null,
-        providerName: row.provider_name ?? null,
-        modelName: row.model_name ?? null,
-        inputRate: row.input_rate ?? null,
-        outputRate: row.output_rate ?? null,
-        modelCapturedAt: row.model_captured_at ?? null,
-    };
-}
-/** All registered repos, most-recently-seen first. */
-export function listRepoRegistry(indexDir = getIndexDir()) {
-    const db = openIndexStore(indexDir);
-    const rows = db.prepare("SELECT * FROM repo_registry ORDER BY last_seen DESC").all();
-    return rows.map(mapRegistryRow);
-}
-/** A single repo's registry row, or undefined. */
-export function getRepoRegistry(repoRoot, indexDir = getIndexDir()) {
-    const db = openIndexStore(indexDir);
-    const row = db.prepare("SELECT * FROM repo_registry WHERE repo_root = ?").get(repoRoot);
-    return row ? mapRegistryRow(row) : undefined;
-}
-/** Close the cached index connection (test teardown only). */
-export function closeIndexStore() {
-    if (indexCache) {
-        indexCache.close();
-        indexCache = undefined;
-        indexCacheDir = undefined;
-    }
-}

package/dist/src/store/sqlite/memory.js

@@ -1,54 +0,0 @@
-/**
- * Durable "save to memory" store (taken over from memory extensions).
- *
- * One SQLite store for user-saved memories, scoped by repo. Mirrors the
- * lessons/sessions pattern: all state lives in SQLite from day one. All queries
- * are parameterized (PREVENT-002).
- */
-import { getStateDir } from "../../store.js";
-import { openStore } from "./connection.js";
-/** Save a memory to the current repo's store. Returns the new row id. */
-export function addMemory(memory, repo, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const now = Math.floor(Date.now() / 1000);
-    const res = db
-        .prepare(`INSERT INTO memories(repo, kind, content, tags, created_at, last_recalled_at)
-       VALUES(?, ?, ?, ?, ?, NULL)`)
-        .run(repo ?? null, memory.kind ?? "note", memory.content, JSON.stringify(memory.tags ?? []), now);
-    return Number(res.lastInsertRowid);
-}
-/** List recent memories for a repo (or all repos when repo is null). */
-export function listMemories(repo, limit = 50, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const rows = repo
-        ? db.prepare("SELECT * FROM memories WHERE repo = ? ORDER BY created_at DESC LIMIT ?").all(repo, limit)
-        : db.prepare("SELECT * FROM memories ORDER BY created_at DESC LIMIT ?").all(limit);
-    return rows.map(mapMemoryRow);
-}
-/** Substring search across content + tags. */
-export function searchMemories(query, repo = null, limit = 50, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const like = `%${query}%`;
-    const rows = repo
-        ? db.prepare("SELECT * FROM memories WHERE repo = ? AND (content LIKE ? OR tags LIKE ?) ORDER BY created_at DESC LIMIT ?").all(repo, like, like, limit)
-        : db.prepare("SELECT * FROM memories WHERE content LIKE ? OR tags LIKE ? ORDER BY created_at DESC LIMIT ?").all(like, like, limit);
-    return rows.map(mapMemoryRow);
-}
-/** Mark a memory as recalled (updates last_recalled_at). Returns true if found. */
-export function recallMemory(id, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const now = Math.floor(Date.now() / 1000);
-    const res = db.prepare("UPDATE memories SET last_recalled_at = ? WHERE id = ?").run(now, id);
-    return res.changes > 0;
-}
-function mapMemoryRow(row) {
-    return {
-        id: row.id,
-        repo: row.repo ?? null,
-        kind: row.kind ?? "note",
-        content: row.content ?? "",
-        tags: row.tags ? JSON.parse(row.tags) : [],
-        createdAt: row.created_at ?? 0,
-        lastRecalledAt: row.last_recalled_at ?? null,
-    };
-}

package/dist/src/store/sqlite/minhash-lsh.js

@@ -1,47 +0,0 @@
-/**
- * Sprint 11: MinHash signatures + LSH bucket persistence and candidate lookup.
- *
- * All queries are parameterized (PREVENT-002) — never string-concatenated.
- */
-import { getStateDir, normalizeSessionId } from "../../store.js";
-import { openStore } from "./connection.js";
-import { withTx } from "./transaction.js";
-/** Persist a checkpoint's MinHash signature (idempotent by chunk_id + version). */
-export function upsertMinhashSignature(chunkId, sessionId, signatureVersion, signatures, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const sid = normalizeSessionId(sessionId);
-    db.prepare(`INSERT INTO minhash_signatures(chunk_id, session_id, signature_version, signatures)
-     VALUES(?, ?, ?, ?)
-     ON CONFLICT(chunk_id, signature_version) DO UPDATE SET
-       session_id=excluded.session_id, signatures=excluded.signatures`).run(chunkId, sid, signatureVersion, JSON.stringify(signatures));
-}
-/** Persist LSH bucket memberships for a chunk (one row per bucket key). */
-export function insertLshBuckets(chunkId, sessionId, signatureVersion, bucketKeys, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const sid = normalizeSessionId(sessionId);
-    const del = db.prepare("DELETE FROM dedup_lsh_buckets WHERE chunk_id = ?");
-    const ins = db.prepare("INSERT OR IGNORE INTO dedup_lsh_buckets(bucket_key, chunk_id, session_id, signature_version) VALUES(?, ?, ?, ?)");
-    withTx(db, () => {
-        del.run(chunkId);
-        for (const key of bucketKeys)
-            ins.run(key, chunkId, sid, signatureVersion);
-    });
-}
-/**
- * Candidate chunk_ids sharing any LSH bucket with `bucketKeys`, scoped to the
- * session, capped at `limit`. Single query (no N loops) — QA #15 amplification
- * guard. Returns DISTINCT chunk_ids excluding `excludeChunkId` (the new row).
- */
-export function lshCandidateChunks(bucketKeys, sessionId, excludeChunkId, stateDir = getStateDir(), limit = 100) {
-    if (bucketKeys.length === 0)
-        return [];
-    const db = openStore(stateDir);
-    const sid = normalizeSessionId(sessionId);
-    const placeholders = bucketKeys.map(() => "?").join(",");
-    const rows = db
-        .prepare(`SELECT DISTINCT chunk_id FROM dedup_lsh_buckets
-       WHERE bucket_key IN (${placeholders}) AND session_id = ? AND chunk_id != ?
-       LIMIT ?`)
-        .all(...bucketKeys, sid, excludeChunkId, limit);
-    return rows.map((r) => r.chunk_id);
-}

package/dist/src/store/sqlite/sessions.js

@@ -1,39 +0,0 @@
-/**
- * Future-feature foundation: resume sessions, daily activity log, lessons learned.
- *
- * Scaffolded tables + minimal helpers so all store data lives in SQLite from
- * day one. Full UI/recall for these lands in later sprints. All queries are
- * parameterized (PREVENT-002).
- */
-import { getStateDir, normalizeSessionId } from "../../store.js";
-import { openStore } from "./connection.js";
-/** Upsert a `sessions` row (resume + per-repo session history). */
-export function touchSession(sessionId, repo, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const sid = normalizeSessionId(sessionId);
-    const existing = db
-        .prepare("SELECT started_at FROM sessions WHERE session_id = ?")
-        .get(sid);
-    const now = Math.floor(Date.now() / 1000);
-    if (!existing) {
-        db.prepare(`INSERT INTO sessions(session_id, repo, started_at, last_compacted_at, status)
-       VALUES(?, ?, ?, ?, 'active')`).run(sid, repo ?? null, now, now);
-    }
-    else {
-        db.prepare("UPDATE sessions SET last_compacted_at = ?, repo = COALESCE(?, repo), status = 'active' WHERE session_id = ?").run(now, repo ?? null, sid);
-    }
-}
-/** Append a `daily_log` entry (day = YYYY-MM-DD, local-naive from Date). */
-export function logDaily(sessionId, event, detail, tokensSaved, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const day = new Date().toISOString().slice(0, 10);
-    const now = Math.floor(Date.now() / 1000);
-    db.prepare(`INSERT INTO daily_log(day, session_id, event, detail, tokens_saved, ts)
-     VALUES(?, ?, ?, ?, ?, ?)`).run(day, normalizeSessionId(sessionId), event, detail ?? null, tokensSaved, now);
-}
-/** Append a `lessons` entry (future lessons-learned browse/recall). */
-export function addLesson(sessionId, repo, lesson, stateDir = getStateDir()) {
-    const db = openStore(stateDir);
-    const now = Math.floor(Date.now() / 1000);
-    db.prepare(`INSERT INTO lessons(session_id, repo, lesson, ts) VALUES(?, ?, ?, ?)`).run(normalizeSessionId(sessionId), repo ?? null, lesson, now);
-}

package/dist/src/store/sqlite/transaction.js

@@ -1,19 +0,0 @@
-/**
- * Transaction wrapper using SAVEPOINT so it nests safely under an outer
- * transaction (unlike `BEGIN`, which SQLite rejects when one is already open).
- *
- * Mirrors better-sqlite3's `db.transaction(fn)` semantics — callers that wrap a
- * batch in withTx (e.g. backfill) can still call helpers that also use withTx.
- */
-export function withTx(db, fn) {
-    db.exec("SAVEPOINT mc_tx");
-    try {
-        fn();
-        db.exec("RELEASE mc_tx");
-    }
-    catch (e) {
-        db.exec("ROLLBACK TO mc_tx");
-        db.exec("RELEASE mc_tx");
-        throw e;
-    }
-}

package/dist/src/vectorStore/add.js

@@ -1,260 +0,0 @@
-/**
- * vectorStore/add.ts — Add/insert operations: the dedup cascade and L1 lookup.
- */
-import { createHash } from "node:crypto";
-import { cosineSimilarity } from "../embedder.js";
-import { normalizeSessionId, compressSmart } from "../store.js";
-import { computeContentDigest } from "../dedup/digest.js";
-import { minhashSignature, SIGNATURE_VERSION, NUM_HASHES } from "../dedup/l1-minhash.js";
-import { lshBands } from "../dedup/l1-lsh.js";
-import { isNearDuplicate } from "../dedup/l1-verify.js";
-import { openBloom, saveBloom } from "../store/bloom.js";
-import { listCheckpoints, nextCheckpointId, upsertCheckpoint, loadSessionState, saveSessionState, upsertMinhashSignature, insertLshBuckets, lshCandidateChunks, bumpDedupStats, addTokensSaved, } from "../store/sqlite.js";
-import { computeRegionHash, recordDecision } from "./utils.js";
-/**
- * L1 near-duplicate lookup: MinHash → LSH candidate retrieval → trigram verify.
- * Returns the matching checkpoint or undefined. Bounded by a 100-candidate cap
- * and a 20ms verify budget (QA #7/#15) so it never hangs a large session.
- */
-export function findL1Duplicate(ctx, sessionId, regionText, all) {
-    if (all.length === 0)
-        return undefined;
-    const sig = minhashSignature(regionText);
-    if (sig.length !== NUM_HASHES)
-        return undefined;
-    const bands = lshBands(sig, sessionId, SIGNATURE_VERSION);
-    // Cheap candidate retrieval (single query, capped). Exclude nothing yet —
-    // the new checkpoint has no id, so pass a sentinel that never matches.
-    const candidateIds = lshCandidateChunks(bands, sessionId, "__new__", ctx.stateDir, 100);
-    if (candidateIds.length === 0)
-        return undefined;
-    const byId = new Map(all.map((cp) => [cp.checkpointId, cp]));
-    const VERIFY_BUDGET_MS = 20;
-    const start = Date.now();
-    for (const id of candidateIds) {
-        if (Date.now() - start > VERIFY_BUDGET_MS)
-            break; // QA #15: abort → "not dup"
-        const cand = byId.get(id);
-        if (!cand)
-            continue;
-        const candText = cand.normalizedText ?? cand.summary ?? "";
-        if (isNearDuplicate(regionText, candText))
-            return cand;
-    }
-    return undefined;
-}
-/**
- * Add a checkpoint. Dedup cascade:
- *   1. regionHash exact match (legacy, backward-compat)
- *   2. summaryHash exact match (new: catches same-topic incremental compactions)
- *   3. content similarity ≥ dedupSim (catches near-identical summaries)
- *   4. If none match → create new checkpoint
- */
-export function addToStore(ctx, input) {
-    const t0 = Date.now();
-    const sessionId = normalizeSessionId(input.sessionId);
-    const regionHash = computeRegionHash(input.regionText);
-    const all = listCheckpoints(sessionId, ctx.stateDir);
-    // Honest "tokens saved" base for this region. For a deduped add the whole
-    // original region is discarded (nothing new stored); for a new checkpoint
-    // we persist (orig − stored). Falls back to stored when orig is unknown.
-    const origTokens = input.originalTokenEstimate ?? input.tokenEstimate ?? 0;
-    const cfg = ctx.cfg;
-    // Live per-tier progress hook (Phase 1). Sync + optional; fired at each tier
-    // so the UI can paint "L0 ✓ → L1 ✓ → L2 0.91 → stored" during a compaction.
-    const onTier = input.onTier;
-    // Tracks whether a tier matched while in MARK_ONLY (record-but-don't-collapse),
-    // and which tier.
-    let markOnly = null;
-    // 0. L0 content-hash dedup (Sprint 9) — catches identical content arriving
-    //    under different regionText. Normalization handles case/whitespace/ANSI so
-    //    variants collapse to one row. Dual-hash guards a single-hash collision.
-    //    Sprint 10: bloom is the accelerator — a miss means "definitely new" and
-    //    skips the scan; a hit is only a candidate, confirmed against `all` below.
-    //    Gated by L0_ENABLED (Sprint 14). MARK_ONLY_L0 records the decision but
-    //    does not collapse — the new region is still stored.
-    onTier?.({ tier: "L0", status: "scanning" });
-    const digest = computeContentDigest(input.regionText);
-    const bloom = openBloom(ctx.stateDir);
-    if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
-        const contentMatch = all.find((cp) => cp.contentHash === digest.contentHash &&
-            cp.contentHash2 === digest.contentHash2);
-        if (contentMatch) {
-            if (cfg.MARK_ONLY_L0) {
-                markOnly = "L0"; // Record-but-don't-collapse: fall through.
-            }
-            else {
-                contentMatch.timestamp = input.timestamp;
-                upsertCheckpoint(contentMatch, ctx.stateDir);
-                bumpDedupStats(true, ctx.stateDir);
-                // Deduped: whole original region discarded, nothing new stored.
-                addTokensSaved(origTokens, ctx.stateDir);
-                const r = { checkpoint: contentMatch, deduped: true, reason: "contentHash" };
-                recordDecision(ctx, "L0", "deduped", "contentHash", Date.now() - t0);
-                onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
-                return r;
-            }
-        }
-    }
-    // 1. Legacy regionHash dedup (backward-compat) — part of L0 tier gating.
-    if (cfg.L0_ENABLED) {
-        const regionMatch = all.find((cp) => cp.regionHash === regionHash);
-        if (regionMatch) {
-            if (cfg.MARK_ONLY_L0) {
-                markOnly = "L0"; // fall through
-            }
-            else {
-                bumpDedupStats(true, ctx.stateDir);
-                // Deduped: whole original region discarded, nothing new stored.
-                addTokensSaved(origTokens, ctx.stateDir);
-                const r = { checkpoint: regionMatch, deduped: true, reason: "regionHash" };
-                recordDecision(ctx, "L0", "deduped", "regionHash", Date.now() - t0);
-                onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
-                return r;
-            }
-        }
-    }
-    // 2. SummaryHash dedup — catches same-topic incremental compactions.
-    //    Full 64-hex SHA-256 (was 16-hex in Sprint 8 — collision-prone).
-    const summaryHash = input.topicSummary
-        ? createHash("sha256").update(input.topicSummary).digest("hex")
-        : undefined;
-    if (summaryHash && cfg.L0_ENABLED) {
-        const summaryMatch = all.find((cp) => cp.summaryHash === summaryHash);
-        if (summaryMatch) {
-            if (cfg.MARK_ONLY_L0) {
-                markOnly = "L0"; // fall through
-            }
-            else {
-                summaryMatch.timestamp = input.timestamp;
-                upsertCheckpoint(summaryMatch, ctx.stateDir);
-                bumpDedupStats(true, ctx.stateDir);
-                // Deduped: whole original region discarded, nothing new stored.
-                addTokensSaved(origTokens, ctx.stateDir);
-                const r = { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" };
-                recordDecision(ctx, "L0", "deduped", "summaryHash", Date.now() - t0);
-                onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
-                return r;
-            }
-        }
-    }
-    // L0 did not collapse this region.
-    onTier?.({ tier: "L0", status: "passed" });
-    // 2b. L1 MinHash/LSH near-duplicate dedup (Sprint 11) — catches one-word
-    //     edits / rewordings that L0's exact hash misses. Cheap LSH bucket
-    //     retrieval → trigram verification (pg_trgm-equivalent) as the final gate.
-    //     Gated by L1_ENABLED (Sprint 14); MARK_ONLY_L1 records but doesn't collapse.
-    onTier?.({ tier: "L1", status: "scanning" });
-    if (cfg.L1_ENABLED) {
-        const l1 = findL1Duplicate(ctx, sessionId, input.regionText, all);
-        if (l1 && !cfg.MARK_ONLY_L1) {
-            l1.timestamp = input.timestamp;
-            upsertCheckpoint(l1, ctx.stateDir);
-            bumpDedupStats(true, ctx.stateDir);
-            const r = { checkpoint: l1, deduped: true, reason: "l1MinHash" };
-            recordDecision(ctx, "L1", "deduped", "l1MinHash", Date.now() - t0);
-            onTier?.({ tier: "L1", status: "deduped", detail: "l1MinHash" });
-            return r;
-        }
-        if (l1 && cfg.MARK_ONLY_L1)
-            markOnly = "L1";
-    }
-    onTier?.({ tier: "L1", status: "passed" });
-    // 3. L2 semantic dedup — catches near-identical / semantically-similar regions
-    //    via cosine over the embedding. topicSummary is used for summaryHash dedup
-    //    (tier 2); the vector index is keyed on the original region for backward-
-    //    compat search semantics. Threshold from cfg (L2_COSINE trigram honest
-    //    firing point). QA #13 timeout guard: if the O(n) scan exceeds the budget,
-    //    degrade to "store without dedup this pass" so we never lose a checkpoint.
-    //    Gated by L2_ENABLED (Sprint 14); MARK_ONLY_L2 records but doesn't collapse.
-    const SIMILARITY_BUDGET_MS = cfg.SIMILARITY_BUDGET_MS;
-    const simThreshold = ctx.l2Threshold; // from cfg.L2_COSINE (default 0.85 trigram)
-    const embedding = ctx.embedder.embed(input.regionText);
-    onTier?.({ tier: "L2", status: "scanning" });
-    if (cfg.L2_ENABLED && all.length > 0) {
-        const start = Date.now();
-        let timedOut = false;
-        const nearest = all.reduce((best, cp) => {
-            if (!timedOut && Date.now() - start > SIMILARITY_BUDGET_MS)
-                timedOut = true;
-            if (timedOut)
-                return best;
-            const sim = cosineSimilarity(embedding, cp.embedding);
-            return sim > best.sim ? { checkpoint: cp, sim } : best;
-        }, { checkpoint: all[0], sim: -1 });
-        if (!timedOut && nearest.sim >= simThreshold) {
-            if (!cfg.MARK_ONLY_L2) {
-                // Near-identical — update timestamp on existing checkpoint
-                nearest.checkpoint.timestamp = input.timestamp;
-                upsertCheckpoint(nearest.checkpoint, ctx.stateDir);
-                bumpDedupStats(true, ctx.stateDir);
-                // Deduped: whole original region discarded, nothing new stored.
-                addTokensSaved(origTokens, ctx.stateDir);
-                const r = { checkpoint: nearest.checkpoint, deduped: true, reason: "contentSimilarity" };
-                recordDecision(ctx, "L2", "deduped", "contentSimilarity", Date.now() - t0);
-                onTier?.({ tier: "L2", status: "deduped", detail: nearest.sim.toFixed(2) });
-                return r;
-            }
-            markOnly = "L2";
-        }
-        onTier?.({ tier: "L2", status: "passed", detail: `best ${nearest.sim.toFixed(2)}` });
-    }
-    // 4. Genuinely new — create checkpoint
-    const checkpointId = nextCheckpointId(sessionId, ctx.stateDir);
-    const checkpoint = {
-        checkpointId,
-        sessionId,
-        summary: input.summary,
-        topicSummary: input.topicSummary,
-        summaryHash,
-        keyDecisions: input.keyDecisions ?? [],
-        nextSteps: input.nextSteps ?? [],
-        filesModified: input.filesModified ?? [],
-        tokenEstimate: input.tokenEstimate ?? 0,
-        originalTokenEstimate: input.originalTokenEstimate,
-        regionHash,
-        contentHash: digest.contentHash,
-        contentHash2: digest.contentHash2,
-        contentHashVersion: digest.contentHashVersion,
-        normalizedText: digest.normalizedText,
-        compressedOriginal: compressSmart(Buffer.from(input.regionText, "utf-8"), input.compressionPressure),
-        embedding,
-        timestamp: input.timestamp,
-    };
-    // Persistence is SQLite (store/sqlite.ts). upsertCheckpoint keeps the
-    // idempotent-by-id semantics the old JSON append implied.
-    upsertCheckpoint(checkpoint, ctx.stateDir);
-    // Cumulative "tokens saved" counter (per-repo SQLite meta). For a NEW
-    // checkpoint the saved amount is (original − stored); for a deduped add the
-    // whole original region is discarded (handled in the deduped return paths
-    // below). Survives sessions and travels with the repo.
-    const stored = input.tokenEstimate ?? 0;
-    addTokensSaved(Math.max(0, origTokens - stored), ctx.stateDir);
-    // L1: persist this checkpoint's MinHash signature + LSH buckets so future
-    // near-duplicate inserts can find it. Deterministic given the seed.
-    const sig = minhashSignature(input.regionText);
-    upsertMinhashSignature(checkpointId, sessionId, SIGNATURE_VERSION, sig, ctx.stateDir);
-    insertLshBuckets(checkpointId, sessionId, SIGNATURE_VERSION, lshBands(sig, sessionId, SIGNATURE_VERSION), ctx.stateDir);
-    // Bloom accelerator: record the new content_hash so a future add() can short-
-    // circuit the scan on a hit (still confirmed by the SELECT-based `all` above).
-    bloom.add(digest.contentHash);
-    saveBloom(ctx.stateDir);
-    // Track the region hash in session state for fast sentinel checks.
-    const state = loadSessionState(sessionId, ctx.stateDir);
-    if (!state.storedRegionHashes.includes(regionHash)) {
-        state.storedRegionHashes.push(regionHash);
-        saveSessionState(sessionId, state, ctx.stateDir);
-    }
-    // A new checkpoint. If a tier matched while MARK_ONLY, record that (the
-    // decision fired but we intentionally did not collapse).
-    if (markOnly) {
-        recordDecision(ctx, markOnly, "mark_only", "mark_only", Date.now() - t0);
-    }
-    else {
-        recordDecision(ctx, "L0", "new", undefined, Date.now() - t0);
-    }
-    // Cumulative store-wide dedup accounting (attempt, not collapsed).
-    bumpDedupStats(false, ctx.stateDir);
-    onTier?.({ tier: "new", status: "stored" });
-    return { checkpoint, deduped: false };
-}

package/dist/src/vectorStore/dedup.js

@@ -1,52 +0,0 @@
-/**
- * vectorStore/dedup.ts — SemDeDup cleanup and dedup sentinel check.
- */
-import { cosineSimilarity } from "../embedder.js";
-import { normalizeSessionId } from "../store.js";
-import { listCheckpoints, setDedupStatus, loadSessionState } from "../store/sqlite.js";
-import { computeRegionHash } from "./utils.js";
-/**
- * SemDeDup offline cleanup (Sprint 12, QA #17): within a session, mark the
- * lower-quality row of any pair scoring cosine > `threshold` as
- * `dedup_status='removed'` (kept, not deleted — retrieval excludes it). Keeps
- * the row with the higher `tokenEstimate` (more context preserved). Runs as a
- * single scan; idempotent (re-running skips already-removed rows).
- *
- * Returns the number of rows marked removed.
- */
-export function semDedupStore(ctx, sessionId, threshold) {
-    const sid = normalizeSessionId(sessionId);
-    const cps = listCheckpoints(sid, ctx.stateDir).filter((c) => c.dedupStatus !== "removed");
-    let removed = 0;
-    for (let i = 0; i < cps.length; i++) {
-        for (let j = i + 1; j < cps.length; j++) {
-            const a = cps[i];
-            const b = cps[j];
-            if (a.dedupStatus === "removed" || b.dedupStatus === "removed")
-                continue;
-            if (cosineSimilarity(a.embedding, b.embedding) > threshold) {
-                // Keep the higher-tokenEstimate row; remove the other.
-                const keep = a.tokenEstimate >= b.tokenEstimate ? a : b;
-                const drop = keep === a ? b : a;
-                setDedupStatus(drop.checkpointId, sid, "removed", ctx.stateDir);
-                drop.dedupStatus = "removed";
-                removed++;
-            }
-        }
-    }
-    return removed;
-}
-/**
- * Dedup sentinel check: has this region already been stored/represented?
- * Consulted by both the persist path and the recall/inline path.
- */
-export function dedupeCheck(ctx, sessionId, regionHashOrText, isText = false) {
-    const sid = normalizeSessionId(sessionId);
-    const hash = isText
-        ? computeRegionHash(regionHashOrText)
-        : regionHashOrText;
-    const state = loadSessionState(sid, ctx.stateDir);
-    if (state.storedRegionHashes.includes(hash))
-        return true;
-    return listCheckpoints(sid, ctx.stateDir).some((c) => c.regionHash === hash);
-}

package/dist/src/vectorStore/index.js

@@ -1,10 +0,0 @@
-/**
- * vectorStore/index.ts — Barrel re-export for the vector store sub-modules.
- *
- * All public symbols previously exported from the monolithic `vectorStore.ts`
- * are re-exported here so existing imports (`from "./vectorStore.js"`) continue
- * to work unchanged.
- */
-export { L2_ENABLED } from "./types.js";
-export { computeRegionHash } from "./utils.js";
-export { VectorStore } from "./store.js";