pi-mega-compact 0.20.86 → 0.20.87

package/dist/src/recall/recall3wf.fixture.js

@@ -0,0 +1,67 @@
+/**
+ * src/recall/recall3wf.fixture.ts — shared fixtures for the 3WF-3 recall tests.
+ *
+ * Split out of recall3wf.test.ts (which crossed the src 300 soft cap) so each
+ * test file stays under the limit. These are REAL fixtures, not mocks/stubs:
+ * a REAL VectorStore over a temp stateDir, REAL checkpoints persisted via
+ * compactSession, and readers that go through the SAME working path the
+ * extension uses (recallRawHits -> vectorSearch -> listCheckpoints, and
+ * vectorWasInjected), mirroring the proven triggerGuard test pattern.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../vectorStore.js";
+import { compactSession } from "../engine.js";
+import { recallAndInline } from "../recall.js";
+import { recallRawHits } from "./readonly.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { initSchema } from "../store/sqlite/schema.js";
+/** Real EngineMessage fixture. */
+export function msg(role, text) {
+    return { role, text };
+}
+/** Fresh isolated state dir per VectorStore. */
+export function freshStore() {
+    const dir = mkdtempSync(join(tmpdir(), "mc-3wf-"));
+    return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
+}
+/** Persist N distinct checkpoints with distinct content + ascending timestamps. */
+export function seed(store, topics, sid = "sess_3wf") {
+    topics.forEach((t, i) => {
+        compactSession({
+            sessionId: sid,
+            messages: [msg("user", t), msg("assistant", "ok")],
+            keepFrom: 2,
+            timestamp: i + 1,
+        }, store);
+    });
+}
+/** Checkpoint ids via the real search path (vectorSearch -> listCheckpoints). */
+export function checkpointIds(store, sid, query) {
+    return recallRawHits({ sessionId: sid, query, limit: 10 }, store).map((h) => h.checkpoint.checkpointId);
+}
+/** Run the real recallAndInline path with skipInjected:false so nothing is
+ *  marked and the block reflects the search result exactly (deterministic). */
+export function recallAndInlineCapture(sid, query, store) {
+    const r = recallAndInline({ sessionId: sid, query, limit: 3, source: "command", skipInjected: false, windowDedupe: false }, store);
+    return { block: r.block, empty: r.empty, toInject: r.toInject };
+}
+/** Count recall-provenance rows (turn_recall) for a session via raw SQL reader. */
+export function countTurnRecallRows(store, sid) {
+    try {
+        const reader = openStore(store.stateDir);
+        // Ensure the turns/turn_recall tables exist so a 0-count is meaningful
+        // (a write on the new path would be visible, not masked by a missing table).
+        initSchema(reader);
+        const row = reader
+            .prepare(`SELECT COUNT(*) AS n FROM turn_recall tr
+				 JOIN turns t ON t.id = tr.turn_id
+				 WHERE t.session_id = ?`)
+            .get(sid);
+        return row?.n ?? 0;
+    }
+    catch {
+        return 0;
+    }
+}

package/dist/src/recall/validator.js

@@ -0,0 +1,113 @@
+/**
+ * recall/validator.ts — independent candidate validator (3WF-3).
+ *
+ * Judges candidates handed to it; it MUST NOT call any search itself. Given the
+ * ranked vote winners + the live-window text (already extracted by the caller,
+ * since src/ cannot import pi types), it walks the winners in order and returns
+ * the first that passes BOTH gates:
+ *
+ *   1. Cosine floor: the winner's score >= the same-repo floor (default 0.12,
+ *      env MEGACOMPACT_RECALL_MIN_COSINE). The cross-repo 0.90 floor
+ *      (config.crossRepoCosine) is SEPARATE and intentionally untouched.
+ *   2. Not already resident in the live window: reuse recall/sync.ts's exact
+ *      comparison — embed each live message, embed the checkpoint summary, and
+ *      treat the checkpoint as resident when cosineSimilarity >= dedupSim. We
+ *      reuse that metric rather than inventing a new one.
+ *
+ * On a failing candidate it advances to the next-ranked winner. If ALL fail it
+ * returns the provenance floor (FloorBlock built from the newest checkpoint —
+ * pure over checkpoints, same semantics as triggerGuard's buildFloorBlock).
+ *
+ * Non-fatal throughout: any error degrades to the next candidate / the floor.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
+// SQLite store, NOT src/store.ts's legacy gzipped-JSON DR reader (that returns
+// [] for live sessions). Mirrors vector-search.ts / tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { RECALL_MIN_COSINE } from "../config.js";
+/** Build the provenance floor block from the session's newest checkpoint. */
+function buildFloorBlock(sessionId, store) {
+    try {
+        const cps = listCheckpoints(sessionId, store.stateDir).filter((c) => c.dedupStatus !== "removed");
+        let newest = cps[0];
+        for (const cp of cps) {
+            if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0))
+                newest = cp;
+        }
+        const summary = newest?.summary?.trim();
+        if (summary) {
+            return {
+                text: "The following compacted context is the most recent checkpoint from " +
+                    "this session (recall found no query-relevant match):\n\n" + summary,
+                basis: "lastCheckpoint",
+            };
+        }
+        return {
+            text: "This session has compacted context but recall could not surface a " +
+                "checkpoint relevant to the current request; the most recent checkpoint " +
+                "summary is unavailable.",
+            basis: "lastCheckpoint",
+        };
+    }
+    catch {
+        return {
+            text: "This session has compacted context but recall could not surface a " +
+                "checkpoint relevant to the current request.",
+            basis: "none",
+        };
+    }
+}
+/**
+ * Validate the ranked vote winners, returning the first that passes both gates,
+ * or the provenance floor if none do. Does NOT mutate the injected set, does NOT
+ * write turns, does NOT emit telemetry. Non-fatal.
+ */
+export function validateRecall(winners, opts, store) {
+    const floor = RECALL_MIN_COSINE();
+    const dedupSim = opts.dedupSim ?? 0.9;
+    const embedder = defaultEmbedder();
+    const liveVecs = (opts.liveWindow ?? []).map((m) => embedder.embed(m));
+    // One checkpoint read for the whole pass (both gates share it).
+    const cps = listCheckpoints(opts.sessionId, store.stateDir);
+    const cpById = new Map(cps.map((c) => [c.checkpointId, c]));
+    const queryVec = opts.query ? embedder.embed(opts.query) : null;
+    for (const cand of winners) {
+        try {
+            const cp = cpById.get(cand.checkpointId);
+            // Gate 1: same-repo COSINE floor. `cand.score` is only a cosine for
+            // source "vector"; fts5 (BM25) and recency (freshness rank) live on
+            // other scales, so for those we re-derive the true cosine locally from
+            // the query + checkpoint embedding. No search call is made.
+            let cosine;
+            if (cand.source === "vector") {
+                cosine = cand.score;
+            }
+            else if (queryVec && cp) {
+                cosine = cosineSimilarity(queryVec, embedder.embed(cp.summary));
+            }
+            else {
+                // No comparable cosine available => cannot clear a cosine gate.
+                continue;
+            }
+            if (cosine < floor)
+                continue;
+            // Gate 2: not already resident in the live window.
+            if (liveVecs.length > 0) {
+                if (!cp)
+                    continue; // cannot verify => skip rather than risk re-inject
+                const hitVec = embedder.embed(cp.summary);
+                const resident = liveVecs.some((v) => cosineSimilarity(v, hitVec) >= dedupSim);
+                if (resident)
+                    continue;
+            }
+            return { kind: "candidate", candidate: cand };
+        }
+        catch {
+            // Non-fatal: skip this candidate, try the next.
+            continue;
+        }
+    }
+    // All candidates rejected -> provenance floor.
+    return { kind: "floor", floor: buildFloorBlock(opts.sessionId, store) };
+}

package/dist/src/recall/vote.js

@@ -0,0 +1,217 @@
+/**
+ * recall/vote.ts — the 3-independent-source recall vote (3WF-3).
+ *
+ * Three INDEPENDENT, read-only sources name candidate checkpoints:
+ *   A vector  — raw semantic hits (recall/readonly.ts), cosine 0..1 scale.
+ *   B fts5    — BM25 trigram hits (hydrated to checkpointIds), FTS5 BM25 scale
+ *               (negative score = better match; ranking order is what matters).
+ *   C recency — the N freshest checkpoints by timestamp, query-INDEPENDENT.
+ *               (NOT turn_recall / TurnReader — that would just echo already-
+ *               injected content, not an independent signal.)
+ *
+ * Each source is a different score scale, so they are NOT directly comparable.
+ * We normalize each source's scores to a 0..1 scale (per-source min-max) BEFORE
+ * combining. Averaging raw cosine (0..1) with raw BM25 (arbitrary negative
+ * magnitude) or a recency rank would be meaningless — the largest-magnitude
+ * scale would always dominate. Normalization makes each source a peer voter.
+ *
+ * Overlap rule: a checkpoint named by >=2 of 3 distinct sources short-circuits
+ * as a winner. Fallback (no 2/3 majority): rank all candidates by the
+ * cross-source MEAN of their normalized scores.
+ *
+ * Non-fatal throughout. Pi-agnostic: no pi runtime imports.
+ */
+import { openStore } from "../store/sqlite/utils.js";
+import { fts5SearchScoped, hydrateFts5Hits } from "../store/sqlite/fts5-search.js";
+// The SQLite store is the source of truth (src/store.ts's same-named helper
+// reads the LEGACY gzipped-JSON DR snapshot, which is empty for live sessions —
+// importing it here would silently starve sources B and C). Mirrors the import
+// in vector-search.ts + tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { computeContentDigest } from "../dedup/digest.js";
+import { recallRawHits } from "./readonly.js";
+import { Logger } from "../log.js";
+/** Per-source normalization: map raw scores to 0..1 via min-max within source. */
+function normalizeScores(scores) {
+    const map = new Map();
+    if (scores.length === 0)
+        return map;
+    const min = Math.min(...scores);
+    const max = Math.max(...scores);
+    const span = max - min;
+    scores.forEach((s, i) => {
+        map.set(i, span === 0 ? 1 : (s - min) / span);
+    });
+    return map;
+}
+/**
+ * Run the three-source recall vote. Returns agreement winners + a per-id vote
+ * count + the names of sources that produced no winning candidate. Non-fatal:
+ * any search failure degrades to the remaining sources (empty winners allowed).
+ */
+export function voteRecall(opts, store) {
+    const logger = new Logger();
+    const limit = opts.limit ?? 3;
+    const recencyCount = opts.recencyCount ?? limit;
+    // ── Source A: vector (raw hits, cosine 0..1). ────────────────────────────
+    const vectorCands = recallRawHits({ sessionId: opts.sessionId, query: opts.query, limit }, store).map((h) => ({
+        checkpointId: h.checkpoint.checkpointId,
+        score: h.score,
+        source: "vector",
+    }));
+    // ── Source B: fts5 (BM25, hydrated to checkpointIds). ─────────────────────
+    // Dedup on L0 content digest so the SAME normalized text under two ids does
+    // not double-vote — collapse to one candidate per digest.
+    const fts5Cands = (() => {
+        try {
+            const reader = openStore(store.stateDir);
+            const hits = fts5SearchScoped(opts.query, reader, opts.sessionId, limit);
+            // FTS5 returns scores ordered best-first (bm25 asc); lower = better.
+            // We keep the raw score (negative-is-better) and flip in normalization.
+            const hydrated = hydrateFts5Hits(hits, opts.sessionId, store.stateDir);
+            // Dedup on checkpointId so the same checkpoint cannot double-count, AND
+            // on the L0 CONTENT digest so identical normalized text stored under two
+            // different ids collapses to one vote. The digest is taken over the
+            // joined `summary` (real content): hashing the id string would be a
+            // no-op tier, since ids are unique by definition.
+            const seenDigest = new Set();
+            const seenId = new Set();
+            const out = [];
+            for (const h of hydrated) {
+                if (seenId.has(h.checkpointId))
+                    continue;
+                seenId.add(h.checkpointId);
+                const digest = computeContentDigest(h.summary).contentHash;
+                if (seenDigest.has(digest))
+                    continue;
+                seenDigest.add(digest);
+                out.push({ checkpointId: h.checkpointId, score: h.score, source: "fts5" });
+            }
+            return out;
+        }
+        catch {
+            return [];
+        }
+    })();
+    // ── Source C: recency (N freshest checkpoints, query-independent). ────────
+    // Timestamp-ordered; the lower the index the fresher. Score = recency rank
+    // (fresh = high) so normalization treats newest as best.
+    const recencyCands = (() => {
+        try {
+            const cps = listCheckpoints(opts.sessionId, store.stateDir)
+                .filter((c) => c.dedupStatus !== "removed")
+                .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0))
+                .slice(0, recencyCount);
+            return cps.map((cp, i) => ({
+                checkpointId: cp.checkpointId,
+                // Fresher => higher raw score (recency rank). Normalized below.
+                score: cps.length - i,
+                source: "recency",
+            }));
+        }
+        catch {
+            return [];
+        }
+    })();
+    const sources = [
+        { name: "vector", cands: vectorCands },
+        { name: "fts5", cands: fts5Cands },
+        { name: "recency", cands: recencyCands },
+    ];
+    // Per-source normalization to 0..1 so the three scales are comparable.
+    const perSource = sources.map((s) => ({
+        name: s.name,
+        norm: normalizeScores(s.cands.map((c) => c.score)),
+    }));
+    // Aggregate: best normalized score per source per checkpointId + vote count.
+    const bestScoreBySource = new Map();
+    const seenIds = new Set();
+    for (const src of sources) {
+        const norm = perSource.find((p) => p.name === src.name).norm;
+        const bestByCp = new Map();
+        src.cands.forEach((c, i) => {
+            const n = norm.get(i) ?? 0;
+            const prev = bestByCp.get(c.checkpointId);
+            if (prev === undefined || n > prev)
+                bestByCp.set(c.checkpointId, n);
+            seenIds.add(c.checkpointId);
+        });
+        bestScoreBySource.set(src.name, bestByCp);
+    }
+    // Vote count = number of DISTINCT sources naming each checkpointId.
+    const votes = {};
+    const sumByCp = new Map();
+    for (const id of seenIds) {
+        let count = 0;
+        let sum = 0;
+        for (const src of sources) {
+            const m = bestScoreBySource.get(src.name);
+            if (m.has(id)) {
+                count++;
+                sum += m.get(id);
+            }
+        }
+        votes[id] = count;
+        sumByCp.set(id, sum);
+    }
+    /** Mean normalized score across the sources that named `id`. */
+    const meanScore = (id) => (sumByCp.get(id) ?? 0) / (votes[id] ?? 1);
+    // Short-circuit: >=2 of 3 distinct sources => winner (agreement). Ranked by
+    // vote count first (stronger agreement wins), then by mean normalized score —
+    // the validator consumes this list in order and takes the first that passes,
+    // so the ordering IS the ranking and must not be Set-insertion order.
+    const winners = [];
+    const divergent = new Set(sources.map((s) => s.name));
+    const agreed = [...seenIds]
+        .filter((id) => (votes[id] ?? 0) >= 2)
+        .sort((a, b) => (votes[b] ?? 0) - (votes[a] ?? 0) || meanScore(b) - meanScore(a));
+    for (const id of agreed) {
+        const cand = (() => {
+            for (const src of sources) {
+                const c = src.cands.find((x) => x.checkpointId === id);
+                if (c)
+                    return c;
+            }
+            return null;
+        })();
+        if (cand)
+            winners.push(cand);
+    }
+    if (winners.length === 0) {
+        // Fallback (no 2-of-3 agreement): rank by cross-source MEAN normalized score.
+        const ranked = [...seenIds].sort((a, b) => meanScore(b) - meanScore(a));
+        for (const id of ranked) {
+            const cand = (() => {
+                for (const src of sources) {
+                    const c = src.cands.find((x) => x.checkpointId === id);
+                    if (c)
+                        return c;
+                }
+                return null;
+            })();
+            if (cand)
+                winners.push(cand);
+        }
+    }
+    // Divergence = a source that named NONE of the winning checkpoints. This must
+    // be computed per SOURCE against the winning ID SET, not from `winner.source`
+    // (a winner is one candidate object carrying a single source label, so an id
+    // agreed on by all three sources would still credit only one of them).
+    const winningIds = new Set(winners.map((w) => w.checkpointId));
+    for (const src of sources) {
+        if (src.cands.some((c) => winningIds.has(c.checkpointId))) {
+            divergent.delete(src.name);
+        }
+    }
+    if (divergent.size > 0) {
+        logger.info("recall_vote_divergence", {
+            divergentSources: [...divergent],
+            winnerCount: winners.length,
+        });
+    }
+    return {
+        winners,
+        votes,
+        divergentSources: [...divergent],
+    };
+}

package/dist/src/store/sqlite/fts5-search.js

@@ -11,6 +11,7 @@
  * time automatically — no manual n-gram splitting needed. The BM25 rank from
  * fts5 ranks results by trigram-overlap density.
  */
+import { listCheckpoints } from "./checkpoints.js";
 /**
  * Search the context_chunks_trgm FTS5 table for checkpoints whose
  * `normalized_text` matches `query` via trigram similarity.
@@ -66,3 +67,28 @@ export function fts5SearchScoped(query, reader, sessionId, limit = 10) {
     // No session filter — plain FTS5 search across all sessions.
     return fts5Search(query, reader, limit);
 }
+/**
+ * Hydrate FTS5 hits (which only carry checkpoint `id` + BM25 `score`, with NO
+ * checkpoint object) by joining against the real stored checkpoints. Mirrors the
+ * private `hydrateHits` in tieredRouter.ts (listCheckpoints → Map on
+ * checkpointId → join, dropping ids with no checkpoint).
+ *
+ * The joined `summary` is returned alongside the id because the 3WF-3 voter
+ * dedups candidates on the L0 CONTENT digest — hashing the id string instead
+ * would be a no-op tier (ids are already unique), silently disabling the check.
+ *
+ * @param hits      Raw Fts5Hit rows (id + BM25 score).
+ * @param sessionId Session scope used to fetch the checkpoint set.
+ * @param stateDir  On-disk state dir for the sync store.
+ */
+export function hydrateFts5Hits(hits, sessionId, stateDir) {
+    const cps = listCheckpoints(sessionId, stateDir);
+    const cpMap = new Map(cps.map((cp) => [cp.checkpointId, cp]));
+    const out = [];
+    for (const h of hits) {
+        const cp = cpMap.get(h.id);
+        if (cp)
+            out.push({ checkpointId: h.id, score: h.score, summary: cp.summary });
+    }
+    return out;
+}

package/extensions/dashboard-server/routes-rag-settings-helpers.ts

@@ -241,6 +241,7 @@ export const SETTINGS: ReadonlyArray<SettingGroup> = [
 			num("MEGACOMPACT_L2_THRESHOLD", "L2 Cosine Threshold", "L2 semantic dedup firing point", 0.85, 0, 1),
 			num("MEGACOMPACT_L1_JACCARD", "L1 Jaccard Threshold", "L1 MinHash near-dup threshold", 0.8, 0, 1),
 			num("MEGACOMPACT_DEDUP_SIM", "Dedup Similarity", "Legacy content-similarity fallback", 0.9, 0, 1),
+			num("MEGACOMPACT_RECALL_MIN_COSINE", "Recall Min Cosine (same-repo)", "3WF-3 same-repo floor the 3-source validator applies to the top winner (cross-repo 0.90 stays separate)", 0.12, 0, 1),
 			num("MEGACOMPACT_MMR_LAMBDA", "MMR Lambda", "Maximal Marginal Relevance diversity", 0.5, 0, 1),
 			num("MEGACOMPACT_SEMDEDUP_COSINE", "SemDeDup Cosine", "Offline SemDeDup pair threshold", 0.95, 0, 1),
 			num("MEGACOMPACT_CONSOLIDATE_COSINE", "Consolidate Cosine", "Memory consolidation merge threshold", 0.7, 0, 1),

package/extensions/mega-config-types.ts

@@ -119,6 +119,11 @@ export interface MegaConfig {
 	 *  tighter than same-repo so only genuinely-relevant cross-repo context is
 	 *  injected. */
 	crossRepoCosine: number;
+	/** Same-repo recall cosine floor (3WF-3). Default 0.12. SEPARATE from
+	 *  `crossRepoCosine` (S17, default 0.90 — stricter, cross-repo only). The
+	 *  3-source validator applies this to the top winner; hits below it are
+	 *  rejected in favor of the next-ranked candidate or the provenance floor. */
+	recallMinCosine: number;
 	/** Memory-RAG auto-review enabled (S20). Every memoryReviewInterval turns the
 	 *  conversation is auto-reviewed into durable add/replace/remove memories. */
 	memoryAutoReview: boolean;

package/extensions/mega-config.ts

@@ -217,6 +217,12 @@ export function loadConfig(): MegaConfig {
 		autoWikiEnabled: envBool("MEGACOMPACT_AUTO_WIKI", true),
 		crossRepoEnabled: envBool("MEGACOMPACT_CROSSREPO_ENABLED", true),
 		crossRepoCosine: Number(process.env.MEGACOMPACT_CROSSREPO_COSINE ?? "0.90"),
+		// 3WF-3: SAME-repo recall cosine floor applied by the 3-source validator to
+		// the top winner. SEPARATE from crossRepoCosine (S17, default 0.90, stricter
+		// and cross-repo only). This same-repo floor is permissive by default (0.12)
+		// so recall still surfaces loosely-relevant within-repo context while
+		// rejecting effectively-unrelated hits. Mirrors src/config.ts RECALL_MIN_COSINE.
+		recallMinCosine: Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12"),
 		memoryAutoReview: envBool("MEGACOMPACT_MEMORY_AUTO_REVIEW", true),
 		memoryReviewInterval: envFlag("MEGACOMPACT_MEMORY_REVIEW_INTERVAL", 10),
 		recallMaxTokens: envFlag("MEGACOMPACT_RECALL_MAX_TOKENS", 1500),