pi-mega-compact 0.20.86 → 0.20.88

package/dist/extensions/mega-pipeline/recall.js

@@ -1,256 +1,9 @@
 /**
- * recall.ts — unified Layer-5 recall pipeline.
+ * recall.ts — shell re-export for the unified Layer-5 recall pipeline (3WF-3 split).
  *
- * `doRecall` is the ONE path that injects (sync). `doRecallAsync` augments with
- * optional cross-repo HNSW on resume / /mega-recall --cross-repo. Both mutate
- * the shared MegaRuntime (token accounting, ticker, dashboard events).
+ * Delegate-shell pattern: the implementation lives in ./recall/impl.ts (kept
+ * under the 300-line soft cap). All public symbols are re-exported here so
+ * `export * from "./mega-pipeline/recall.js"` (mega-pipeline.ts) and any direct
+ * importers keep resolving with byte-identical names.
  */
-import { sessionEntryToContextMessages } from "@earendil-works/pi-coding-agent";
-import { recallAndInline, recallAndInlineAsync, formatRecallBlock, } from "../../src/recall.js";
-import { normalizeSessionId } from "../../src/store.js";
-import { incRecallInjected, incCacheHitTokens, getIndexDir, } from "../../src/store/sqlite.js";
-import { ensureConversationIdFor, recordTurnWrite, recordRecallWrite, } from "../mega-turn-store.js";
-import { C } from "../mega-runtime.js";
-import { recordRecallLatency } from "../mega-runtime/vc-observer.js";
-/**
- * Unified recall (Layer 5). The ONE path that injects. Returns the recall
- * result; callers decide whether to stage it for before_agent_start (resume)
- * or report it (command).
- */
-export function doRecall(runtime, config, ctx, query, source) {
-    runtime.bindRepo(ctx.cwd);
-    const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-    // Live window text for inline dedupe (Fix C): drop recalled checkpoints that
-    // are already resident in the session, so recall never re-injects context the
-    // model can already see. Best-effort — an empty window just skips dedupe.
-    const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
-    const recallStartMs = Date.now();
-    const result = recallAndInline({
-        sessionId: sid,
-        query,
-        limit: config.autoInlineK,
-        source,
-        skipInjected: true,
-        recallMaxTokens: config.recallMaxTokens,
-        windowDedupe: config.windowDedupe,
-        liveWindow,
-        dedupSim: config.dedupSim,
-    }, runtime.store);
-    runtime.dashboard.event("recall", {
-        source,
-        query: query.slice(0, 120),
-        injected: result.toInject.length,
-        empty: result.empty,
-    });
-    if (config.ragRecallMetrics && result.hydeInfo) {
-        runtime.dashboard.event("hyde_executed", {
-            sessionId: sid,
-            ran: result.hydeInfo.ran,
-            skipped: result.hydeInfo.skipped,
-            reason: result.hydeInfo.reason,
-            hypotheticalDoc: result.hydeInfo.hypotheticalDoc.slice(0, 400),
-            generationMs: result.hydeInfo.generationMs,
-            rawHitCount: result.hydeInfo.rawHitCount,
-            hydeHitCount: result.hydeInfo.hydeHitCount,
-            fusedHitCount: result.hydeInfo.fusedHitCount,
-            lift: result.hydeInfo.lift,
-        });
-    }
-    if (config.ragRecallMetrics && result.recallMetrics) {
-        runtime.dashboard.event("recall_metrics", {
-            sessionId: sid,
-            hitCount: result.recallMetrics.hitCount,
-            score: result.recallMetrics.score,
-            pass: result.recallMetrics.pass,
-            relevance: result.recallMetrics.relevance,
-            coverage: result.recallMetrics.coverage,
-            diversity: result.recallMetrics.diversity,
-            specificity: result.recallMetrics.specificity,
-        });
-    }
-    if (!result.empty && result.toInject.length > 0) {
-        const top = result.toInject[0];
-        const scorePct = Math.round((top.score ?? 0) * 100);
-        const files = top.checkpoint.filesModified ?? [];
-        const label = files.length
-            ? files
-                .map((f) => f.split("/").pop() ?? f)
-                .slice(0, 2)
-                .join(", ")
-            : top.checkpoint.checkpointId;
-        runtime.pushTicker(`${C.amber}↩${C.reset} recalled ${top.checkpoint.checkpointId} · ${scorePct}% · ${label}`);
-        runtime.lastWhy = `why: recalled@${scorePct}% (${result.toInject.length} chkpt)`;
-    }
-    let sumTokens = 0;
-    for (const h of result.toInject)
-        sumTokens += h.checkpoint.tokenEstimate;
-    if (result.toInject.length > 0) {
-        runtime.rt.recallInjections += result.toInject.length;
-        runtime.rt.cacheHitTokens += sumTokens;
-        incRecallInjected(result.toInject.length, runtime.currentStateDir);
-        incCacheHitTokens(sumTokens, runtime.currentStateDir);
-    }
-    // S43: record recall provenance — which checkpoints/summaries served this
-    // turn, their score + source path. Linked to the turn row written at
-    // turn_end via the conversation+turnIndex. Best-effort + non-fatal.
-    // Persists telemetry (HyDE + recall metrics) even when recall returned
-    // no hits, so empty-recall HyDE invocations are still visible in the
-    // dashboard Turns/Metrics tabs.
-    const hasTelemetry = result.hydeInfo != null || result.recallMetrics != null;
-    if (result.toInject.length > 0 || hasTelemetry) {
-        try {
-            const convId = ensureConversationIdFor(config, sid, runtime.currentStateDir);
-            const turnId = recordTurnWrite(config, {
-                conversationId: convId,
-                sessionId: sid,
-                turnIndex: runtime.currentTurn,
-                role: "assistant",
-                startedAt: Date.now(),
-                hyde: result.hydeInfo ?? undefined,
-                recallMetrics: result.recallMetrics ?? undefined,
-            }, runtime.currentStateDir);
-            if (result.toInject.length > 0) {
-                recordRecallWrite(config, turnId, result.toInject.map((h) => ({
-                    checkpointId: h.checkpoint.checkpointId,
-                    score: h.score,
-                    source: h.raptorLevel !== undefined
-                        ? "raptor"
-                        : h.repoId
-                            ? "cross-repo"
-                            : "flat",
-                    raptorLevel: h.raptorLevel,
-                })), runtime.currentStateDir);
-            }
-        }
-        catch {
-            /* non-fatal: recall provenance never breaks the recall path */
-        }
-    }
-    // VC0A: record recall latency on the eval observer (mode A) so the dashboard
-    // histogram reflects real data. No-op when the observer is absent (flag off /
-    // construction failure).
-    try {
-        recordRecallLatency(runtime, Date.now() - recallStartMs, sid, 0);
-    }
-    catch {
-        /* non-fatal: latency recording never breaks recall */
-    }
-    return result;
-}
-/**
- * S17: async recall with optional cross-repo augmentation. Used on resume
- * (session_start) and /mega-recall --cross-repo — NEVER from the mid-turn
- * context handler (that stays sync). Runs the sync same-repo scan first; if it
- * returns < config.autoInlineK hits AND crossRepo is enabled, awaits the PGlite
- * HNSW cross-repo path and merges (source-labeled, deduped by checkpointId). The
- * recallMaxTokens cap + windowDedupe apply to the merged set so cross-repo can
- * never net-inflate the window. Cross-repo uses a stricter cosine floor
- * (config.crossRepoCosine) than same-repo. Non-fatal: any async failure returns
- * the same-repo result unchanged.
- */
-export async function doRecallAsync(runtime, config, ctx, query, source, opts = {}) {
-    runtime.bindRepo(ctx.cwd);
-    const sid = normalizeSessionId(ctx.sessionManager.getSessionId());
-    const liveWindow = config.windowDedupe ? extractLiveWindow(ctx) : undefined;
-    // Sync same-repo first (fast, never blocks).
-    const sameRepo = recallAndInline({
-        sessionId: sid,
-        query,
-        limit: config.autoInlineK,
-        source,
-        skipInjected: true,
-        recallMaxTokens: config.recallMaxTokens,
-        windowDedupe: config.windowDedupe,
-        liveWindow,
-        dedupSim: config.dedupSim,
-    }, runtime.store);
-    if (!config.crossRepoEnabled || !opts.crossRepo)
-        return sameRepo;
-    if (sameRepo.toInject.length >= config.autoInlineK)
-        return sameRepo; // same-repo satisfied
-    // Augment: cross-repo HNSW (async) with the stricter floor. Non-fatal.
-    try {
-        const x = await recallAndInlineAsync({
-            sessionId: sid,
-            query,
-            limit: config.autoInlineK,
-            source,
-            skipInjected: true,
-            recallMaxTokens: config.recallMaxTokens,
-            windowDedupe: config.windowDedupe,
-            liveWindow,
-            dedupSim: config.crossRepoCosine,
-            crossRepo: true,
-            // F2: resolve the machine-wide index dir via the shared resolver so the
-            // cross-repo injected-set dedup works even when MEGACOMPACT_INDEX_DIR is
-            // unset. The env var still wins when set (getIndexDir checks it first);
-            // the default (~/.mega-compact-index) is the same DB mega-commands and the
-            // dashboard read, so injection counts stay consistent. Without this, a
-            // bare `process.env` read returns undefined → cross-repo hits re-inject in
-            // every new session (the global injected-set is never consulted).
-            globalIndexDir: getIndexDir(),
-        }, runtime.store);
-        runtime.dashboard.event("recall-crossrepo", {
-            source,
-            query: query.slice(0, 120),
-            injected: x.toInject.length,
-            sourceRepos: x.toInject.map((h) => h.repoId).filter(Boolean),
-        });
-        // Merge, dedup by checkpointId, respect the same token cap by reformatting.
-        const seen = new Set(sameRepo.toInject.map((h) => h.checkpoint.checkpointId));
-        const merged = [...sameRepo.toInject];
-        for (const h of x.toInject) {
-            if (!seen.has(h.checkpoint.checkpointId)) {
-                merged.push(h);
-                seen.add(h.checkpoint.checkpointId);
-            }
-        }
-        const block = merged.length ? formatRecallBlock(merged) : "";
-        if (merged.length > 0) {
-            let sumTokens = 0;
-            for (const h of merged)
-                sumTokens += h.checkpoint.tokenEstimate;
-            runtime.rt.recallInjections += merged.length;
-            runtime.rt.cacheHitTokens += sumTokens;
-            incRecallInjected(merged.length, runtime.currentStateDir);
-            incCacheHitTokens(sumTokens, runtime.currentStateDir);
-        }
-        return {
-            toInject: merged,
-            report: merged.map((h) => `  • ${h.checkpoint.checkpointId}${h.repoId ? ` (from ${h.repoId.split("/").filter(Boolean).pop()})` : ""}`),
-            block,
-            empty: merged.length === 0,
-            // H1: merged cross-repo result reuses the same-repo pass's telemetry.
-            hydeInfo: sameRepo.hydeInfo,
-            recallMetrics: sameRepo.recallMetrics,
-        };
-    }
-    catch {
-        return sameRepo; // cross-repo failure → same-repo only (non-fatal)
-    }
-}
-/**
- * Extract the live-window message texts from the session manager (Fix C),
- * for inline-dedupe of recalled checkpoints. Best-effort: returns [] on any
- * error so recall falls back to unbounded (still correct, just no dedupe).
- * Mirrors recentUserQuery's use of sessionEntryToContextMessages.
- */
-function extractLiveWindow(ctx) {
-    try {
-        const entries = ctx.sessionManager.getEntries();
-        const texts = [];
-        for (const e of entries) {
-            for (const m of sessionEntryToContextMessages(e)) {
-                const c = m.content;
-                if (typeof c === "string")
-                    texts.push(c);
-                else if (Array.isArray(c))
-                    texts.push(c.map((b) => b.text ?? "").join(" "));
-            }
-        }
-        return texts;
-    }
-    catch {
-        return [];
-    }
-}
+export { doRecall, doRecallAsync, extractLiveWindow, } from "./recall/impl.js";

package/dist/src/config.js

@@ -110,6 +110,15 @@ export const RAG_HYDE_ENABLED = () => ragEnabled("MEGACOMPACT_HYDE");
 /** Spec 1: vbrainstorm visual design migration for the dashboard. */
 export const NEW_UI = () => ragEnabled("MEGACOMPACT_NEW_UI");
 // ---------------------------------------------------------------------------
+// 3WF-3 same-repo recall cosine floor. SEPARATE from the S17 cross-repo floor
+// (config.crossRepoCosine, default 0.90 — stricter, cross-repo only). This is
+// the same-repo floor the 3-source validator applies to the top winner. A low
+// default (0.12) keeps recall permissive within a repo while still rejecting
+// effectively-unrelated hits. Call-time read so tests can set the env per-test.
+// ---------------------------------------------------------------------------
+/** Same-repo recall cosine floor: top winner must be >= this to be injected. */
+export const RECALL_MIN_COSINE = () => Number(process.env.MEGACOMPACT_RECALL_MIN_COSINE ?? "0.12");
+// ---------------------------------------------------------------------------
 // Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
 // default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
 // so root consumers share one source of truth.

package/dist/src/failback/floor.js

@@ -0,0 +1,35 @@
+/** Floor text when the newest checkpoint summary is available (prefix). */
+const WITH_SUMMARY_PREFIX = "The following compacted context is the most recent checkpoint from " +
+    "this session (recall found no query-relevant match):\n\n";
+/** Floor text when checkpoints exist but no usable summary does. */
+const NO_SUMMARY_TEXT = "This session has compacted context but recall could not surface a " +
+    "checkpoint relevant to the current request; the most recent checkpoint " +
+    "summary is unavailable.";
+/** Floor text when the checkpoint read itself failed (hard last resort). */
+export const FLOOR_UNAVAILABLE_TEXT = "This session has compacted context but recall could not surface a " +
+    "checkpoint relevant to the current request.";
+/** The newest checkpoint by timestamp (first element wins ties, as before). */
+export function newestCheckpoint(cps) {
+    let newest = cps[0];
+    for (const cp of cps) {
+        if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0))
+            newest = cp;
+    }
+    return newest;
+}
+/**
+ * Build the provenance floor from an already-read checkpoint list. Pure: the
+ * caller owns the read (and any dedup-status filtering), so both legacy call
+ * sites keep byte-identical output.
+ */
+export function buildFloorBlock(cps) {
+    const summary = newestCheckpoint(cps)?.summary?.trim();
+    if (summary) {
+        return { text: WITH_SUMMARY_PREFIX + summary, basis: "lastCheckpoint" };
+    }
+    return { text: NO_SUMMARY_TEXT, basis: "lastCheckpoint" };
+}
+/** The hard last-resort floor (checkpoint read unavailable or threw). */
+export function unavailableFloorBlock() {
+    return { text: FLOOR_UNAVAILABLE_TEXT, basis: "none" };
+}

package/dist/src/recall/readonly.js

@@ -0,0 +1,39 @@
+/**
+ * recall/readonly.ts — read-only recall variant (3WF-3 Source A).
+ *
+ * A pure search+rank seam wrapping `engine.recall`'s RAW `hits` path. It is the
+ * canonical read-only entry point going forward (triggerGuard.ts still inlines
+ * `recall(...).hits` for its own need; this module is additive and does NOT
+ * refactor it).
+ *
+ * HARD contract (QA): this module MUST NOT call `vectorMarkInjected`, must NOT
+ * write any turn/recall rows, and must NOT emit S43 telemetry. It only searches
+ * and returns hits for the vote. RecallAndInline's inject loop is the ONLY place
+ * the injected-set is mutated; keying the vote on raw `hits` (skipInjected:false
+ * => hits === newHits) is deliberate — `newHits` is post-`skipInjected` filter,
+ * which would distort overlap appearance.
+ *
+ * Non-fatal: any failure returns [] so the caller degrades to other sources.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { recall } from "../engine.js";
+/**
+ * Raw, read-only recall hits for the 3-source vote. Returns `engine.recall`'s
+ * RAW `.hits` (skipInjected:false => equals the unfiltered vector result). No
+ * injected-set mutation, no turn writes, no telemetry. Returns [] on failure.
+ */
+export function recallRawHits(opts, store) {
+    try {
+        const result = recall({
+            sessionId: opts.sessionId,
+            query: opts.query,
+            limit: opts.limit ?? 3,
+            skipInjected: false,
+        }, store);
+        return result.hits;
+    }
+    catch {
+        // Non-fatal: never break the agent loop. Degrade to other sources.
+        return [];
+    }
+}

package/dist/src/recall/recall3wf.fixture.js

@@ -0,0 +1,67 @@
+/**
+ * src/recall/recall3wf.fixture.ts — shared fixtures for the 3WF-3 recall tests.
+ *
+ * Split out of recall3wf.test.ts (which crossed the src 300 soft cap) so each
+ * test file stays under the limit. These are REAL fixtures, not mocks/stubs:
+ * a REAL VectorStore over a temp stateDir, REAL checkpoints persisted via
+ * compactSession, and readers that go through the SAME working path the
+ * extension uses (recallRawHits -> vectorSearch -> listCheckpoints, and
+ * vectorWasInjected), mirroring the proven triggerGuard test pattern.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { VectorStore } from "../vectorStore.js";
+import { compactSession } from "../engine.js";
+import { recallAndInline } from "../recall.js";
+import { recallRawHits } from "./readonly.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { initSchema } from "../store/sqlite/schema.js";
+/** Real EngineMessage fixture. */
+export function msg(role, text) {
+    return { role, text };
+}
+/** Fresh isolated state dir per VectorStore. */
+export function freshStore() {
+    const dir = mkdtempSync(join(tmpdir(), "mc-3wf-"));
+    return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
+}
+/** Persist N distinct checkpoints with distinct content + ascending timestamps. */
+export function seed(store, topics, sid = "sess_3wf") {
+    topics.forEach((t, i) => {
+        compactSession({
+            sessionId: sid,
+            messages: [msg("user", t), msg("assistant", "ok")],
+            keepFrom: 2,
+            timestamp: i + 1,
+        }, store);
+    });
+}
+/** Checkpoint ids via the real search path (vectorSearch -> listCheckpoints). */
+export function checkpointIds(store, sid, query) {
+    return recallRawHits({ sessionId: sid, query, limit: 10 }, store).map((h) => h.checkpoint.checkpointId);
+}
+/** Run the real recallAndInline path with skipInjected:false so nothing is
+ *  marked and the block reflects the search result exactly (deterministic). */
+export function recallAndInlineCapture(sid, query, store) {
+    const r = recallAndInline({ sessionId: sid, query, limit: 3, source: "command", skipInjected: false, windowDedupe: false }, store);
+    return { block: r.block, empty: r.empty, toInject: r.toInject };
+}
+/** Count recall-provenance rows (turn_recall) for a session via raw SQL reader. */
+export function countTurnRecallRows(store, sid) {
+    try {
+        const reader = openStore(store.stateDir);
+        // Ensure the turns/turn_recall tables exist so a 0-count is meaningful
+        // (a write on the new path would be visible, not masked by a missing table).
+        initSchema(reader);
+        const row = reader
+            .prepare(`SELECT COUNT(*) AS n FROM turn_recall tr
+				 JOIN turns t ON t.id = tr.turn_id
+				 WHERE t.session_id = ?`)
+            .get(sid);
+        return row?.n ?? 0;
+    }
+    catch {
+        return 0;
+    }
+}

package/dist/src/recall/validator.js

@@ -0,0 +1,99 @@
+/**
+ * recall/validator.ts — independent candidate validator (3WF-3).
+ *
+ * Judges candidates handed to it; it MUST NOT call any search itself. Given the
+ * ranked vote winners + the live-window text (already extracted by the caller,
+ * since src/ cannot import pi types), it walks the winners in order and returns
+ * the first that passes BOTH gates:
+ *
+ *   1. Cosine floor: the winner's score >= the same-repo floor (default 0.12,
+ *      env MEGACOMPACT_RECALL_MIN_COSINE). The cross-repo 0.90 floor
+ *      (config.crossRepoCosine) is SEPARATE and intentionally untouched.
+ *   2. Not already resident in the live window: reuse recall/sync.ts's exact
+ *      comparison — embed each live message, embed the checkpoint summary, and
+ *      treat the checkpoint as resident when cosineSimilarity >= dedupSim. We
+ *      reuse that metric rather than inventing a new one.
+ *
+ * On a failing candidate it advances to the next-ranked winner. If ALL fail it
+ * returns the provenance floor (FloorBlock built from the newest checkpoint —
+ * pure over checkpoints, same semantics as triggerGuard's buildFloorBlock).
+ *
+ * Non-fatal throughout: any error degrades to the next candidate / the floor.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
+// SQLite store, NOT src/store.ts's legacy gzipped-JSON DR reader (that returns
+// [] for live sessions). Mirrors vector-search.ts / tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { RECALL_MIN_COSINE } from "../config.js";
+import { buildFloorBlock as sharedFloorBlock, unavailableFloorBlock, } from "../failback/floor.js";
+/**
+ * Build the provenance floor block from the session's newest checkpoint.
+ *
+ * 3WF-4: the text construction moved to the SHARED pure builder
+ * (src/failback/floor.ts). This wrapper keeps THIS call site's read semantics —
+ * `listCheckpoints` filtered to `dedupStatus !== "removed"` — so the output is
+ * byte-identical to the pre-refactor 3WF-3 version.
+ */
+function buildFloorBlock(sessionId, store) {
+    try {
+        const cps = listCheckpoints(sessionId, store.stateDir).filter((c) => c.dedupStatus !== "removed");
+        return sharedFloorBlock(cps);
+    }
+    catch {
+        return unavailableFloorBlock();
+    }
+}
+/**
+ * Validate the ranked vote winners, returning the first that passes both gates,
+ * or the provenance floor if none do. Does NOT mutate the injected set, does NOT
+ * write turns, does NOT emit telemetry. Non-fatal.
+ */
+export function validateRecall(winners, opts, store) {
+    const floor = RECALL_MIN_COSINE();
+    const dedupSim = opts.dedupSim ?? 0.9;
+    const embedder = defaultEmbedder();
+    const liveVecs = (opts.liveWindow ?? []).map((m) => embedder.embed(m));
+    // One checkpoint read for the whole pass (both gates share it).
+    const cps = listCheckpoints(opts.sessionId, store.stateDir);
+    const cpById = new Map(cps.map((c) => [c.checkpointId, c]));
+    const queryVec = opts.query ? embedder.embed(opts.query) : null;
+    for (const cand of winners) {
+        try {
+            const cp = cpById.get(cand.checkpointId);
+            // Gate 1: same-repo COSINE floor. `cand.score` is only a cosine for
+            // source "vector"; fts5 (BM25) and recency (freshness rank) live on
+            // other scales, so for those we re-derive the true cosine locally from
+            // the query + checkpoint embedding. No search call is made.
+            let cosine;
+            if (cand.source === "vector") {
+                cosine = cand.score;
+            }
+            else if (queryVec && cp) {
+                cosine = cosineSimilarity(queryVec, embedder.embed(cp.summary));
+            }
+            else {
+                // No comparable cosine available => cannot clear a cosine gate.
+                continue;
+            }
+            if (cosine < floor)
+                continue;
+            // Gate 2: not already resident in the live window.
+            if (liveVecs.length > 0) {
+                if (!cp)
+                    continue; // cannot verify => skip rather than risk re-inject
+                const hitVec = embedder.embed(cp.summary);
+                const resident = liveVecs.some((v) => cosineSimilarity(v, hitVec) >= dedupSim);
+                if (resident)
+                    continue;
+            }
+            return { kind: "candidate", candidate: cand };
+        }
+        catch {
+            // Non-fatal: skip this candidate, try the next.
+            continue;
+        }
+    }
+    // All candidates rejected -> provenance floor.
+    return { kind: "floor", floor: buildFloorBlock(opts.sessionId, store) };
+}