pi-mega-compact 0.20.85 → 0.20.87

package/src/failback/compact.ts

@@ -0,0 +1,122 @@
+/**
+ * src/failback/compact.ts — 3WF-2 candidate-veto + vote module (pure, advisory).
+ *
+ * The production bug this fixes: compaction "succeeded" (a checkpoint was
+ * persisted, `saved` grew) while the LIVE WINDOW (`currentTokens`) never
+ * shrank — because `saved` is a cumulative SQLite total, not the working-set
+ * delta. This module builds competing summary candidates (extractive vs
+ * cluster/raptor) and VOTES which one, if any, is worth replacing the
+ * supersede-only result. It is purely advisory/observational: it never mutates
+ * a checkpoint, never overwrites `result.summary`, and returning `null` means
+ * "keep the supersede-only result" — the caller must NOT substitute a summary.
+ *
+ * Pure: no store mutation, no I/O, no network, no console.*. pi-agnostic
+ * (imports only from src/). Designed for the 3WF umbrella flag gate at the
+ * extension layer (see extensions/mega-pipeline/compact/vote.ts).
+ */
+
+import type { EngineMessage } from "../types.js";
+import { collectRecentUserRequests, summarizeMessages } from "../compact.js";
+import { summarizeCluster } from "../dedup/raptor/summarizer.js";
+import { estimateBlockTokens } from "../tokens.js";
+import type { CompactCandidate } from "./types.js";
+
+/**
+ * Default floor (tokens of net reduction) below which a candidate vote is
+ * REJECTED, returning `null` (keep the supersede-only result).
+ *
+ * Rationale: a candidate that reduces the region by fewer than 1 token is not
+ * meaningfully smaller than the compacted region it would replace — swapping
+ * the supersede-only result for it buys nothing and only adds a (possibly
+ * less faithful) summary. The floor therefore requires the voted summary to be
+ * STRICTLY smaller than the compacted region. Set to 1 (minimally defensible:
+ * the summary must actually be smaller). Overridable via `opts.floor`.
+ */
+export const DEFAULT_VOTE_FLOOR_TOKENS = 1;
+
+/** Strip a trailing ellipsis/truncation marker from a needle before containment. */
+function stripEllipsis(s: string): string {
+	// collectRecentUserRequests truncates to 160 chars via compact.ts's truncate,
+	// which appends the U+2026 ellipsis when it cuts. Drop it for a fair test.
+	return s.replace(/…\s*$/u, "").trim();
+}
+
+/** Normalize for containment: collapse whitespace, lowercase. */
+function normalize(s: string): string {
+	return s.replace(/\s+/g, " ").trim().toLowerCase();
+}
+
+/** True when `summary` contains the content of EVERY recent user request. */
+export function signalPreserved(summary: string, messages: EngineMessage[]): boolean {
+	const requests = collectRecentUserRequests(messages, 3);
+	if (requests.length === 0) return true; // nothing to preserve
+	const haystack = normalize(summary);
+	return requests.every((r) => haystack.includes(normalize(stripEllipsis(r))));
+}
+
+/**
+ * Build the two competing candidates (extractive + cluster/raptor) for a
+ * compacted message region. Degenerate (empty/whitespace-only) summaries are
+ * VETOED — never returned. Both candidates use estimateBlockTokens(summary) for
+ * a single consistent token basis so the vote compares like with like (the
+ * cluster path's own tokenEstimate is intentionally ignored for fairness).
+ */
+export function buildCandidates(messages: EngineMessage[]): CompactCandidate[] {
+	const out: CompactCandidate[] = [];
+
+	const extractive = summarizeMessages(messages);
+	if (extractive.trim().length > 0) {
+		out.push({
+			source: "extractive",
+			summary: extractive,
+			tokenEstimate: estimateBlockTokens(extractive),
+			signalPreserved: signalPreserved(extractive, messages),
+		});
+	}
+
+	// summarizeCluster returns deterministic extractive when MEGACOMPACT_RAPTOR_MODEL
+	// is unset and the local-only Ollama variant when set — so using it makes the
+	// Ollama path an insertion that adds NO new LLM call site for the on-by-default
+	// extraction. No behavior change for the default config.
+	const cluster = summarizeCluster(messages).summary;
+	if (cluster.trim().length > 0) {
+		out.push({
+			source: "cluster",
+			summary: cluster,
+			tokenEstimate: estimateBlockTokens(cluster),
+			signalPreserved: signalPreserved(cluster, messages),
+		});
+	}
+
+	return out;
+}
+
+/**
+ * Vote the best candidate. `score = reduction * (signalPreserved ? 1 : 0.5)`
+ * where `reduction = tokensBefore - candidate.tokenEstimate`. Ties resolve to
+ * the EARLIER (extractive) candidate for determinism. Returns `null` when the
+ * winner's score is below `opts.floor` (default DEFAULT_VOTE_FLOOR_TOKENS) —
+ * caller MUST keep the supersede-only result and must NOT substitute a summary.
+ */
+export function voteCandidate(
+	messages: EngineMessage[],
+	tokensBefore: number,
+	opts: { floor?: number } = {},
+): CompactCandidate | null {
+	const floor = opts.floor ?? DEFAULT_VOTE_FLOOR_TOKENS;
+	const candidates = buildCandidates(messages);
+	let best: CompactCandidate | null = null;
+	let bestScore = -Infinity;
+	for (const c of candidates) {
+		const reduction = tokensBefore - c.tokenEstimate;
+		const score = reduction * (c.signalPreserved ? 1 : 0.5);
+		// Earlier candidate wins ties (strict > keeps insertion order = extractive first).
+		if (score > bestScore) {
+			bestScore = score;
+			best = c;
+		}
+	}
+	if (best === null) return null;
+	if (bestScore < floor) return null;
+	return best;
+}

package/src/failback/types.ts

@@ -44,3 +44,75 @@ export interface GuardOpts {
 	/** Max hits to recall (mirrors autoInlineK). */
 	limit: number;
 }
+
+/** A competing compaction summary candidate produced by the 3-source vote. */
+export interface CompactCandidate {
+	/** Which generator produced this candidate (structural telemetry label —
+	 *  never infer the source by sniffing the summary text). */
+	source: "extractive" | "cluster";
+	/** The candidate summary text (extractive or cluster/raptor variant). */
+	summary: string;
+	/** Estimated token cost of the candidate summary (estimateBlockTokens basis). */
+	tokenEstimate: number;
+	/** True when the summary preserves every recent user request signal. */
+	signalPreserved: boolean;
+}
+
+/**
+ * The measured reduction verdict across consecutive `context` events in the
+ * LIVE WINDOW (the model's current working tokens), NOT the stored-checkpoint
+ * `saved` metric. The live-window `currentTokens` delta is the real signal of
+ * whether compaction actually freed working context; the stored `saved` field
+ * is a cumulative SQLite total that can look healthy while the live window is
+ * unchanged — the false metric behind the production thrash bug this sprint
+ * fixes. `liveBefore`/`liveAfter` are the live-window token counts bracketing
+ * the compaction.
+ */
+export interface ReductionVerdict {
+	/** True when the live window measurably shrank after compaction. */
+	effective: boolean;
+	/** Live-window token count before the compaction event. */
+	liveBefore: number;
+	/** Live-window token count after the compaction event. */
+	liveAfter: number;
+}
+
+/**
+ * State of the compaction thrash guard. Arms after a compaction that produced
+ * no live-window reduction, so we do not re-fire into a window that cannot
+ * shrink. Re-arms or clears as the live window grows again.
+ */
+export interface ThrashGuardState {
+	/** Live-window token count below which re-firing is refused (guard active). */
+	blockedUntilTokens: number;
+	/** ms epoch at which the guard was armed. */
+	armedAt: number;
+}
+
+/**
+ * A single recall candidate surfaced by one of the three independent, read-only
+ * recall sources in the 3WF-3 vote. `score` is the raw per-source score; the
+ * voter normalizes each source to a comparable 0..1 before combining.
+ */
+export interface RecallCandidate {
+	/** Checkpoint id named by the source. */
+	checkpointId: string;
+	/** Raw per-source relevance score (scale depends on `source`). */
+	score: number;
+	/** Which independent source named this candidate. */
+	source: "vector" | "fts5" | "recency";
+}
+
+/**
+ * Outcome of the three-source recall vote. `winners` are the agreed candidates
+ * (ranked), `votes` counts how many distinct sources named each checkpointId,
+ * and `divergentSources` lists the sources that contributed no winner.
+ */
+export interface VoteResult {
+	/** Agreed candidates, ranked best-first. */
+	winners: RecallCandidate[];
+	/** Per-checkpointId vote count (1..3 distinct sources). */
+	votes: Record<string, number>;
+	/** Source names that produced no winning candidate. */
+	divergentSources: string[];
+}

package/src/recall/readonly.ts

@@ -0,0 +1,57 @@
+/**
+ * recall/readonly.ts — read-only recall variant (3WF-3 Source A).
+ *
+ * A pure search+rank seam wrapping `engine.recall`'s RAW `hits` path. It is the
+ * canonical read-only entry point going forward (triggerGuard.ts still inlines
+ * `recall(...).hits` for its own need; this module is additive and does NOT
+ * refactor it).
+ *
+ * HARD contract (QA): this module MUST NOT call `vectorMarkInjected`, must NOT
+ * write any turn/recall rows, and must NOT emit S43 telemetry. It only searches
+ * and returns hits for the vote. RecallAndInline's inject loop is the ONLY place
+ * the injected-set is mutated; keying the vote on raw `hits` (skipInjected:false
+ * => hits === newHits) is deliberate — `newHits` is post-`skipInjected` filter,
+ * which would distort overlap appearance.
+ *
+ * Non-fatal: any failure returns [] so the caller degrades to other sources.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { recall } from "../engine.js";
+import type { VectorStore } from "../vectorStore.js";
+import type { SearchHit } from "../vectorStore.js";
+
+/** Options for the read-only recall seam. */
+export interface ReadonlyRecallOptions {
+	/** Normalized session id. */
+	sessionId: string;
+	/** Recall query text. */
+	query: string;
+	/** Max hits to return (default 3). */
+	limit?: number;
+}
+
+/**
+ * Raw, read-only recall hits for the 3-source vote. Returns `engine.recall`'s
+ * RAW `.hits` (skipInjected:false => equals the unfiltered vector result). No
+ * injected-set mutation, no turn writes, no telemetry. Returns [] on failure.
+ */
+export function recallRawHits(
+	opts: ReadonlyRecallOptions,
+	store: VectorStore,
+): SearchHit[] {
+	try {
+		const result = recall(
+			{
+				sessionId: opts.sessionId,
+				query: opts.query,
+				limit: opts.limit ?? 3,
+				skipInjected: false,
+			},
+			store,
+		);
+		return result.hits;
+	} catch {
+		// Non-fatal: never break the agent loop. Degrade to other sources.
+		return [];
+	}
+}

package/src/recall/recall3wf.fixture.ts

@@ -0,0 +1,87 @@
+/**
+ * src/recall/recall3wf.fixture.ts — shared fixtures for the 3WF-3 recall tests.
+ *
+ * Split out of recall3wf.test.ts (which crossed the src 300 soft cap) so each
+ * test file stays under the limit. These are REAL fixtures, not mocks/stubs:
+ * a REAL VectorStore over a temp stateDir, REAL checkpoints persisted via
+ * compactSession, and readers that go through the SAME working path the
+ * extension uses (recallRawHits -> vectorSearch -> listCheckpoints, and
+ * vectorWasInjected), mirroring the proven triggerGuard test pattern.
+ */
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { VectorStore } from "../vectorStore.js";
+import { compactSession } from "../engine.js";
+import { recallAndInline } from "../recall.js";
+import { recallRawHits } from "./readonly.js";
+import { openStore } from "../store/sqlite/utils.js";
+import { initSchema } from "../store/sqlite/schema.js";
+
+/** Real EngineMessage fixture. */
+export function msg(role: "user" | "assistant", text: string): any {
+	return { role, text };
+}
+
+/** Fresh isolated state dir per VectorStore. */
+export function freshStore(): { store: VectorStore; dir: string } {
+	const dir = mkdtempSync(join(tmpdir(), "mc-3wf-"));
+	return { store: new VectorStore({ dedupSim: 0.9, stateDir: dir }), dir };
+}
+
+/** Persist N distinct checkpoints with distinct content + ascending timestamps. */
+export function seed(store: VectorStore, topics: string[], sid = "sess_3wf"): void {
+	topics.forEach((t, i) => {
+		compactSession(
+			{
+				sessionId: sid,
+				messages: [msg("user", t), msg("assistant", "ok")],
+				keepFrom: 2,
+				timestamp: i + 1,
+			},
+			store,
+		);
+	});
+}
+
+/** Checkpoint ids via the real search path (vectorSearch -> listCheckpoints). */
+export function checkpointIds(store: VectorStore, sid: string, query: string): string[] {
+	return recallRawHits({ sessionId: sid, query, limit: 10 }, store).map(
+		(h) => h.checkpoint.checkpointId,
+	);
+}
+
+/** Run the real recallAndInline path with skipInjected:false so nothing is
+ *  marked and the block reflects the search result exactly (deterministic). */
+export function recallAndInlineCapture(
+	sid: string,
+	query: string,
+	store: VectorStore,
+): { block: string; empty: boolean; toInject: unknown[] } {
+	const r = recallAndInline(
+		{ sessionId: sid, query, limit: 3, source: "command", skipInjected: false, windowDedupe: false },
+		store,
+	);
+	return { block: r.block, empty: r.empty, toInject: r.toInject };
+}
+
+/** Count recall-provenance rows (turn_recall) for a session via raw SQL reader. */
+export function countTurnRecallRows(store: VectorStore, sid: string): number {
+	try {
+		const reader = openStore(store.stateDir);
+		// Ensure the turns/turn_recall tables exist so a 0-count is meaningful
+		// (a write on the new path would be visible, not masked by a missing table).
+		initSchema(reader);
+		const row = reader
+			.prepare(
+				`SELECT COUNT(*) AS n FROM turn_recall tr
+				 JOIN turns t ON t.id = tr.turn_id
+				 WHERE t.session_id = ?`,
+			)
+			.get(sid) as { n: number };
+		return row?.n ?? 0;
+	} catch {
+		return 0;
+	}
+}

package/src/recall/validator.ts

@@ -0,0 +1,150 @@
+/**
+ * recall/validator.ts — independent candidate validator (3WF-3).
+ *
+ * Judges candidates handed to it; it MUST NOT call any search itself. Given the
+ * ranked vote winners + the live-window text (already extracted by the caller,
+ * since src/ cannot import pi types), it walks the winners in order and returns
+ * the first that passes BOTH gates:
+ *
+ *   1. Cosine floor: the winner's score >= the same-repo floor (default 0.12,
+ *      env MEGACOMPACT_RECALL_MIN_COSINE). The cross-repo 0.90 floor
+ *      (config.crossRepoCosine) is SEPARATE and intentionally untouched.
+ *   2. Not already resident in the live window: reuse recall/sync.ts's exact
+ *      comparison — embed each live message, embed the checkpoint summary, and
+ *      treat the checkpoint as resident when cosineSimilarity >= dedupSim. We
+ *      reuse that metric rather than inventing a new one.
+ *
+ * On a failing candidate it advances to the next-ranked winner. If ALL fail it
+ * returns the provenance floor (FloorBlock built from the newest checkpoint —
+ * pure over checkpoints, same semantics as triggerGuard's buildFloorBlock).
+ *
+ * Non-fatal throughout: any error degrades to the next candidate / the floor.
+ * Pi-agnostic: no pi runtime imports.
+ */
+import { defaultEmbedder, cosineSimilarity } from "../embedder.js";
+// SQLite store, NOT src/store.ts's legacy gzipped-JSON DR reader (that returns
+// [] for live sessions). Mirrors vector-search.ts / tieredRouter.ts.
+import { listCheckpoints } from "../store/sqlite.js";
+import { RECALL_MIN_COSINE } from "../config.js";
+import type { VectorStore } from "../vectorStore.js";
+import type { RecallCandidate, FloorBlock } from "../failback/types.js";
+
+/** Options for the recall validator. */
+export interface ValidateOptions {
+	/** Normalized session id (for floor-block construction). */
+	sessionId: string;
+	/**
+	 * The recall query. Required for a TRUE cosine gate: only `source:"vector"`
+	 * candidates carry a cosine in `score` (fts5 carries BM25, recency carries a
+	 * freshness rank), so comparing a raw mixed-scale score against a cosine
+	 * floor would be meaningless. When supplied, the validator re-derives each
+	 * candidate's cosine against the query locally (embedder only — never a
+	 * search call, so the "independent of all three search calls" contract
+	 * holds). When omitted, only `vector` candidates can clear the gate.
+	 */
+	query?: string;
+	/** Live-window message texts already extracted by the caller. */
+	liveWindow?: string[];
+	/** Dedup similarity threshold for the live-window resident check. */
+	dedupSim?: number;
+}
+
+/** A validated winner, or the provenance floor when all candidates fail. */
+export type ValidationOutcome =
+	| { kind: "candidate"; candidate: RecallCandidate }
+	| { kind: "floor"; floor: FloorBlock };
+
+/** Build the provenance floor block from the session's newest checkpoint. */
+function buildFloorBlock(sessionId: string, store: VectorStore): FloorBlock {
+	try {
+		const cps = listCheckpoints(sessionId, store.stateDir).filter(
+			(c) => c.dedupStatus !== "removed",
+		);
+		let newest = cps[0];
+		for (const cp of cps) {
+			if (!newest || (cp.timestamp ?? 0) > (newest.timestamp ?? 0)) newest = cp;
+		}
+		const summary = newest?.summary?.trim();
+		if (summary) {
+			return {
+				text:
+					"The following compacted context is the most recent checkpoint from " +
+					"this session (recall found no query-relevant match):\n\n" + summary,
+				basis: "lastCheckpoint",
+			};
+		}
+		return {
+			text:
+				"This session has compacted context but recall could not surface a " +
+				"checkpoint relevant to the current request; the most recent checkpoint " +
+				"summary is unavailable.",
+			basis: "lastCheckpoint",
+		};
+	} catch {
+		return {
+			text:
+				"This session has compacted context but recall could not surface a " +
+				"checkpoint relevant to the current request.",
+			basis: "none",
+		};
+	}
+}
+
+/**
+ * Validate the ranked vote winners, returning the first that passes both gates,
+ * or the provenance floor if none do. Does NOT mutate the injected set, does NOT
+ * write turns, does NOT emit telemetry. Non-fatal.
+ */
+export function validateRecall(
+	winners: RecallCandidate[],
+	opts: ValidateOptions,
+	store: VectorStore,
+): ValidationOutcome {
+	const floor = RECALL_MIN_COSINE();
+	const dedupSim = opts.dedupSim ?? 0.9;
+	const embedder = defaultEmbedder();
+	const liveVecs = (opts.liveWindow ?? []).map((m) => embedder.embed(m));
+	// One checkpoint read for the whole pass (both gates share it).
+	const cps = listCheckpoints(opts.sessionId, store.stateDir);
+	const cpById = new Map(cps.map((c) => [c.checkpointId, c]));
+	const queryVec = opts.query ? embedder.embed(opts.query) : null;
+
+	for (const cand of winners) {
+		try {
+			const cp = cpById.get(cand.checkpointId);
+
+			// Gate 1: same-repo COSINE floor. `cand.score` is only a cosine for
+			// source "vector"; fts5 (BM25) and recency (freshness rank) live on
+			// other scales, so for those we re-derive the true cosine locally from
+			// the query + checkpoint embedding. No search call is made.
+			let cosine: number;
+			if (cand.source === "vector") {
+				cosine = cand.score;
+			} else if (queryVec && cp) {
+				cosine = cosineSimilarity(queryVec, embedder.embed(cp.summary));
+			} else {
+				// No comparable cosine available => cannot clear a cosine gate.
+				continue;
+			}
+			if (cosine < floor) continue;
+
+			// Gate 2: not already resident in the live window.
+			if (liveVecs.length > 0) {
+				if (!cp) continue; // cannot verify => skip rather than risk re-inject
+				const hitVec = embedder.embed(cp.summary);
+				const resident = liveVecs.some(
+					(v) => cosineSimilarity(v, hitVec) >= dedupSim,
+				);
+				if (resident) continue;
+			}
+
+			return { kind: "candidate", candidate: cand };
+		} catch {
+			// Non-fatal: skip this candidate, try the next.
+			continue;
+		}
+	}
+
+	// All candidates rejected -> provenance floor.
+	return { kind: "floor", floor: buildFloorBlock(opts.sessionId, store) };
+}