pi-mega-compact 0.20.6 → 0.20.7

package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts

@@ -0,0 +1,142 @@
+/**
+ * dashboard-server/routes-rag-settings-vector-cortex.ts — Vector Cortex SETTINGS.
+ *
+ * The VC0A..VC5C chain flag inventory, split out of routes-rag-settings-helpers.ts
+ * to keep that file under the 400-line extensions/ soft limit. Every VC flag is a
+ * user-visible dashboard toggle (never in EXCLUDED_SETTINGS) per the chain spec,
+ * and each carries the long description the Setup panel renders as help text.
+ *
+ * PREVENT-011: no `any` type.
+ */
+
+import type { SettingSpec, SettingGroup } from "./routes-rag-settings-types.js";
+
+const boolDirect = (
+	key: string,
+	label: string,
+	description: string,
+	def: boolean,
+): SettingSpec => ({
+	key,
+	label,
+	description,
+	type: "boolean",
+	default: def,
+	disabledConvention: false,
+	requiresLlm: false,
+});
+
+/** The Vector Cortex chain flags, as one SETTINGS category. */
+export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
+	name: "Vector Cortex",
+	settings: [
+		boolDirect(
+			"MEGACOMPACT_VC0A",
+			"VC0A Baseline Observability",
+			"Structured evaluation observer (MetricEventV1 + latency histogram). OFF = mode C, byte-identical to predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC0B",
+			"VC0B Replay Correctness",
+			"ReplayCutV2 effective-cut (min of boundary-safe/commit/capture high-water + pair retreat + anchor floor) and M3 effective-cut-v2 migration. OFF = legacy capped replay, byte-identical.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC1A",
+			"VC1A Canonical Byte Events",
+			"EventV2 byte-authority ledger codec (original bytes + SHA-256, strict UTF-8, derived NFC) and canonical validator (EVT_DIGEST_MISMATCH / EVT_UTF8_TAG_INVALID / EVT_DUPLICATE_ID). OFF = mode C, transcript codec unchanged, byte-identical.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC1B",
+			"VC1B Occurrence Ledger + Tool Identity",
+			"Neutral occurrence ledger (LedgerReader/Writer/Admin + CompatJournalV1): per-session monotonic seq, tool result references one earlier call, uniqueness by (eventId,digest) only, and the M2 copy-validate-switch downgrade journal. OFF = mode C, ledger unwritten, byte-identical.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC0C",
+			"VC0C Live Safety Envelope",
+			"TriadResult/Breaker live circuit breaker (60s window, 20 attempts, 30s cooldown, 3 probes, 5min healthy residence) + durable spool before provider invocation; manual reset clears cooldown but never evidence. OFF = mode C, unchanged transcript, byte-identical.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC1C",
+			"VC1C Cross-Language Conformance v2",
+			"FixtureManifestV2 canonical manifest validator + DowngradeReport deterministic downgrade export + MinHashV2 exact big-integer signatures and the M4 copy/validate/switch minhash-v2 migration (seed table frozen, cross-language byte-exact). OFF = mode C, v1 sync dedup scan unchanged, byte-identical.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC2A",
+			"VC2A Offline Model Runtime",
+			"ModelManifestV1 digest-before-load ONNX runtime (opset17/batch1/max512) + asset-free trigram demotion. Asset path assets/vector-cortex/encoder-v1 is immutable/digest-pinned. OFF = mode C, byte-identical to predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC2B",
+			"VC2B Multi-Head Encoder",
+			"VectorSetV1 five L2-normalized heads (384/128/128/64/32) with head-calibration draft + asset-free trigram B (512d) and lexical C fallbacks, plus the per-head emit seam. OFF = mode C, no per-head vectors emitted, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC2C",
+			"VC2C Encoder Qualification + Calibration",
+			"QualifiedEncoderV1/CalibrationV1: calibration fit on the calibration split only (held-out labels prohibited) + atomic selection across MODEL_ASSET and per-head EVALUATION thresholds (any field failure demotes all of A). OFF = mode C, no qualification/calibration selection, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC3A",
+			"VC3A Cortex Store",
+			"Capability-gated derived cortex store (CortexReader/Writer/Admin + CortexRecordV1): additive, keyed (sourceHighWater, algorithmVersion, id), immutable records, deterministic generation rebuild + one root digest. OFF = mode C, no cortex records written, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC3B",
+			"VC3B Deterministic Topology",
+			"Deterministic cortical topology (TopologyV1/EdgeV1): per-(source,head) top-k=16/head calibrated-threshold edges, stable score-desc/then-target-ID sort, dependency directed + contradiction symmetric paired records, one stable generation digest. OFF = mode C, no topology graph built/emitted, predecessor-precise topology view, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC3C",
+			"VC3C Topology Query + Router Invalidation",
+			"TopologyQueryV1/RouterKeyV2 structured keys (length-delimited, unsigned-byte order, no prefix ambiguity), exact (session,generation) invalidation, stale-generation rejection (TOP_GENERATION_STALE), and the M6 router-generation-v2 copy/validate/switch migration. OFF = mode C, no structured router key / generation invalidation, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC4A",
+			"VC4A Dual-Tier Shards",
+			"SemanticShardV1/ExactShardV1/ShardManifestV1: partition a session ONLY at complete EventV2 boundaries; exact shards preserve every tool call/result pair, anchor and invalid UTF-8 event as original bytes (pairs never split across exact shards); manifest enforces disjoint sorted ranges + complete protected-span coverage. OFF = mode C, exact anchors/current transcript only, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC4B",
+			"VC4B Residual Basis Parity",
+			"Residual codec: orthonormal DCT-II basis + int16 block quantization + block-scoped exact correction stream + (9,6) Reed-Solomon parity shards with SHA-256 corruption detection; admission gates on encodedSize <= 95% of exact-compressed size. OFF = mode C, no residual artifact produced, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC4C",
+			"VC4C Reconstruction Fidelity",
+			"Conservative closure + source-order assembly + reconstruction validator: recursively closes dependencies and whole tool pairs to a fixed point, resolves contradictions by retaining the later exact source resolution, assembles spans solely by source range, and rejects missing anchors / split pairs / digest mismatch / unresolved contradiction. Mandatory token estimate is content-only and handed unchanged to VC5A. OFF = mode C, no closure/validator, byte-identical predecessor (VC4B).",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC5A",
+			"VC5A PromptDagV1 + Budgeted Planner",
+			"Single-session DAG (PromptDagV1) + budgeted 0/1 portfolio planner: builds a stable Kahn-ordered DAG, computes the mandatory dependency/tool/anchor closure before optional selection, returns MANDATORY_CLOSURE_OVER_BUDGET with evidence preserved on overflow, and runs a utility-per-token portfolio that never exceeds the remaining budget. Framing is owned here, not in VC4C. OFF = byte-identical predecessor (VC4C).",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC5B",
+			"VC5B Validated Renderer + Provider Profiles",
+			"Validated prompt renderer: replays VC5A's stable Kahn order verbatim, preserves exact tool bytes (PREVENT-PI-002), places compacted context via the host before_agent_start prepend seam — never role:system (PREVENT-PI-003) — and SHA-256 hashes the entire canonical outbound request before provider invocation. Unknown provider/model cleanly bypasses to the predecessor prompt path. OFF = byte-identical predecessor (VC5A).",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_VC5C",
+			"VC5C Live Graduated Rollout",
+			"Live graduated rollout: deterministically hashes each session into a stable 10,000-bucket cohort and advances the exposure gate (1/5/25/50/100%) only after a 72h monotonic residency, a powered sample, >=10,000 events, and >=200 sessions — advancing ONE gate at a time. A hard causal/tool/anchor/exact failure freezes promotion and selects the pre-VC path. OFF = byte-identical predecessor (VC5B).",
+			true,
+		),
+	],
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.6",
+  "version": "0.20.7",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/dedup.ts

@@ -75,6 +75,14 @@ export interface DedupConfigShape {
    *  topical structure before detailed checkpoint hits. */
   RAPTOR_INJECT_SUMMARIES: boolean;
   // Monitoring / alerting.
+  /**
+   * Durable dedup audit trail (external-audit item #2). ON appends one
+   * structured JSON line per tier DECISION to events.log so an operator can see
+   * which layer collapsed a region, onto what, and at what similarity — the
+   * inputs needed to tune the thresholds above. OFF writes nothing at all;
+   * dedup behavior is identical either way (pure instrumentation).
+   */
+  DEDUP_AUDIT: boolean;
   FP_RATE_L0: number; // FP alert threshold for exact tier
   FP_RATE_L1L2: number; // FP alert threshold for fuzzy tiers
   ALERT_WINDOW_MS: number;
@@ -115,6 +123,7 @@ export function loadDedupConfig(): DedupConfigShape {
     RAPTOR_MAX_LEAF_EXPANSION: envNum("MEGACOMPACT_RAPTOR_MAX_LEAF_EXP", 10),
     RAPTOR_FRESHNESS_HOURS: envNum("MEGACOMPACT_RAPTOR_FRESHNESS_HOURS", 4),
     RAPTOR_INJECT_SUMMARIES: envBool("MEGACOMPACT_RAPTOR_INJECT_SUMMARIES", true),
+    DEDUP_AUDIT: envBool("MEGACOMPACT_DEDUP_AUDIT", true),
     FP_RATE_L0: envNum("MEGACOMPACT_FP_RATE_L0", 0.01),
     FP_RATE_L1L2: envNum("MEGACOMPACT_FP_RATE_L1L2", 0.05),
     ALERT_WINDOW_MS: envNum("MEGACOMPACT_ALERT_WINDOW_MS", 600_000),

package/src/dedup-audit.test-helpers.ts

@@ -0,0 +1,49 @@
+/**
+ * dedup-audit.test-helpers.ts — shared harness for the dedup audit trail tests.
+ *
+ * Extracted so dedup-audit.test.ts and dedup-audit-gating.test.ts share one
+ * temp-dir lifecycle and one events.log reader instead of duplicating them, and
+ * so each test file stays under the 300-line src/ soft limit.
+ *
+ * Hermetic: every store gets its own temp state dir, so no test can observe
+ * another's audit lines. No network (PREVENT-PI-004).
+ */
+
+import { mkdtempSync, readFileSync, writeFileSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { loadDedupConfig, type DedupConfigShape } from "./config/dedup.js";
+import { defaultEventsPath, type DedupAuditEvent } from "./monitoring.js";
+
+/** Root temp dir shared by both audit test files (each run gets a subdir). */
+export const baseTmp = mkdtempSync(join(tmpdir(), "mc-dedup-audit-"));
+
+/** The live config with per-test overrides applied. */
+export function cfg(over: Partial<DedupConfigShape> = {}): DedupConfigShape {
+	return { ...loadDedupConfig(), ...over };
+}
+
+let seq = 0;
+
+/** A never-before-used state dir under `baseTmp`. */
+export function freshDir(): string {
+	return join(baseTmp, `run-${seq++}-${Math.floor(performance.now() * 1000)}`);
+}
+
+/** Read every dedup_audit line from a store's default events.log. */
+export function auditLines(stateDir: string): DedupAuditEvent[] {
+	const path = defaultEventsPath(stateDir);
+	if (!existsSync(path)) return [];
+	return readFileSync(path, "utf-8")
+		.split("\n")
+		.filter((l) => l.trim())
+		.map((l) => JSON.parse(l) as DedupAuditEvent)
+		.filter((e) => e.type === "dedup_audit");
+}
+
+/** A path whose parent directory can never be created: it sits under a FILE. */
+export function unwritablePath(): string {
+	const blocker = join(baseTmp, `blocker-${seq++}`);
+	writeFileSync(blocker, "not a directory");
+	return join(blocker, "sub", "events.log");
+}

package/src/monitoring.ts

@@ -204,3 +204,13 @@ export function logRecallQuality(path: string, ev: RecallQualityEvent): void {
     /* best-effort — never break the extension */
   }
 }
+
+// ---------------------------------------------------------------------------
+// Dedup audit trail (external-audit item #2)
+// ---------------------------------------------------------------------------
+// The event shape and its append helper are DEFINED in vectorStore/dedup-audit.ts,
+// co-located with the only recorder that emits them (this file already carries
+// decision events + the metrics snapshot + FP alerting and must stay under its
+// 300-line soft limit). Re-exported here so callers that treat monitoring.ts as
+// the events.log barrel — including the dashboard SSE tail — keep one import.
+export { logDedupAudit, type DedupAuditEvent } from "./vectorStore/dedup-audit.js";

package/src/vectorStore/add-l0.ts

@@ -0,0 +1,152 @@
+/**
+ * add-l0.ts — the L0 exact-match tier of the dedup cascade.
+ *
+ * L0 is three exact-hash probes sharing one enable flag and one MARK_ONLY flag:
+ *   1. contentHash — normalized dual-hash, bloom-accelerated (Sprint 9/10)
+ *   2. regionHash  — the legacy pre-normalization hash (backward-compat)
+ *   3. summaryHash — same-topic incremental compactions
+ *
+ * Split out of add.ts so both files stay under the 300-line src/ soft limit.
+ * The tier is a pure decision: it either produces a collapse (returning the
+ * matched result) or reports the fall-through, and it never writes a new
+ * checkpoint — that stays in add.ts.
+ *
+ * All three probes are hash comparisons, so none of them computes a similarity
+ * score and none is invented for the audit trail.
+ */
+import { createHash } from "node:crypto";
+import type { StoredCheckpoint } from "../store.js";
+import type { ContentDigest } from "../dedup/digest.js";
+import { upsertCheckpoint, addTokensSaved, bumpDedupStats } from "../store/sqlite.js";
+import type { BloomFilter } from "../store/bloom.js";
+import type { DedupAuditRecorder } from "./dedup-audit.js";
+import type { AddInput, AddResult } from "./types.js";
+import type { VectorStore } from "./class.js";
+
+/** Everything the L0 tier needs from the enclosing add() cascade. */
+export interface L0Context {
+	store: VectorStore;
+	input: AddInput;
+	/** Checkpoints already stored for this session. */
+	all: StoredCheckpoint[];
+	/** Normalized content digest of the incoming region. */
+	digest: ContentDigest;
+	/** Legacy region hash of the incoming region. */
+	regionHash: string;
+	/** SHA-256 of the topic summary, when one was supplied. */
+	summaryHash?: string;
+	/** Tokens the original region occupied (the dedup "saved" base). */
+	origTokens: number;
+	/** Warm bloom accelerator for the content-hash probe. */
+	bloom: BloomFilter;
+	/** Cascade start time, for the monitoring latency figure. */
+	t0: number;
+	audit: DedupAuditRecorder;
+}
+
+/**
+ * Outcome of the L0 tier.
+ *
+ * `result` set → the tier collapsed the region and add() returns immediately.
+ * `markOnly` set → a probe matched but MARK_ONLY policy said store anyway, so
+ * the cascade continues and records the mark at the end.
+ */
+export interface L0Outcome {
+	result?: AddResult;
+	markOnly: "L0" | null;
+}
+
+/** Run the L0 exact-match tier. */
+export function runL0Tier(ctx: L0Context): L0Outcome {
+	const { store, input, all, digest, regionHash, origTokens, bloom, t0, audit } = ctx;
+	const cfg = store.cfg;
+	const onTier = input.onTier;
+	let markOnly: "L0" | null = null;
+
+	// 0. Content-hash dedup (Sprint 9) — catches identical content arriving under
+	//    different regionText. Normalization handles case/whitespace/ANSI so
+	//    variants collapse to one row. Dual-hash guards a single-hash collision.
+	//    Sprint 10: bloom is the accelerator — a miss means "definitely new" and
+	//    skips the scan; a hit is only a candidate, confirmed against `all` here.
+	//    MARK_ONLY_L0 records the decision but does not collapse.
+	onTier?.({ tier: "L0", status: "scanning" });
+	if (cfg.L0_ENABLED && bloom.maybeHas(digest.contentHash)) {
+		const contentMatch = all.find(
+			(cp) =>
+				cp.contentHash === digest.contentHash &&
+				cp.contentHash2 === digest.contentHash2,
+		);
+		if (contentMatch) {
+			if (cfg.MARK_ONLY_L0) {
+				markOnly = "L0"; // Record-but-don't-collapse: fall through.
+			} else {
+				contentMatch.timestamp = input.timestamp;
+				upsertCheckpoint(contentMatch, store.stateDir);
+				bumpDedupStats(true, store.stateDir);
+				// Deduped: whole original region discarded, nothing new stored.
+				addTokensSaved(origTokens, store.stateDir);
+				store.record("L0", "deduped", "contentHash", Date.now() - t0, 1, contentMatch.checkpointId);
+				audit.deduped("L0", contentMatch.checkpointId, "contentHash");
+				onTier?.({ tier: "L0", status: "deduped", detail: "contentHash" });
+				return {
+					result: { checkpoint: contentMatch, deduped: true, reason: "contentHash" },
+					markOnly,
+				};
+			}
+		}
+	}
+
+	// 1. Legacy regionHash dedup (backward-compat) — part of L0 tier gating.
+	if (cfg.L0_ENABLED) {
+		const regionMatch = all.find((cp) => cp.regionHash === regionHash);
+		if (regionMatch) {
+			if (cfg.MARK_ONLY_L0) {
+				markOnly = "L0"; // fall through
+			} else {
+				bumpDedupStats(true, store.stateDir);
+				// Deduped: whole original region discarded, nothing new stored.
+				addTokensSaved(origTokens, store.stateDir);
+				store.record("L0", "deduped", "regionHash", Date.now() - t0, 1, regionMatch.checkpointId);
+				audit.deduped("L0", regionMatch.checkpointId, "regionHash");
+				onTier?.({ tier: "L0", status: "deduped", detail: "regionHash" });
+				return {
+					result: { checkpoint: regionMatch, deduped: true, reason: "regionHash" },
+					markOnly,
+				};
+			}
+		}
+	}
+
+	// 2. SummaryHash dedup — catches same-topic incremental compactions.
+	if (ctx.summaryHash && cfg.L0_ENABLED) {
+		const summaryMatch = all.find((cp) => cp.summaryHash === ctx.summaryHash);
+		if (summaryMatch) {
+			if (cfg.MARK_ONLY_L0) {
+				markOnly = "L0"; // fall through
+			} else {
+				summaryMatch.timestamp = input.timestamp;
+				upsertCheckpoint(summaryMatch, store.stateDir);
+				bumpDedupStats(true, store.stateDir);
+				// Deduped: whole original region discarded, nothing new stored.
+				addTokensSaved(origTokens, store.stateDir);
+				store.record("L0", "deduped", "summaryHash", Date.now() - t0, 1, summaryMatch.checkpointId);
+				audit.deduped("L0", summaryMatch.checkpointId, "summaryHash");
+				onTier?.({ tier: "L0", status: "deduped", detail: "summaryHash" });
+				return {
+					result: { checkpoint: summaryMatch, deduped: true, reason: "summaryHash" },
+					markOnly,
+				};
+			}
+		}
+	}
+	// L0 did not collapse this region.
+	onTier?.({ tier: "L0", status: "passed" });
+	return { markOnly };
+}
+
+/** SHA-256 of the topic summary (full 64-hex; 16-hex was collision-prone). */
+export function computeSummaryHash(topicSummary?: string): string | undefined {
+	return topicSummary
+		? createHash("sha256").update(topicSummary).digest("hex")
+		: undefined;
+}

package/src/vectorStore/add-l1.ts

@@ -0,0 +1,57 @@
+/**
+ * add-l1.ts — the L1 MinHash/LSH near-duplicate lookup.
+ *
+ * Split out of add.ts to keep both files under the 300-line src/ soft limit.
+ * L1 catches the one-word edits and rewordings L0's exact hashes miss: a cheap
+ * LSH bucket retrieval narrows the field, then a trigram verification
+ * (pg_trgm-equivalent) is the final gate. The verify is boolean, so the tier
+ * produces no similarity score.
+ */
+import type { StoredCheckpoint } from "../store.js";
+import {
+	minhashSignature,
+	SIGNATURE_VERSION,
+	NUM_HASHES,
+} from "../dedup/l1-minhash.js";
+import { lshBands } from "../dedup/l1-lsh.js";
+import { isNearDuplicate } from "../dedup/l1-verify.js";
+import { lshCandidateChunks } from "../store/sqlite.js";
+import type { VectorStore } from "./class.js";
+
+export /**
+ * L1 near-duplicate lookup: MinHash → LSH candidate retrieval → trigram verify.
+ * Returns the matching checkpoint or undefined. Bounded by a 100-candidate cap
+ * and a 20ms verify budget (QA #7/#15) so it never hangs a large session.
+ */
+function findL1Duplicate(
+	store: VectorStore,
+	sessionId: string,
+	regionText: string,
+	all: StoredCheckpoint[],
+): StoredCheckpoint | undefined {
+	if (all.length === 0) return undefined;
+	const sig = minhashSignature(regionText);
+	if (sig.length !== NUM_HASHES) return undefined;
+	const bands = lshBands(sig, sessionId, SIGNATURE_VERSION);
+	// Cheap candidate retrieval (single query, capped). Exclude nothing yet —
+	// the new checkpoint has no id, so pass a sentinel that never matches.
+	const candidateIds = lshCandidateChunks(
+		bands,
+		sessionId,
+		"__new__",
+		store.stateDir,
+		100,
+	);
+	if (candidateIds.length === 0) return undefined;
+	const byId = new Map(all.map((cp) => [cp.checkpointId, cp]));
+	const VERIFY_BUDGET_MS = 20;
+	const start = Date.now();
+	for (const id of candidateIds) {
+		if (Date.now() - start > VERIFY_BUDGET_MS) break; // QA #15: abort → "not dup"
+		const cand = byId.get(id);
+		if (!cand) continue;
+		const candText = cand.normalizedText ?? cand.summary ?? "";
+		if (isNearDuplicate(regionText, candText)) return cand;
+	}
+	return undefined;
+}