pi-mega-compact 0.20.6 → 0.20.7

package/src/vectorStore/dedup-audit.ts

@@ -0,0 +1,199 @@
+/**
+ * dedup-audit.ts — durable audit trail for dedup tier decisions
+ * (external-audit item #2).
+ *
+ * Before this module a tier decision existed only as the in-process `onTier`
+ * callback that paints the live UI; nothing survived the process, so an
+ * operator could not answer "which layer collapsed this region, onto what, at
+ * what similarity?" — the inputs needed to tune the thresholds in
+ * config/dedup.ts. Here each decision is appended to the repo's events.log as
+ * one structured JSON line (see `DedupAuditEvent` below).
+ *
+ * The event type and its append helper live HERE rather than in monitoring.ts:
+ * monitoring.ts already owns three concerns (decision events, the dashboard.json
+ * metrics snapshot, FP alerting) and sits close to its 300-line soft limit, so
+ * co-locating the shape with the only recorder that produces it keeps both files
+ * under the headroom gate. monitoring.ts re-exports both for callers (and the
+ * dashboard SSE tail) that treat it as the events.log barrel.
+ *
+ * Design constraints:
+ *  - PURE INSTRUMENTATION. Nothing in this file may influence a dedup outcome.
+ *  - Best-effort/non-fatal: `logDedupAudit` swallows IO errors, and the emitter
+ *    itself is wrapped so a malformed field can never break add().
+ *  - Honest fields only: a value is emitted only where the caller actually
+ *    computed it. L0/L1 are hash/verify tiers and pass no `similarity`.
+ *  - Signal, not chatter: callers emit on DECISIONS (a match, a scored
+ *    candidate, the final outcome), never on every "scanning" transition.
+ *  - Flag-gated by cfg.DEDUP_AUDIT (default ON; OFF writes nothing at all).
+ *
+ * PREVENT-PI-004: local filesystem append only, no network.
+ */
+
+import { appendFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { defaultEventsPath } from "../monitoring.js";
+
+/**
+ * One dedup decision, as persisted to events.log.
+ *
+ * `ts` is an ISO-8601 string (not the epoch millis DedupDecisionEvent uses):
+ * the dashboard SSE tail only forwards lines carrying a `type` discriminator,
+ * and its consumers parse the timestamp as a date. Optional fields are omitted
+ * rather than nulled — a missing `similarity` means the tier computed no score,
+ * which is itself the honest signal (L0/L1 are hash/verify tiers).
+ */
+export interface DedupAuditEvent {
+	/** SSE discriminator — the dashboard streams only typed lines. */
+	type: "dedup_audit";
+	/** ISO-8601 decision timestamp. */
+	ts: string;
+	/** Normalized session the region belongs to. */
+	sessionId: string;
+	/** Which layer produced the decision ("new" = nothing collapsed). */
+	tier: "L0" | "L1" | "L2" | "new";
+	/** What the layer decided. */
+	status: "deduped" | "passed" | "stored";
+	/** Checkpoint the region collapsed onto, or the nearest one scored. */
+	matchedEntry?: string;
+	/** Checkpoint created, when the outcome was a new write. */
+	storedEntry?: string;
+	/** Cosine similarity — present only where a tier actually scored. */
+	similarity?: number;
+	/** Why the decision went the way it did (e.g. "contentHash", "mark_only"). */
+	dedupReason?: string;
+	/** Tokens the original region occupied before compaction. */
+	originalTokenEstimate?: number;
+	/** Tokens the stored summary occupies. */
+	tokenEstimate?: number;
+}
+
+/**
+ * Append one audit event to events.log (best-effort, never throws).
+ *
+ * Same append-one-JSON-line contract as monitoring.ts's logDecision — an
+ * unwritable path is swallowed so instrumentation can never break add().
+ */
+export function logDedupAudit(path: string, ev: DedupAuditEvent): void {
+	try {
+		mkdirSync(dirname(path), { recursive: true });
+		appendFileSync(path, `${JSON.stringify(ev)}\n`, "utf-8");
+	} catch {
+		/* best-effort — never break the extension on a log failure */
+	}
+}
+
+/** The slice of VectorStore the audit emitter reads. */
+export interface DedupAuditContext {
+	/** Per-repo state dir — resolves the events.log the dashboard tails. */
+	readonly stateDir: string;
+	/** Explicit events.log override (tests / the Sprint-14 monitoring opt-in). */
+	readonly eventsPath?: string;
+	/** Whether the audit trail is enabled (cfg.DEDUP_AUDIT). */
+	readonly auditEnabled: boolean;
+}
+
+/** The decision payload a call site supplies; ts/type are filled in here. */
+export type DedupAuditInput = Omit<DedupAuditEvent, "type" | "ts">;
+
+/**
+ * The per-add() facts every decision in one cascade shares: which session, and
+ * the token accounting already computed by the caller.
+ */
+export interface DedupAuditScope {
+	sessionId: string;
+	/** Tokens the original region occupied (VectorStore.add's `origTokens`). */
+	originalTokenEstimate?: number;
+	/** Tokens the stored summary occupies (AddInput.tokenEstimate). */
+	tokenEstimate?: number;
+}
+
+/**
+ * A decision recorder bound to one add() cascade.
+ *
+ * `deduped` / `passed` / `stored` are the three shapes the cascade actually
+ * produces; binding them here keeps VectorStore.add's call sites to one line
+ * each (delegate-shell pattern) and keeps the field-honesty rules — which tier
+ * may carry a similarity, matched vs. created id — in a single place.
+ */
+export interface DedupAuditRecorder {
+	/** A tier collapsed the region onto `matchedEntry`. */
+	deduped(
+		tier: "L0" | "L1" | "L2",
+		matchedEntry: string,
+		dedupReason: string,
+		similarity?: number,
+	): void;
+	/** A tier scored a candidate but did not collapse (threshold near-miss). */
+	passed(
+		tier: "L0" | "L1" | "L2",
+		matchedEntry: string,
+		similarity: number,
+	): void;
+	/** Final outcome: nothing collapsed, a new checkpoint was written. */
+	stored(storedEntry: string, dedupReason: string, tokenEstimate: number): void;
+}
+
+/** Build a recorder bound to one add() cascade. */
+export function dedupAuditRecorder(
+	ctx: DedupAuditContext,
+	scope: DedupAuditScope,
+): DedupAuditRecorder {
+	const base = {
+		sessionId: scope.sessionId,
+		originalTokenEstimate: scope.originalTokenEstimate,
+		tokenEstimate: scope.tokenEstimate,
+	};
+	return {
+		deduped: (tier, matchedEntry, dedupReason, similarity) =>
+			emitDedupAudit(ctx, {
+				...base,
+				tier,
+				status: "deduped",
+				matchedEntry,
+				dedupReason,
+				...(similarity === undefined ? {} : { similarity }),
+			}),
+		passed: (tier, matchedEntry, similarity) =>
+			emitDedupAudit(ctx, {
+				...base,
+				tier,
+				status: "passed",
+				matchedEntry,
+				similarity,
+			}),
+		stored: (storedEntry, dedupReason, tokenEstimate) =>
+			emitDedupAudit(ctx, {
+				...base,
+				tier: "new",
+				status: "stored",
+				storedEntry,
+				dedupReason,
+				tokenEstimate,
+			}),
+	};
+}
+
+/**
+ * Append one dedup decision to events.log.
+ *
+ * Resolves the target path from the explicit `eventsPath` when a caller opted
+ * in (Sprint 14 monitoring / tests), otherwise from the store's own per-repo
+ * state dir — production never passes `eventsPath`, so defaulting is what makes
+ * the audit trail actually exist on a real device.
+ */
+export function emitDedupAudit(
+	ctx: DedupAuditContext,
+	input: DedupAuditInput,
+): void {
+	if (!ctx.auditEnabled) return;
+	try {
+		const path = ctx.eventsPath ?? defaultEventsPath(ctx.stateDir);
+		logDedupAudit(path, {
+			type: "dedup_audit",
+			ts: new Date().toISOString(),
+			...input,
+		});
+	} catch {
+		/* instrumentation must never break the add() path */
+	}
+}

package/src/vectorStore.ts

@@ -8,6 +8,7 @@
  * "./vectorStore.js" are unchanged.
  */
 export { VectorStore } from "./vectorStore/class.js";
+export { addCheckpoint } from "./vectorStore/add.js";
 export { computeRegionHash } from "./vectorStore/hash.js";
 export {
 	L2_ENABLED,