pi-mega-compact 0.20.41 → 0.20.42

package/extensions/dashboard-server/routes-dedup-attribution.ts

@@ -0,0 +1,166 @@
+/**
+ * dashboard-server/routes-dedup-attribution.ts — DEDUP-ATTR tier attribution route.
+ *
+ * GET /api/dedup-tier-attribution?windowMs=<n> — reader-only aggregate answering
+ * "L0/L1/L2/new percent of dedup decisions in window W". Reads the repo's local
+ * events.log (per-repo stateDir, bounded 8 MiB tail), parses `dedup_audit` lines
+ * (PREVENT-001 guarded), and sends the pure rollup with a shared-derived status.
+ *
+ * Flag-off (MEGACOMPACT_DEDUP_ATTR=0) returns 404 and writes NO durable cache
+ * file — byte-identical predecessor. Reader-only: emits tier counts + shares,
+ * never matched checkpoint paths/text, never raw user query (EVAL-REDACT-002).
+ *
+ * Memoization mirrors routes-vector-cortex-health.ts: an in-memory facts cache
+ * keyed on {events.log mtime+size, windowMs} is served when ≤5s old, so repeated
+ * dashboard polls stay cheap while any on-disk log mutation still invalidates.
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read + loopback response only),
+ * PREVENT-001 (JSON.parse guarded), PREVENT-011 (no `any`).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { readFileSync, statSync, writeFileSync, mkdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+
+import type { RouteContext } from "./routes-core.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import { DEDUP_ATTR_ENABLED } from "../../src/config.js";
+import { computeDedupTierRollup } from "../../src/vector-cortex/dedup-attr/rollup.js";
+import type { DedupTierAttributionResponse } from "./api-contracts/dedup-attribution.js";
+import type { DedupAuditEvent } from "../../src/vectorStore/dedup-audit.js";
+import { deriveVcStatus } from "./vc-status.js";
+
+const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000; // 24h
+const MAX_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+const MAX_TAIL_BYTES = 8 * 1024 * 1024; // 8 MiB
+const MEMO_TTL_MS = 5000; // ≤5s per stateDir
+
+interface MemoEntry {
+  key: string;
+  at: number;
+  body: DedupTierAttributionResponse;
+}
+
+let memo: MemoEntry | null = null;
+
+/** Parse one events.log line into a DedupAuditEvent, or null (PREVENT-001). */
+function parseAuditLine(line: string): DedupAuditEvent | null {
+  if (!line.includes('"dedup_audit"')) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+  const obj = parsed as Record<string, unknown>;
+  if (obj.type !== "dedup_audit") return null;
+  if (typeof obj.ts !== "string") return null;
+  const tier = obj.tier;
+  const status = obj.status;
+  if (tier !== "L0" && tier !== "L1" && tier !== "L2" && tier !== "new") return null;
+  if (status !== "deduped" && status !== "passed" && status !== "stored") return null;
+  // sessionId is never read by the rollup; a parsed line may omit richer fields.
+  return { type: "dedup_audit", ts: obj.ts, tier, status, sessionId: "" };
+}
+
+/** Read the bounded tail of events.log into parsed dedup_audit events. */
+function readAuditEvents(eventsPath: string): DedupAuditEvent[] {
+  let buf: Buffer;
+  try {
+    buf = readFileSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log read (loopback dashboard)
+  } catch {
+    return [];
+  }
+  const tail = buf.length > MAX_TAIL_BYTES ? buf.subarray(buf.length - MAX_TAIL_BYTES) : buf;
+  const text = tail.toString("utf-8");
+  const lines = text.split("\n");
+  // Drop a leading fragment when we truncated mid-line (first line may be partial).
+  const start = buf.length > MAX_TAIL_BYTES ? 1 : 0;
+  const out: DedupAuditEvent[] = [];
+  for (let i = start; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+    const ev = parseAuditLine(line);
+    if (ev) out.push(ev);
+  }
+  return out;
+}
+
+/** Resolve windowMs from the ?windowMs= query param (default 24h, capped 30d). */
+function resolveWindowMs(url: string): number {
+  if (!url.includes("?")) return DEFAULT_WINDOW_MS;
+  const params = new URLSearchParams(url.slice(url.indexOf("?")));
+  const raw = params.get("windowMs");
+  if (raw == null) return DEFAULT_WINDOW_MS;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n <= 0) return DEFAULT_WINDOW_MS;
+  return Math.min(n, MAX_WINDOW_MS);
+}
+
+/** Best-effort durable snapshot write (never breaks the response). */
+function writeDurableSnapshot(stateDir: string, rollup: DedupTierAttributionResponse): void {
+  try {
+    mkdirSync(dirname(join(stateDir, "dedup-tier-attribution.json")), { recursive: true });
+    writeFileSync(
+      join(stateDir, "dedup-tier-attribution.json"),
+      `${JSON.stringify(rollup)}\n`,
+      "utf-8",
+    );
+  } catch {
+    /* best-effort — instrumentation must never break the read path */
+  }
+}
+
+export function handleDedupTierAttribution(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?", 1)[0] ?? url;
+  if (path !== "/api/dedup-tier-attribution") return false;
+  if (req.method !== "GET") {
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+  if (!DEDUP_ATTR_ENABLED()) {
+    // Flag-off: 404 + no durable cache write — byte-identical predecessor.
+    sendJson(res, 404, { error: "not_found" });
+    return true;
+  }
+
+  const eventsPath = ctx.eventsPath;
+  const windowMs = resolveWindowMs(url);
+  const now = new Date();
+
+  // Memoized-facts pattern (mirrors routes-vector-cortex-health.ts): serve a
+  // still-fresh entry keyed on {events.log mtime+size, windowMs}.
+  let keyBase: { mtimeMs: number; size: number } | null = null;
+  try {
+    const s = statSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log stat (loopback)
+    keyBase = { mtimeMs: s.mtimeMs, size: s.size };
+  } catch {
+    keyBase = { mtimeMs: 0, size: 0 };
+  }
+  const key = `${keyBase.mtimeMs}:${keyBase.size}:${windowMs}`;
+  if (memo !== null && memo.key === key && now.getTime() - memo.at <= MEMO_TTL_MS) {
+    sendJson(res, 200, memo.body);
+    return true;
+  }
+
+  const auditEvents = readAuditEvents(eventsPath);
+  const rollup = computeDedupTierRollup(auditEvents, windowMs, now);
+  const status = deriveVcStatus({
+    enabled: true,
+    hasData: auditEvents.length > 0,
+    structuralOnly: false,
+  });
+  const body: DedupTierAttributionResponse = { ...rollup, status };
+
+  memo = { key, at: now.getTime(), body };
+  writeDurableSnapshot(ctx.stateDir, body);
+
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts

@@ -246,5 +246,11 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
 			"ML5-E nightly retraining + feedback loop: the user's own post-redaction conversation turns become fresh training signal, the five heads are re-fit nightly via system cron, calibration is re-validated, and mode-A promotion is re-checked without human intervention. The cron is system-configured (crontab -e); the extension never installs or writes a crontab. Candidates are written to ~/.pi/mega-compact-encoder/candidates/ and promoted via the ML5-D Improve Cortex flow. ON (default) = the scripts may be invoked by the system cron. OFF = scripts are never invoked; byte-identical to ML5-D.",
 			true,
 		),
+		boolDirect(
+			"MEGACOMPACT_DEDUP_ATTR",
+			"Dedup Tier Attribution Rollup",
+			"Dedup tier-attribution rollup: per-tier dedup catch shares (L0/L1/L2/new percent of dedup decisions) read from the local events.log dedup_audit stream (GET /api/dedup-tier-attribution). OFF = 404 + no cache file, byte-identical predecessor.",
+			true,
+		),
 	],
 };

package/extensions/dashboard-server/routes.ts

@@ -66,3 +66,4 @@ export {
 	handleImproveCortex,
 	handleImproveCortexStatus,
 } from "./routes-cortex-improve.js";
+export { handleDedupTierAttribution } from "./routes-dedup-attribution.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.41",
+  "version": "0.20.42",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/vector-cortex-dedup-attr.ts

@@ -0,0 +1,16 @@
+/**
+ * config/vector-cortex-dedup-attr.ts — DEDUP-ATTR tier-attribution flag.
+ *
+ * The single positive sprint flag for the dedup tier-attribution rollup
+ * (external-audit item #6). Default ON; `MEGACOMPACT_DEDUP_ATTR=0` / `=false` /
+ * `_DISABLED=true` disables and is byte-identical to the predecessor: the
+ * `/api/dedup-tier-attribution` endpoint 404s and no rollup cache file is
+ * written.
+ *
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { sprintFlag } from "./vector-cortex-flag.js";
+
+/** Positive sprint flag: `=0` or `_DISABLED=true` disables (default ON). */
+export const DEDUP_ATTR_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_DEDUP_ATTR");

package/src/config/vector-cortex.ts

@@ -270,7 +270,6 @@ export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
 
 // VC8C extracted to vector-cortex-vc8c.ts; re-exported so existing imports keep resolving.
 export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
-// Sibling extracts: stay under the 300-line soft limit.
 export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
 export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
 export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
@@ -281,6 +280,7 @@ export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
 export { ML5C_ENABLED } from "./vector-cortex-ml5c.js";
 export { ML5D_ENABLED } from "./vector-cortex-ml5d.js";
 export { ML5E_ENABLED } from "./vector-cortex-ml5e.js";
+export { DEDUP_ATTR_ENABLED } from "./vector-cortex-dedup-attr.js";
 
 // Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
 export {

package/src/config.ts

@@ -188,6 +188,7 @@ export {
   ML5C_ENABLED,
   ML5D_ENABLED,
   ML5E_ENABLED,
+  DEDUP_ATTR_ENABLED,
   BREAKER_WINDOW_MS,
   BREAKER_MIN_ATTEMPTS,
   BREAKER_PERF_FAILURES,

package/src/vector-cortex/dedup-attr/rollup.ts

@@ -0,0 +1,136 @@
+/**
+ * vector-cortex/dedup-attr/rollup.ts — DEDUP-ATTR pure tier-attribution rollup.
+ *
+ * Pure function: reads an array of parsed dedup-audit events, buckets them by
+ * tier over a query window, and returns the per-tier catch shares. No I/O, no
+ * clock — the route passes `now: Date` (new Date()) at request time, so this
+ * module is deterministically reproducible and trivially testable (DEDUP-ATTR-004).
+ *
+ * Missing `similarity` is fine here: L0/L1 are hash/verify tiers and emit no
+ * score, and the rollup never consults it. Malformed lines (non-object, wrong
+ * type) are skipped silently — the route's events.log reader already guards its
+ * JSON.parse (PREVENT-001), and this layer defends against a hostile/mixed
+ * `dedup_audit` array as well.
+ *
+ * The returned shape is structurally identical to the authoritative contract in
+ * `extensions/dashboard-server/api-contracts/dedup-attribution.ts` (the existing
+ * src-mirrors-dashboard-contract pattern); src/ stays pi-agnostic and never
+ * imports dashboard types. Status here is computed from the pure signal the
+ * rollup owns (a non-empty window) — the route additionally derives the sent
+ * `status` via `deriveVcStatus` from the shared vc-status module.
+ *
+ * Guardrails: PREVENT-PI-004 (zero I/O/network), PREVENT-011 (no `any`).
+ */
+
+import type { DedupAuditEvent } from "../../vectorStore/dedup-audit.js";
+
+/** Per-tier dedup/passed counts (same shape as the api-contract DedupTierCounts). */
+export interface DedupTierRollupCounts {
+  /** Decisions this tier collapsed onto a matched checkpoint. */
+  readonly deduped: number;
+  /** Decisions this tier scored a candidate but did not collapse. */
+  readonly passed: number;
+}
+
+/** The status the rollup can produce from its pure signal. */
+export type RollupStatus = "live" | "awaiting_data";
+
+/** Purely-computed tier attribution for one window (mirrors DedupTierRollupV1). */
+export interface DedupTierRollup {
+  readonly schema: "dedup-tier-rollup-v1";
+  readonly windowStart: string;
+  readonly windowEnd: string;
+  readonly totalDecisions: number;
+  readonly byTier: {
+    readonly l0: DedupTierRollupCounts;
+    readonly l1: DedupTierRollupCounts;
+    readonly l2: DedupTierRollupCounts;
+    readonly new: number;
+  };
+  readonly l0Share: number;
+  readonly l1Share: number;
+  readonly l2Share: number;
+  readonly status: RollupStatus;
+}
+
+interface Accumulator {
+  l0: { deduped: number; passed: number };
+  l1: { deduped: number; passed: number };
+  l2: { deduped: number; passed: number };
+  storedNew: number;
+}
+
+/**
+ * Roll up dedup-audit events over [now - windowMs, now] into per-tier shares.
+ *
+ * Only events whose ISO-8601 `ts` falls inside the window are counted; events of
+ * any other `type` (or a missing/unparseable `ts`) are skipped silently. Shares
+ * are fractions of the window's total decisions, each in [0, 1]; an empty window
+ * yields all-zero shares with `status:"awaiting_data"` — never a fabricated
+ * zero-share table presented as real.
+ */
+export function computeDedupTierRollup(
+  events: readonly DedupAuditEvent[],
+  windowMs: number,
+  now: Date,
+): DedupTierRollup {
+  const winMs = windowMs > 0 ? windowMs : 24 * 60 * 60 * 1000;
+  const windowStart = now.getTime() - winMs;
+  const windowEnd = now.getTime();
+
+  const acc: Accumulator = {
+    l0: { deduped: 0, passed: 0 },
+    l1: { deduped: 0, passed: 0 },
+    l2: { deduped: 0, passed: 0 },
+    storedNew: 0,
+  };
+  let total = 0;
+
+  for (const ev of events) {
+    if (!ev || typeof ev !== "object") continue;
+    if (ev.type !== "dedup_audit") continue;
+    const ts = Date.parse(ev.ts);
+    if (Number.isNaN(ts) || ts < windowStart || ts > windowEnd) continue;
+    total += 1;
+    switch (ev.tier) {
+      case "L0":
+        if (ev.status === "deduped") acc.l0.deduped += 1;
+        else if (ev.status === "passed") acc.l0.passed += 1;
+        break;
+      case "L1":
+        if (ev.status === "deduped") acc.l1.deduped += 1;
+        else if (ev.status === "passed") acc.l1.passed += 1;
+        break;
+      case "L2":
+        if (ev.status === "deduped") acc.l2.deduped += 1;
+        else if (ev.status === "passed") acc.l2.passed += 1;
+        break;
+      case "new":
+        acc.storedNew += 1;
+        break;
+      default:
+        break;
+    }
+  }
+
+  const byTier = {
+    l0: { deduped: acc.l0.deduped, passed: acc.l0.passed },
+    l1: { deduped: acc.l1.deduped, passed: acc.l1.passed },
+    l2: { deduped: acc.l2.deduped, passed: acc.l2.passed },
+    new: acc.storedNew,
+  };
+
+  const shareOf = (n: number): number => (total > 0 ? n / total : 0);
+
+  return {
+    schema: "dedup-tier-rollup-v1",
+    windowStart: new Date(windowStart).toISOString(),
+    windowEnd: now.toISOString(),
+    totalDecisions: total,
+    byTier,
+    l0Share: shareOf(acc.l0.deduped + acc.l0.passed),
+    l1Share: shareOf(acc.l1.deduped + acc.l1.passed),
+    l2Share: shareOf(acc.l2.deduped + acc.l2.passed),
+    status: total > 0 ? "live" : "awaiting_data",
+  };
+}