pi-mega-compact 0.20.41 → 0.20.43

package/extensions/dashboard-server/routes-dedup-attribution.ts

@@ -0,0 +1,166 @@
+/**
+ * dashboard-server/routes-dedup-attribution.ts — DEDUP-ATTR tier attribution route.
+ *
+ * GET /api/dedup-tier-attribution?windowMs=<n> — reader-only aggregate answering
+ * "L0/L1/L2/new percent of dedup decisions in window W". Reads the repo's local
+ * events.log (per-repo stateDir, bounded 8 MiB tail), parses `dedup_audit` lines
+ * (PREVENT-001 guarded), and sends the pure rollup with a shared-derived status.
+ *
+ * Flag-off (MEGACOMPACT_DEDUP_ATTR=0) returns 404 and writes NO durable cache
+ * file — byte-identical predecessor. Reader-only: emits tier counts + shares,
+ * never matched checkpoint paths/text, never raw user query (EVAL-REDACT-002).
+ *
+ * Memoization mirrors routes-vector-cortex-health.ts: an in-memory facts cache
+ * keyed on {events.log mtime+size, windowMs} is served when ≤5s old, so repeated
+ * dashboard polls stay cheap while any on-disk log mutation still invalidates.
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read + loopback response only),
+ * PREVENT-001 (JSON.parse guarded), PREVENT-011 (no `any`).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { readFileSync, statSync, writeFileSync, mkdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+
+import type { RouteContext } from "./routes-core.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import { DEDUP_ATTR_ENABLED } from "../../src/config.js";
+import { computeDedupTierRollup } from "../../src/vector-cortex/dedup-attr/rollup.js";
+import type { DedupTierAttributionResponse } from "./api-contracts/dedup-attribution.js";
+import type { DedupAuditEvent } from "../../src/vectorStore/dedup-audit.js";
+import { deriveVcStatus } from "./vc-status.js";
+
+const DEFAULT_WINDOW_MS = 24 * 60 * 60 * 1000; // 24h
+const MAX_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+const MAX_TAIL_BYTES = 8 * 1024 * 1024; // 8 MiB
+const MEMO_TTL_MS = 5000; // ≤5s per stateDir
+
+interface MemoEntry {
+  key: string;
+  at: number;
+  body: DedupTierAttributionResponse;
+}
+
+let memo: MemoEntry | null = null;
+
+/** Parse one events.log line into a DedupAuditEvent, or null (PREVENT-001). */
+function parseAuditLine(line: string): DedupAuditEvent | null {
+  if (!line.includes('"dedup_audit"')) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+  const obj = parsed as Record<string, unknown>;
+  if (obj.type !== "dedup_audit") return null;
+  if (typeof obj.ts !== "string") return null;
+  const tier = obj.tier;
+  const status = obj.status;
+  if (tier !== "L0" && tier !== "L1" && tier !== "L2" && tier !== "new") return null;
+  if (status !== "deduped" && status !== "passed" && status !== "stored") return null;
+  // sessionId is never read by the rollup; a parsed line may omit richer fields.
+  return { type: "dedup_audit", ts: obj.ts, tier, status, sessionId: "" };
+}
+
+/** Read the bounded tail of events.log into parsed dedup_audit events. */
+function readAuditEvents(eventsPath: string): DedupAuditEvent[] {
+  let buf: Buffer;
+  try {
+    buf = readFileSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log read (loopback dashboard)
+  } catch {
+    return [];
+  }
+  const tail = buf.length > MAX_TAIL_BYTES ? buf.subarray(buf.length - MAX_TAIL_BYTES) : buf;
+  const text = tail.toString("utf-8");
+  const lines = text.split("\n");
+  // Drop a leading fragment when we truncated mid-line (first line may be partial).
+  const start = buf.length > MAX_TAIL_BYTES ? 1 : 0;
+  const out: DedupAuditEvent[] = [];
+  for (let i = start; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+    const ev = parseAuditLine(line);
+    if (ev) out.push(ev);
+  }
+  return out;
+}
+
+/** Resolve windowMs from the ?windowMs= query param (default 24h, capped 30d). */
+function resolveWindowMs(url: string): number {
+  if (!url.includes("?")) return DEFAULT_WINDOW_MS;
+  const params = new URLSearchParams(url.slice(url.indexOf("?")));
+  const raw = params.get("windowMs");
+  if (raw == null) return DEFAULT_WINDOW_MS;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n <= 0) return DEFAULT_WINDOW_MS;
+  return Math.min(n, MAX_WINDOW_MS);
+}
+
+/** Best-effort durable snapshot write (never breaks the response). */
+function writeDurableSnapshot(stateDir: string, rollup: DedupTierAttributionResponse): void {
+  try {
+    mkdirSync(dirname(join(stateDir, "dedup-tier-attribution.json")), { recursive: true });
+    writeFileSync(
+      join(stateDir, "dedup-tier-attribution.json"),
+      `${JSON.stringify(rollup)}\n`,
+      "utf-8",
+    );
+  } catch {
+    /* best-effort — instrumentation must never break the read path */
+  }
+}
+
+export function handleDedupTierAttribution(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?", 1)[0] ?? url;
+  if (path !== "/api/dedup-tier-attribution") return false;
+  if (req.method !== "GET") {
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+  if (!DEDUP_ATTR_ENABLED()) {
+    // Flag-off: 404 + no durable cache write — byte-identical predecessor.
+    sendJson(res, 404, { error: "not_found" });
+    return true;
+  }
+
+  const eventsPath = ctx.eventsPath;
+  const windowMs = resolveWindowMs(url);
+  const now = new Date();
+
+  // Memoized-facts pattern (mirrors routes-vector-cortex-health.ts): serve a
+  // still-fresh entry keyed on {events.log mtime+size, windowMs}.
+  let keyBase: { mtimeMs: number; size: number } | null = null;
+  try {
+    const s = statSync(eventsPath); // guardrails-allow PREVENT-PI-004: local events.log stat (loopback)
+    keyBase = { mtimeMs: s.mtimeMs, size: s.size };
+  } catch {
+    keyBase = { mtimeMs: 0, size: 0 };
+  }
+  const key = `${keyBase.mtimeMs}:${keyBase.size}:${windowMs}`;
+  if (memo !== null && memo.key === key && now.getTime() - memo.at <= MEMO_TTL_MS) {
+    sendJson(res, 200, memo.body);
+    return true;
+  }
+
+  const auditEvents = readAuditEvents(eventsPath);
+  const rollup = computeDedupTierRollup(auditEvents, windowMs, now);
+  const status = deriveVcStatus({
+    enabled: true,
+    hasData: auditEvents.length > 0,
+    structuralOnly: false,
+  });
+  const body: DedupTierAttributionResponse = { ...rollup, status };
+
+  memo = { key, at: now.getTime(), body };
+  writeDurableSnapshot(ctx.stateDir, body);
+
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts

@@ -246,5 +246,17 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
 			"ML5-E nightly retraining + feedback loop: the user's own post-redaction conversation turns become fresh training signal, the five heads are re-fit nightly via system cron, calibration is re-validated, and mode-A promotion is re-checked without human intervention. The cron is system-configured (crontab -e); the extension never installs or writes a crontab. Candidates are written to ~/.pi/mega-compact-encoder/candidates/ and promoted via the ML5-D Improve Cortex flow. ON (default) = the scripts may be invoked by the system cron. OFF = scripts are never invoked; byte-identical to ML5-D.",
 			true,
 		),
+		boolDirect(
+			"MEGACOMPACT_DEDUP_ATTR",
+			"Dedup Tier Attribution Rollup",
+			"Dedup tier-attribution rollup: per-tier dedup catch shares (L0/L1/L2/new percent of dedup decisions) read from the local events.log dedup_audit stream (GET /api/dedup-tier-attribution). OFF = 404 + no cache file, byte-identical predecessor.",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_ENC_0A",
+			"ENC-0a Encoder Backend Decision",
+			"ENC-0a learned-encoder backend-decision lock: records the transformers.js/WASM vs onnxruntime-node choice, per-platform install matrix, opset-21 baseline and pinned digests in docs/vector-cortex/encoder-backend-decision.md. OFF = no decision record written / no resolver runs, mode B trigram byte-identical predecessor.",
+			true,
+		),
 	],
 };

package/extensions/dashboard-server/routes.ts

@@ -66,3 +66,4 @@ export {
 	handleImproveCortex,
 	handleImproveCortexStatus,
 } from "./routes-cortex-improve.js";
+export { handleDedupTierAttribution } from "./routes-dedup-attribution.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.41",
+  "version": "0.20.43",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/vector-cortex-dedup-attr.ts

@@ -0,0 +1,16 @@
+/**
+ * config/vector-cortex-dedup-attr.ts — DEDUP-ATTR tier-attribution flag.
+ *
+ * The single positive sprint flag for the dedup tier-attribution rollup
+ * (external-audit item #6). Default ON; `MEGACOMPACT_DEDUP_ATTR=0` / `=false` /
+ * `_DISABLED=true` disables and is byte-identical to the predecessor: the
+ * `/api/dedup-tier-attribution` endpoint 404s and no rollup cache file is
+ * written.
+ *
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { sprintFlag } from "./vector-cortex-flag.js";
+
+/** Positive sprint flag: `=0` or `_DISABLED=true` disables (default ON). */
+export const DEDUP_ATTR_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_DEDUP_ATTR");

package/src/config/vector-cortex-enc0a.ts

@@ -0,0 +1,34 @@
+/**
+ * config/vector-cortex-enc0a.ts — ENC-0a learned-encoder backend-decision flag.
+ *
+ * Extracted from vector-cortex.ts so that file stays under the 300-line soft
+ * limit (soft-as-hard gate), exactly as vector-cortex-vc9a.ts..vector-cortex-vc9d.ts
+ * were. This is the first ENC (real learned encoder) sprint flag. vector-cortex.ts
+ * re-exports the ENUM below and root src/config.ts re-exports it, so no consumer
+ * import path changes.
+ *
+ * ENC-0a locks the runtime-backend choice (transformers.js/WASM vs
+ * onnxruntime-node native), the per-platform install-size matrix, the opset
+ * baseline (re-baselined 17 -> 21) and the license/pinning audit. It writes a
+ * durable decision record and bench JSON but touches neither the store schema
+ * nor stateDir tables (pure migration).
+ *
+ * The split is purely mechanical: ENC_0A_ENABLED is byte-identical in name,
+ * semantics, and default to the definition it replaces, and vector-cortex.ts
+ * re-exports it so every existing `from "./config/vector-cortex.js"` import
+ * keeps resolving unchanged.
+ *
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { sprintFlag } from "./vector-cortex-flag.js";
+
+/**
+ * ENC-0a — learned-encoder backend-decision lock. Default ON.
+ * `MEGACOMPACT_ENC_0A=0` disables and is byte-identical to the predecessor
+ * (placeholder encoder): no decision record is written and no newer
+ * backend-resolution script runs — the runtime keeps serving mode B trigram
+ * exactly as before. This flag MUST also be a dashboard SETTINGS toggle (visible
+ * in config UI, never in EXCLUDED_SETTINGS), mirroring VC4A..VC9D.
+ */
+export const ENC_0A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_ENC_0A");

package/src/config/vector-cortex.ts

@@ -5,8 +5,7 @@
  * `=0`/`_DISABLED` off. Flag-OFF is byte-identical to the predecessor sprint's
  * behavior (for VC0A: mode C — observer absent, zero evaluation writes).
  *
- * The breaker/triad constants (TRIAD_RESILIENCE.md) live here so VC0C consumes
- * them without re-declaring the ownership boundary. Pi-agnostic, dependency-free.
+ * Breaker/triad constants (TRIAD_RESILIENCE.md) live here; pi-agnostic, dep-free.
  */
 
 import { sprintFlag } from "./vector-cortex-flag.js";
@@ -270,7 +269,6 @@ export const VC8B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC8B");
 
 // VC8C extracted to vector-cortex-vc8c.ts; re-exported so existing imports keep resolving.
 export { VC8C_ENABLED } from "./vector-cortex-vc8c.js";
-// Sibling extracts: stay under the 300-line soft limit.
 export { VC9A_ENABLED } from "./vector-cortex-vc9a.js";
 export { VC9B_ENABLED } from "./vector-cortex-vc9b.js";
 export { VC9C_ENABLED } from "./vector-cortex-vc9c.js";
@@ -281,6 +279,8 @@ export { ML5B_ENABLED } from "./vector-cortex-ml5b.js";
 export { ML5C_ENABLED } from "./vector-cortex-ml5c.js";
 export { ML5D_ENABLED } from "./vector-cortex-ml5d.js";
 export { ML5E_ENABLED } from "./vector-cortex-ml5e.js";
+export { DEDUP_ATTR_ENABLED } from "./vector-cortex-dedup-attr.js";
+export { ENC_0A_ENABLED } from "./vector-cortex-enc0a.js";
 
 // Breaker constants (TRIAD_RESILIENCE.md §breaker) extracted to vector-cortex-breakers.ts.
 export {

package/src/config.ts

@@ -188,6 +188,8 @@ export {
   ML5C_ENABLED,
   ML5D_ENABLED,
   ML5E_ENABLED,
+  DEDUP_ATTR_ENABLED,
+  ENC_0A_ENABLED,
   BREAKER_WINDOW_MS,
   BREAKER_MIN_ATTEMPTS,
   BREAKER_PERF_FAILURES,

package/src/vector-cortex/_acceptance-enc0a-contract.ts

@@ -0,0 +1,71 @@
+/** ENC-0a buildDecision contract constructor suite — extracted from
+ *  enc0a-acceptance.test.ts for soft-limit compliance. Receives the contract
+ *  constructor + constants from the aggregator (no import cycle).
+ */
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import type { EncoderPlatformRow } from "./encoder/decision.js";
+
+export interface Enc0aContractCtx {
+  buildDecision: typeof import("./encoder/decision.js").buildDecision;
+  PLATFORMS: readonly ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64"];
+}
+
+export function registerEnc0aContract(ctx: Enc0aContractCtx): void {
+  const { buildDecision, PLATFORMS } = ctx;
+
+  describe("buildDecision contract constructor", () => {
+    test("rejects an incomplete platform matrix", () => {
+      const matrix = {
+        "linux-x64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        "linux-arm64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        "darwin-x64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "wasm" },
+        "darwin-arm64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        // win32-x64 omitted
+      } as Record<(typeof PLATFORMS)[number], EncoderPlatformRow>;
+      assert.throws(() =>
+        buildDecision({
+          backend: "wasm",
+          budgetOk: true,
+          p95Ms: 18.2,
+          platformMatrix: matrix,
+          modelPath: "model.onnx",
+          modelBytes: 1,
+          modelSha256: "a".repeat(64),
+          tokenizerPath: "tokenizer.json",
+          tokenizerBytes: 1,
+          tokenizerSha256: "b".repeat(64),
+          blockedBy: [],
+        }),
+      );
+    });
+
+    test("builds a valid decision with opset 21 + MIT license for a complete matrix", () => {
+      const matrix = {
+        "linux-x64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        "linux-arm64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        "darwin-x64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "wasm" },
+        "darwin-arm64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+        "win32-x64": { runtime: "onnxruntime-web", installMiB: 33, demotion: "none" },
+      } as Record<(typeof PLATFORMS)[number], EncoderPlatformRow>;
+      const d = buildDecision({
+        backend: "wasm",
+        budgetOk: true,
+        p95Ms: 18.2,
+        platformMatrix: matrix,
+        modelPath: "model.onnx",
+        modelBytes: 24117248,
+        modelSha256: "a".repeat(64),
+        tokenizerPath: "tokenizer.json",
+        tokenizerBytes: 50000,
+        tokenizerSha256: "b".repeat(64),
+        blockedBy: [],
+      });
+      assert.equal(d.schema, "encoder-backend-decision-v1");
+      assert.equal(d.opset, 21);
+      assert.equal(d.backend, "wasm");
+      assert.deepEqual(d.license, { spdx: "MIT", redistribution: true });
+      assert.equal(d.platformMatrix["darwin-x64"].demotion, "wasm");
+    });
+  });
+}

package/src/vector-cortex/_acceptance-vc2a-conformance.ts

@@ -0,0 +1,119 @@
+/**
+ * _acceptance-vc2a-conformance.ts — ENC-001..008 conformance rows + the named
+ * assertions (ENC-ASSET-001 / ENC-DIGEST-002 / ENC-PLATFORM-003), each driven
+ * through the REAL encoder asset/runtime (no mocks). Extracted from
+ * vc2a-acceptance.test.ts so the aggregator stays under the tests/ soft limit.
+ *
+ * The aggregator passes its shared imports/helpers as a context object, so this
+ * sibling never imports the aggregator (no import cycle at module load).
+ */
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import type { EncoderLoadResult } from "./encoder/types.js";
+
+/** Context from the aggregator — only domain-specific helpers, not node:test/assert. */
+export interface Vc2aConformanceCtx {
+  ENC_IDS: readonly string[];
+  ENC_FAIL: typeof import("./encoder/types.js").ENC_FAIL;
+  fixture: (id: string) => {
+    id: string;
+    input: { scenario: string };
+    expected: { ok: boolean; code?: string; mode?: string };
+  };
+  buildDir: (scenario: string) => Built;
+  rmBuilt: (b: Built) => void;
+  optionsFor: (scenario: string) => import("./encoder/runtime.js").CreateEncoderRuntimeOptions;
+  createEncoderRuntime: typeof import("./encoder/runtime.js").createEncoderRuntime;
+  verifyEncoderAsset: typeof import("./encoder/asset.js").verifyEncoderAsset;
+  readEncoderManifest: typeof import("./encoder/asset.js").readEncoderManifest;
+}
+
+/* Structural stand-in for the aggregator's local `Built` (dir + scenario). */
+interface Built {
+  dir: string;
+  scenario: string;
+}
+
+export function registerVc2aConformance(ctx: Vc2aConformanceCtx): void {
+  const {
+    ENC_IDS,
+    ENC_FAIL,
+    fixture,
+    buildDir,
+    rmBuilt,
+    optionsFor,
+    createEncoderRuntime,
+    verifyEncoderAsset,
+    readEncoderManifest,
+  } = ctx;
+
+  // -------------------------------------------------------------------------
+  // ENC-001..008 conformance rows through the real runtime
+  // -------------------------------------------------------------------------
+  describe("ENC-001..008 conformance rows", () => {
+    for (const id of ENC_IDS) {
+      test(`${id}: resolves through the real runtime to the documented result`, () => {
+        const fx = fixture(id);
+        const built = buildDir(fx.input.scenario);
+        try {
+          const rt = createEncoderRuntime(optionsFor(fx.input.scenario));
+          const load: EncoderLoadResult = rt.load(built.dir);
+          assert.equal(load.ok, fx.expected.ok, `${id} ok`);
+          if (!load.ok) {
+            assert.equal(load.code, fx.expected.code, `${id} exact failure code`);
+            if (fx.expected.mode) assert.equal(load.mode, fx.expected.mode, `${id} mode`);
+          } else {
+            assert.equal(load.mode, "A", `${id} should be mode A`);
+          }
+        } finally {
+          rmBuilt(built);
+        }
+      });
+    }
+  });
+
+  // -------------------------------------------------------------------------
+  // VC2A named assertions
+  // -------------------------------------------------------------------------
+  describe("VC2A named assertions", () => {
+    test("ENC-ASSET-001: opset21 manifest + matching digests load as mode A", () => {
+      const fx = fixture("ENC-ASSET-001");
+      assert.equal(fx.expected.ok, true);
+      const built = buildDir("valid");
+      try {
+        const load = createEncoderRuntime().load(built.dir);
+        assert.equal(load.ok, true);
+        if (load.ok) assert.equal(load.mode, "A");
+      } finally {
+        rmBuilt(built);
+      }
+    });
+    test("ENC-DIGEST-002: one-byte model mutation demotes before load", () => {
+      const fx = fixture("ENC-DIGEST-002");
+      assert.equal(fx.expected.code, ENC_FAIL.DIGEST_MISMATCH);
+      const built = buildDir("mutate-onnx");
+      try {
+        const res = verifyEncoderAsset(built.dir, readEncoderManifest(built.dir));
+        assert.equal(res.ok, false);
+        if (!res.ok) assert.equal(res.code, ENC_FAIL.DIGEST_MISMATCH);
+      } finally {
+        rmBuilt(built);
+      }
+    });
+    test("ENC-PLATFORM-003: unsupported architecture selects trigram B", () => {
+      const fx = fixture("ENC-PLATFORM-003");
+      assert.equal(fx.expected.mode, "B");
+      const built = buildDir("valid");
+      try {
+        const load = createEncoderRuntime({ platform: () => null }).load(built.dir);
+        assert.equal(load.ok, false);
+        if (!load.ok) {
+          assert.equal(load.mode, "B", "trigram B selected");
+          assert.equal(load.code, ENC_FAIL.PLATFORM_UNSUPPORTED);
+        }
+      } finally {
+        rmBuilt(built);
+      }
+    });
+  });
+}