pi-mega-compact 0.19.3 → 0.20.1

package/extensions/dashboard-server/routes-cache.test/_helpers.ts

@@ -34,6 +34,7 @@ export async function withServer<T>(
 	fn: (port: number) => Promise<T>,
 ): Promise<T> {
 	process.env.MEGACOMPACT_DASHBOARD_PORT = port;
+	process.env.MEGACOMPACT_DASHBOARD_HOST = "127.0.0.1";
 	const child = spawn(process.execPath, [SERVER_ENTRY, dir], { stdio: "ignore" });
 	try {
 		await waitFor(async () => {
@@ -50,6 +51,7 @@ export async function withServer<T>(
 	} finally {
 		child.kill("SIGTERM");
 		delete process.env.MEGACOMPACT_DASHBOARD_PORT;
+		delete process.env.MEGACOMPACT_DASHBOARD_HOST;
 		rmSync(dir, { recursive: true, force: true });
 	}
 }

package/extensions/dashboard-server/routes-rag-settings-helpers.ts

@@ -349,6 +349,18 @@ export const SETTINGS: ReadonlyArray<{
 				"TopologyQueryV1/RouterKeyV2 structured keys (length-delimited, unsigned-byte order, no prefix ambiguity), exact (session,generation) invalidation, stale-generation rejection (TOP_GENERATION_STALE), and the M6 router-generation-v2 copy/validate/switch migration. OFF = mode C, no structured router key / generation invalidation, byte-identical predecessor.",
 				true,
 			),
+			boolDirect(
+				"MEGACOMPACT_VC4A",
+				"VC4A Dual-Tier Shards",
+				"SemanticShardV1/ExactShardV1/ShardManifestV1: partition a session ONLY at complete EventV2 boundaries; exact shards preserve every tool call/result pair, anchor and invalid UTF-8 event as original bytes (pairs never split across exact shards); manifest enforces disjoint sorted ranges + complete protected-span coverage. OFF = mode C, exact anchors/current transcript only, byte-identical predecessor.",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_VC4B",
+				"VC4B Residual Basis Parity",
+				"Residual codec: orthonormal DCT-II basis + int16 block quantization + block-scoped exact correction stream + (9,6) Reed-Solomon parity shards with SHA-256 corruption detection; admission gates on encodedSize <= 95% of exact-compressed size. OFF = mode C, no residual artifact produced, byte-identical predecessor.",
+				true,
+			),
 		],
 	},
 	{

package/extensions/dashboard-server/routes-vector-cortex-helpers.ts

@@ -0,0 +1,110 @@
+/**
+ * routes-vector-cortex-helpers.ts — shared spawn-and-fetch harness for the
+ * /api/vector-cortex/* route test files.
+ *
+ * The dashboard-server.js main block fires when the entry argv includes
+ * "dashboard-server", so tests under dist/extensions/dashboard-server/ MUST
+ * spawn the server as a child process rather than import launchDashboardServer.
+ * Extracted from routes-vector-cortex.test.ts so per-sprint route test files
+ * can share the harness without each re-implementing it.
+ *
+ * Reader-only: no payload bytes are ever returned by the routes under test.
+ */
+import { createHash } from "node:crypto";
+import { rmSync, readFileSync, existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+/** Locate the repo root (the directory holding `conformance/vector-cortex`). */
+const HERE = dirname(fileURLToPath(import.meta.url));
+export function repoRoot(from: string = HERE): string {
+  let dir = from;
+  for (let i = 0; i < 8; i++) {
+    if (existsSync(join(dir, "conformance", "vector-cortex"))) return dir;
+    const next = dirname(dir);
+    if (next === dir) break;
+    dir = next;
+  }
+  throw new Error("conformance corpus not found above " + from);
+}
+
+const REPO_ROOT = repoRoot();
+
+/** SHA-256 of the REAL committed ModelManifestV1 — the health card's digest. */
+export function realManifestDigest(): string {
+  // guardrails-allow PREVENT-PI-004: local committed asset filesystem read (loopback)
+  return createHash("sha256")
+    .update(readFileSync(join(REPO_ROOT, "assets", "vector-cortex", "encoder-v1", "manifest.json")))
+    .digest("hex");
+}
+
+export const SERVER_ENTRY = new URL("./server.js", import.meta.url).pathname;
+
+export function waitFor(
+  cond: () => boolean | Promise<boolean>,
+  timeoutMs = 6000,
+): Promise<void> {
+  const start = Date.now();
+  return new Promise((resolve, reject) => {
+    const tick = async () => {
+      if (await cond()) return resolve();
+      if (Date.now() - start > timeoutMs) return reject(new Error("timeout"));
+      setTimeout(tick, 50);
+    };
+    tick();
+  });
+}
+
+export async function withServer<T>(
+  port: string,
+  dir: string,
+  fn: (port: number) => Promise<T>,
+): Promise<T> {
+  process.env.MEGACOMPACT_DASHBOARD_PORT = port;
+  // Pin loopback so the spawn-and-fetch harness stays hermetic even on a
+  // machine with tailscale0 (the production auto-bind would otherwise grab the
+  // tailnet IP and the localhost probe would connection-refuse). Tests that
+  // need a different host can set MEGACOMPACT_DASHBOARD_HOST before calling.
+  process.env.MEGACOMPACT_DASHBOARD_HOST = "127.0.0.1";
+  const child = spawn(process.execPath, [SERVER_ENTRY, dir], {
+    stdio: "ignore",
+    env: {
+      ...process.env,
+      MEGACOMPACT_INDEX_DIR: dir,
+    },
+  });
+  try {
+    await waitFor(async () => {
+      try {
+        const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+        const res = await fetch(`http://localhost:${raw.port}/api/version`);
+        return res.ok;
+      } catch {
+        return false;
+      }
+    });
+    const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+    return await fn(raw.port);
+  } finally {
+    child.kill("SIGTERM");
+    delete process.env.MEGACOMPACT_DASHBOARD_PORT;
+    delete process.env.MEGACOMPACT_DASHBOARD_HOST;
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** Seed redacted metric rows directly on disk so the server's reader sees them. */
+export async function seedEval(dir: string): Promise<void> {
+  const { appendEvalRow } = await import(
+    "../../src/vector-cortex/eval/persist.js"
+  );
+  // Buckets (inclusive): 1,5,10,25,50,100,250. 1ms→cell0, 12ms→cell3
+  // (<=25), 250ms→cell6, 300ms→overflow.
+  appendEvalRow(dir, [
+    { session: "s1", seq: 1, event: "encode", value: 1, unit: "ms", mode: "A" },
+    { session: "s1", seq: 2, event: "encode", value: 250, unit: "ms", mode: "A" },
+    { session: "s1", seq: 3, event: "encode", value: 300, unit: "ms", mode: "A" },
+    { session: "s1", seq: 4, event: "encode", value: 12, unit: "ms", mode: "A" },
+  ]);
+}

package/extensions/dashboard-server/routes-vector-cortex-residual.ts

@@ -0,0 +1,60 @@
+/**
+ * dashboard-server/routes-vector-cortex-residual.ts — VC4B residual basis parity
+ * dashboard route.
+ *
+ * Reader-only GET /api/vector-cortex/residual returning a COUNT/BYTE aggregate
+ * (encode attempts, admitted/rejected counts, recovery failures, encoded/exact
+ * byte totals) — never residual payloads, correction streams, shard bytes, or
+ * original source bytes. Flag-gated on MEGACOMPACT_VC4B: `enabled:false` when off
+ * (byte-identical to the pre-VC4B predecessor).
+ *
+ * The VC4B residual codec is PURE IN-MEMORY logic in this sprint (it has no
+ * durable metrics store yet), so the aggregate reports the current status
+ * truthfully: `enabled` reflects the flag, and the count/byte fields are zero
+ * until a future sprint persists a metrics store. This route is the seam that
+ * sprint will populate. Non-fatal: a missing state dir degrades to
+ * `enabled:false`.
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read only / in-process state),
+ * PREVENT-011 (no `any`), reader-only aggregate (counts/bytes only).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteContext } from "./routes-core.js";
+import { VC4B_ENABLED } from "../../src/config.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import type { VectorCortexResidualView } from "./api-contracts/vector-cortex.js";
+
+/**
+ * Reader-only GET /api/vector-cortex/residual (VC4B).
+ */
+export function handleVectorCortexResidual(
+  req: IncomingMessage,
+  res: ServerResponse,
+  _ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?")[0] ?? url;
+  if (path !== "/api/vector-cortex/residual") return false;
+  if (req.method !== "GET") {
+    // Reader-only path: cannot be "off" without a GET; it is genuinely read-only.
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+
+  const enabled = VC4B_ENABLED();
+  const body: VectorCortexResidualView = {
+    enabled,
+    // No durable residual metrics store is staged yet in this sprint (pure
+    // in-memory encode), so the aggregates are truthfully zero.
+    encodeAttempts: 0,
+    admittedCount: 0,
+    rejectedCount: 0,
+    recoveryFailures: 0,
+    encodedByteTotal: 0,
+    exactByteTotal: 0,
+    updatedAt: new Date().toISOString(),
+  };
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes-vector-cortex-shards.ts

@@ -0,0 +1,57 @@
+/**
+ * dashboard-server/routes-vector-cortex-shards.ts — VC4A dual-tier shard
+ * dashboard route.
+ *
+ * Reader-only GET /api/vector-cortex/shards returning a COUNT/BYTE aggregate
+ * (semantic shards, exact shards, protected byte total) — never shard payloads,
+ * verbatim exact bytes, or prompt text. Flag-gated on MEGACOMPACT_VC4A:
+ * `enabled:false` when off (byte-identical to the pre-VC4A predecessor).
+ *
+ * The VC4A shard partition is PURE IN-MEMORY logic in this sprint (it has no
+ * durable manifest store yet), so the aggregate reports the current status
+ * truthfully: `enabled` reflects the flag, and the count/byte fields are zero
+ * until a future sprint persists a staged manifest. This route is the seam that
+ * sprint will populate. Non-fatal: a missing state dir degrades to
+ * `enabled:false`.
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read only / in-process state),
+ * PREVENT-011 (no `any`), reader-only aggregate (counts/bytes only).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteContext } from "./routes-core.js";
+import { VC4A_ENABLED } from "../../src/config.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import type { VectorCortexShardsView } from "./api-contracts/vector-cortex.js";
+
+/**
+ * Reader-only GET /api/vector-cortex/shards (VC4A).
+ */
+export function handleVectorCortexShards(
+  req: IncomingMessage,
+  res: ServerResponse,
+  _ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?")[0] ?? url;
+  if (path !== "/api/vector-cortex/shards") return false;
+  if (req.method !== "GET") {
+    // Reader-only path: cannot be "off" without a GET; it is genuinely read-only.
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+
+  const enabled = VC4A_ENABLED();
+  const body: VectorCortexShardsView = {
+    enabled,
+    // No durable shard manifest is staged yet in this sprint (pure in-memory
+    // partition), so the aggregates are truthfully zero.
+    semanticCount: 0,
+    exactCount: 0,
+    byteTotal: 0,
+    protectedByteTotal: 0,
+    updatedAt: new Date().toISOString(),
+  };
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes-vector-cortex.ts

@@ -19,3 +19,5 @@ export { handleVectorCortexHealth, handleVectorCortexBreakersReset } from "./rou
 export { handleVectorCortexLedger } from "./routes-vector-cortex-ledger.js";
 export { handleVectorCortexTopology } from "./routes-vector-cortex-topology.js";
 export { handleVectorCortexQuery } from "./routes-vector-cortex-query.js";
+export { handleVectorCortexShards } from "./routes-vector-cortex-shards.js";
+export { handleVectorCortexResidual } from "./routes-vector-cortex-residual.js";

package/extensions/dashboard-server/routes.ts

@@ -47,4 +47,6 @@ export {
 	handleVectorCortexLedger,
 	handleVectorCortexTopology,
 	handleVectorCortexQuery,
+	handleVectorCortexShards,
+	handleVectorCortexResidual,
 } from "./routes-vector-cortex.js";

package/extensions/dashboard-server/server.ts

@@ -25,6 +25,7 @@ import { createRequire } from "node:module";
 
 import { NEW_UI } from "../../src/config.js";
 import { log, setLogPath, setDashboardServerVersion } from "./state.js";
+import { detectTailscaleIP } from "./tailscale.js";
 import {
 	buildRouteContext,
 	handleIndex,
@@ -62,6 +63,8 @@ import {
 	handleVectorCortexLedger,
 	handleVectorCortexTopology,
 	handleVectorCortexQuery,
+	handleVectorCortexShards,
+	handleVectorCortexResidual,
 	handleStatic,
 } from "./routes.js";
 
@@ -231,15 +234,31 @@ export async function launchDashboardServer(
 		detectCrossRepoDrift,
 	});
 
+	// Bind host resolution: explicit MEGACOMPACT_DASHBOARD_HOST override wins;
+	// otherwise auto-bind to the Tailscale interface (tailnet access); fall back
+	// to loopback when Tailscale is absent. 0.0.0.0 means "all interfaces" and
+	// opts into permissive CORS below.
+	const host =
+		process.env.MEGACOMPACT_DASHBOARD_HOST ??
+		detectTailscaleIP() ??
+		"127.0.0.1";
+	const allInterfaces = host === "0.0.0.0";
+
+	// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard server (tailnet-local via Tailscale mesh or loopback); no remote outbound calls.
 	const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-		// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard localhost server (loopback-only) — CORS restricted to same-origin localhost browsers.
-		// CORS for local access — restricted to loopback origins (the dashboard server only binds to localhost).
+		// CORS for local access — restricted to loopback/same-origin tailnet
+		// origins (the dashboard server binds to localhost or the tailscale IP).
+		// With MEGACOMPACT_DASHBOARD_HOST=0.0.0.0 the user opted into
+		// all-interfaces binding, so we allow any origin.
 		const origin = req.headers.origin;
-		if (
-			typeof origin === "string" &&
-			/^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/.test(origin)
-		) {
-			res.setHeader("Access-Control-Allow-Origin", origin);
+		const originAllowed =
+			allInterfaces ||
+			(typeof origin === "string" &&
+				new RegExp(
+					`^http://(localhost|127\\.0\\.0\\.1|::1|${host.replace(/\./g, "\\.")})(:\\d+)?$`,
+				).test(origin));
+		if (originAllowed) {
+			res.setHeader("Access-Control-Allow-Origin", origin ?? "*");
 			res.setHeader("Vary", "Origin");
 		}
 		res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, OPTIONS");
@@ -287,6 +306,8 @@ export async function launchDashboardServer(
 		if (handleVectorCortexLedger(req, res, ctx)) return;
 		if (handleVectorCortexTopology(req, res, ctx)) return;
 		if (handleVectorCortexQuery(req, res, ctx)) return;
+		if (handleVectorCortexShards(req, res, ctx)) return;
+		if (handleVectorCortexResidual(req, res, ctx)) return;
 		handleStatic(req, res, ctx);
 	});
 
@@ -310,20 +331,28 @@ export async function launchDashboardServer(
 				}
 			});
 
-			server.listen(port, "127.0.0.1", () => {
-				const url = `http://localhost:${port}`; // guardrails-allow PREVENT-PI-004: localhost dashboard URL (loopback-only)
-				log("server running", { url });
+			server.listen(port, host, () => {
+				// When bound to the loopback, keep the legacy localhost URL so the
+				// user's browser (which resolves localhost → ::1 or 127.0.0.1) works.
+				// When bound to a tailnet/tailscale IP, surface that address instead.
+				const url =
+					host === "127.0.0.1"
+						? `http://localhost:${port}`
+						: `http://${host}:${port}`; // guardrails-allow PREVENT-PI-004: dashboard URL (tailnet-local via Tailscale mesh or loopback)
+				log("server running", { url, host });
 				// eslint-disable-next-line no-console
 				console.log(`[mega-compact] dashboard server running: ${url}`);
 
-				// v0.8.2: also bind the IPv6 loopback (::1). On many systems `localhost`
-				// resolves to ::1 first (see /etc/hosts), so an IPv4-only bind makes the
-				// browser hit ::1:port and get connection refused. PREVENT-PI-004
-				// (loopback-only) means BOTH 127.0.0.1 and ::1. Non-fatal: IPv4-only
-				// hosts or a ::1 already in use just skip the mirror.
+				// v0.8.2: also bind the IPv6 loopback (::1) when bound to the IPv4
+				// loopback. On many systems `localhost` resolves to ::1 first (see
+				// /etc/hosts), so an IPv4-only bind makes the browser hit ::1:port and
+				// get connection refused. PREVENT-PI-004 (loopback-only) means BOTH
+				// 127.0.0.1 and ::1. When bound to a tailscale/0.0.0.0 host, we do NOT
+				// bind ::1 — the tailnet address is what the user reaches. Non-fatal:
+				// IPv4-only hosts or a ::1 already in use just skip the mirror.
 				let v6: ReturnType<typeof createServer> | undefined;
 				const v4Handler = server.listeners("request")[0];
-				if (v4Handler) {
+				if (host === "127.0.0.1" && v4Handler) {
 					v6 = createServer((r, s) =>
 						(v4Handler as (a: IncomingMessage, b: ServerResponse) => void).call(
 							server,
@@ -338,7 +367,7 @@ export async function launchDashboardServer(
 							message: e.message,
 						}),
 					);
-					v6.listen(port, "::1", () => log("ipv6 loopback bound", { port })); // guardrails-allow PREVENT-PI-004: IPv6 loopback (::1) mirror of the localhost dashboard server
+					v6.listen(port, "::1", () => log("ipv6 loopback bound", { port })); // guardrails-allow PREVENT-PI-004: IPv6 loopback (::1) mirror of the dashboard server
 				}
 
 				// Write port.pid

package/extensions/dashboard-server/tailscale.ts

@@ -1,3 +1,5 @@
+import { networkInterfaces } from "node:os";
+
 export function setupTailscaleServe(port = 3000, httpsPort = 443): boolean {
 	// Tailscale serve: exposes localhost dashboard securely.
 	// Only activates when TAILSCALE_ENABLED=1 is set.
@@ -5,3 +7,33 @@ export function setupTailscaleServe(port = 3000, httpsPort = 443): boolean {
 	console.log(`[tailscale] Serve enabled on port ${port} (https:${httpsPort})`);
 	return true;
 }
+
+// CGNAT block used by Tailscale for client addresses (100.64.0.0/10).
+const TAILSCALE_CGNAT_PREFIX = "100.";
+
+/**
+ * Detect the IPv4 address Tailscale has assigned to this host, so the dashboard
+ * server can auto-bind to the tailnet interface and be reachable from other
+ * devices on the tailnet. Returns null when Tailscale is not present.
+ *
+ * The default Tailscale interface name is `tailscale0` on Linux and `Tailscale`
+ * on macOS/Windows. We prefer an address inside Tailscale's CGNAT range
+ * (100.64.0.0/10) when one exists; otherwise we return the first IPv4 address
+ * found on the interface.
+ */
+export function detectTailscaleIP(): string | null {
+	const ifaces = networkInterfaces();
+	const candidates: Array<string> = [];
+	for (const [name, addrs] of Object.entries(ifaces)) {
+		if (name !== "tailscale0" && name !== "Tailscale") continue;
+		if (!addrs) continue;
+		for (const addr of addrs) {
+			if (addr.family === "IPv4") {
+				candidates.push(addr.address);
+			}
+		}
+	}
+	if (candidates.length === 0) return null;
+	const cgnat = candidates.find((ip) => ip.startsWith(TAILSCALE_CGNAT_PREFIX));
+	return cgnat ?? candidates[0];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.19.3",
+  "version": "0.20.1",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/vector-cortex.ts

@@ -134,6 +134,32 @@ export const VC3B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC3B");
  */
 export const VC3C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC3C");
 
+/**
+ * VC4A — dual-tier shard contract (SemanticShardV1 / ExactShardV1 /
+ * ShardManifestV1).
+ * Default ON. `MEGACOMPACT_VC4A=0` disables and is byte-identical to the
+ * predecessor (mode C: exact anchors/current transcript only, no semantic or
+ * exact shard manifest is assembled or emitted, zero
+ * `vector_cortex_shard_manifest_built` / `vector_cortex_protected_span_rejected`
+ * emissions; the predecessor derived pointer and goldens are unchanged). The
+ * real consumers are the shard assembly seam (manifest.ts) and the reader-only
+ * dashboard shards aggregate view.
+ */
+export const VC4A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC4A");
+
+/**
+ * VC4B — residual codec and numeric parity (ResidualCodecV1 / ParityShardV1).
+ * Default ON. `MEGACOMPACT_VC4B=0` disables and is byte-identical to the
+ * predecessor (mode C: no residual artifact is admitted, the exact compressed
+ * payload / ledger bytes remain the only representation, zero
+ * `vector_cortex_residual_admitted` / `vector_cortex_parity_recovery_failed`
+ * emissions; the VC4A shard manifest and its goldens are unchanged). The real
+ * consumers are the codec admission seam (codec.ts) and the reader-only
+ * dashboard residual aggregate view. The codec math itself is PURE — flag OFF
+ * gates the reporter/admission seam, never the arithmetic.
+ */
+export const VC4B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC4B");
+
 // ---------------------------------------------------------------------------
 // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
 // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.

package/src/config.ts

@@ -163,6 +163,8 @@ export {
   VC3A_ENABLED,
   VC3B_ENABLED,
   VC3C_ENABLED,
+  VC4A_ENABLED,
+  VC4B_ENABLED,
   BREAKER_WINDOW_MS,
   BREAKER_MIN_ATTEMPTS,
   BREAKER_PERF_FAILURES,