pi-mega-compact 0.19.3 → 0.20.0

package/extensions/dashboard-server/routes-vector-cortex-helpers.ts

@@ -0,0 +1,110 @@
+/**
+ * routes-vector-cortex-helpers.ts — shared spawn-and-fetch harness for the
+ * /api/vector-cortex/* route test files.
+ *
+ * The dashboard-server.js main block fires when the entry argv includes
+ * "dashboard-server", so tests under dist/extensions/dashboard-server/ MUST
+ * spawn the server as a child process rather than import launchDashboardServer.
+ * Extracted from routes-vector-cortex.test.ts so per-sprint route test files
+ * can share the harness without each re-implementing it.
+ *
+ * Reader-only: no payload bytes are ever returned by the routes under test.
+ */
+import { createHash } from "node:crypto";
+import { rmSync, readFileSync, existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+
+/** Locate the repo root (the directory holding `conformance/vector-cortex`). */
+const HERE = dirname(fileURLToPath(import.meta.url));
+export function repoRoot(from: string = HERE): string {
+  let dir = from;
+  for (let i = 0; i < 8; i++) {
+    if (existsSync(join(dir, "conformance", "vector-cortex"))) return dir;
+    const next = dirname(dir);
+    if (next === dir) break;
+    dir = next;
+  }
+  throw new Error("conformance corpus not found above " + from);
+}
+
+const REPO_ROOT = repoRoot();
+
+/** SHA-256 of the REAL committed ModelManifestV1 — the health card's digest. */
+export function realManifestDigest(): string {
+  // guardrails-allow PREVENT-PI-004: local committed asset filesystem read (loopback)
+  return createHash("sha256")
+    .update(readFileSync(join(REPO_ROOT, "assets", "vector-cortex", "encoder-v1", "manifest.json")))
+    .digest("hex");
+}
+
+export const SERVER_ENTRY = new URL("./server.js", import.meta.url).pathname;
+
+export function waitFor(
+  cond: () => boolean | Promise<boolean>,
+  timeoutMs = 6000,
+): Promise<void> {
+  const start = Date.now();
+  return new Promise((resolve, reject) => {
+    const tick = async () => {
+      if (await cond()) return resolve();
+      if (Date.now() - start > timeoutMs) return reject(new Error("timeout"));
+      setTimeout(tick, 50);
+    };
+    tick();
+  });
+}
+
+export async function withServer<T>(
+  port: string,
+  dir: string,
+  fn: (port: number) => Promise<T>,
+): Promise<T> {
+  process.env.MEGACOMPACT_DASHBOARD_PORT = port;
+  // Pin loopback so the spawn-and-fetch harness stays hermetic even on a
+  // machine with tailscale0 (the production auto-bind would otherwise grab the
+  // tailnet IP and the localhost probe would connection-refuse). Tests that
+  // need a different host can set MEGACOMPACT_DASHBOARD_HOST before calling.
+  process.env.MEGACOMPACT_DASHBOARD_HOST = "127.0.0.1";
+  const child = spawn(process.execPath, [SERVER_ENTRY, dir], {
+    stdio: "ignore",
+    env: {
+      ...process.env,
+      MEGACOMPACT_INDEX_DIR: dir,
+    },
+  });
+  try {
+    await waitFor(async () => {
+      try {
+        const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+        const res = await fetch(`http://localhost:${raw.port}/api/version`);
+        return res.ok;
+      } catch {
+        return false;
+      }
+    });
+    const raw = JSON.parse(readFileSync(join(dir, "port.pid"), "utf-8"));
+    return await fn(raw.port);
+  } finally {
+    child.kill("SIGTERM");
+    delete process.env.MEGACOMPACT_DASHBOARD_PORT;
+    delete process.env.MEGACOMPACT_DASHBOARD_HOST;
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+/** Seed redacted metric rows directly on disk so the server's reader sees them. */
+export async function seedEval(dir: string): Promise<void> {
+  const { appendEvalRow } = await import(
+    "../../src/vector-cortex/eval/persist.js"
+  );
+  // Buckets (inclusive): 1,5,10,25,50,100,250. 1ms→cell0, 12ms→cell3
+  // (<=25), 250ms→cell6, 300ms→overflow.
+  appendEvalRow(dir, [
+    { session: "s1", seq: 1, event: "encode", value: 1, unit: "ms", mode: "A" },
+    { session: "s1", seq: 2, event: "encode", value: 250, unit: "ms", mode: "A" },
+    { session: "s1", seq: 3, event: "encode", value: 300, unit: "ms", mode: "A" },
+    { session: "s1", seq: 4, event: "encode", value: 12, unit: "ms", mode: "A" },
+  ]);
+}

package/extensions/dashboard-server/routes-vector-cortex-shards.ts

@@ -0,0 +1,57 @@
+/**
+ * dashboard-server/routes-vector-cortex-shards.ts — VC4A dual-tier shard
+ * dashboard route.
+ *
+ * Reader-only GET /api/vector-cortex/shards returning a COUNT/BYTE aggregate
+ * (semantic shards, exact shards, protected byte total) — never shard payloads,
+ * verbatim exact bytes, or prompt text. Flag-gated on MEGACOMPACT_VC4A:
+ * `enabled:false` when off (byte-identical to the pre-VC4A predecessor).
+ *
+ * The VC4A shard partition is PURE IN-MEMORY logic in this sprint (it has no
+ * durable manifest store yet), so the aggregate reports the current status
+ * truthfully: `enabled` reflects the flag, and the count/byte fields are zero
+ * until a future sprint persists a staged manifest. This route is the seam that
+ * sprint will populate. Non-fatal: a missing state dir degrades to
+ * `enabled:false`.
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read only / in-process state),
+ * PREVENT-011 (no `any`), reader-only aggregate (counts/bytes only).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteContext } from "./routes-core.js";
+import { VC4A_ENABLED } from "../../src/config.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import type { VectorCortexShardsView } from "./api-contracts/vector-cortex.js";
+
+/**
+ * Reader-only GET /api/vector-cortex/shards (VC4A).
+ */
+export function handleVectorCortexShards(
+  req: IncomingMessage,
+  res: ServerResponse,
+  _ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?")[0] ?? url;
+  if (path !== "/api/vector-cortex/shards") return false;
+  if (req.method !== "GET") {
+    // Reader-only path: cannot be "off" without a GET; it is genuinely read-only.
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+
+  const enabled = VC4A_ENABLED();
+  const body: VectorCortexShardsView = {
+    enabled,
+    // No durable shard manifest is staged yet in this sprint (pure in-memory
+    // partition), so the aggregates are truthfully zero.
+    semanticCount: 0,
+    exactCount: 0,
+    byteTotal: 0,
+    protectedByteTotal: 0,
+    updatedAt: new Date().toISOString(),
+  };
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes-vector-cortex.ts

@@ -19,3 +19,4 @@ export { handleVectorCortexHealth, handleVectorCortexBreakersReset } from "./rou
 export { handleVectorCortexLedger } from "./routes-vector-cortex-ledger.js";
 export { handleVectorCortexTopology } from "./routes-vector-cortex-topology.js";
 export { handleVectorCortexQuery } from "./routes-vector-cortex-query.js";
+export { handleVectorCortexShards } from "./routes-vector-cortex-shards.js";

package/extensions/dashboard-server/routes.ts

@@ -47,4 +47,5 @@ export {
 	handleVectorCortexLedger,
 	handleVectorCortexTopology,
 	handleVectorCortexQuery,
+	handleVectorCortexShards,
 } from "./routes-vector-cortex.js";

package/extensions/dashboard-server/server.ts

@@ -25,6 +25,7 @@ import { createRequire } from "node:module";
 
 import { NEW_UI } from "../../src/config.js";
 import { log, setLogPath, setDashboardServerVersion } from "./state.js";
+import { detectTailscaleIP } from "./tailscale.js";
 import {
 	buildRouteContext,
 	handleIndex,
@@ -62,6 +63,7 @@ import {
 	handleVectorCortexLedger,
 	handleVectorCortexTopology,
 	handleVectorCortexQuery,
+	handleVectorCortexShards,
 	handleStatic,
 } from "./routes.js";
 
@@ -231,15 +233,31 @@ export async function launchDashboardServer(
 		detectCrossRepoDrift,
 	});
 
+	// Bind host resolution: explicit MEGACOMPACT_DASHBOARD_HOST override wins;
+	// otherwise auto-bind to the Tailscale interface (tailnet access); fall back
+	// to loopback when Tailscale is absent. 0.0.0.0 means "all interfaces" and
+	// opts into permissive CORS below.
+	const host =
+		process.env.MEGACOMPACT_DASHBOARD_HOST ??
+		detectTailscaleIP() ??
+		"127.0.0.1";
+	const allInterfaces = host === "0.0.0.0";
+
+	// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard server (tailnet-local via Tailscale mesh or loopback); no remote outbound calls.
 	const server = createServer((req: IncomingMessage, res: ServerResponse) => {
-		// guardrails-allow PREVENT-PI-004: optional, user-triggered /dashboard localhost server (loopback-only) — CORS restricted to same-origin localhost browsers.
-		// CORS for local access — restricted to loopback origins (the dashboard server only binds to localhost).
+		// CORS for local access — restricted to loopback/same-origin tailnet
+		// origins (the dashboard server binds to localhost or the tailscale IP).
+		// With MEGACOMPACT_DASHBOARD_HOST=0.0.0.0 the user opted into
+		// all-interfaces binding, so we allow any origin.
 		const origin = req.headers.origin;
-		if (
-			typeof origin === "string" &&
-			/^http:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/.test(origin)
-		) {
-			res.setHeader("Access-Control-Allow-Origin", origin);
+		const originAllowed =
+			allInterfaces ||
+			(typeof origin === "string" &&
+				new RegExp(
+					`^http://(localhost|127\\.0\\.0\\.1|::1|${host.replace(/\./g, "\\.")})(:\\d+)?$`,
+				).test(origin));
+		if (originAllowed) {
+			res.setHeader("Access-Control-Allow-Origin", origin ?? "*");
 			res.setHeader("Vary", "Origin");
 		}
 		res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, OPTIONS");
@@ -287,6 +305,7 @@ export async function launchDashboardServer(
 		if (handleVectorCortexLedger(req, res, ctx)) return;
 		if (handleVectorCortexTopology(req, res, ctx)) return;
 		if (handleVectorCortexQuery(req, res, ctx)) return;
+		if (handleVectorCortexShards(req, res, ctx)) return;
 		handleStatic(req, res, ctx);
 	});
 
@@ -310,20 +329,28 @@ export async function launchDashboardServer(
 				}
 			});
 
-			server.listen(port, "127.0.0.1", () => {
-				const url = `http://localhost:${port}`; // guardrails-allow PREVENT-PI-004: localhost dashboard URL (loopback-only)
-				log("server running", { url });
+			server.listen(port, host, () => {
+				// When bound to the loopback, keep the legacy localhost URL so the
+				// user's browser (which resolves localhost → ::1 or 127.0.0.1) works.
+				// When bound to a tailnet/tailscale IP, surface that address instead.
+				const url =
+					host === "127.0.0.1"
+						? `http://localhost:${port}`
+						: `http://${host}:${port}`; // guardrails-allow PREVENT-PI-004: dashboard URL (tailnet-local via Tailscale mesh or loopback)
+				log("server running", { url, host });
 				// eslint-disable-next-line no-console
 				console.log(`[mega-compact] dashboard server running: ${url}`);
 
-				// v0.8.2: also bind the IPv6 loopback (::1). On many systems `localhost`
-				// resolves to ::1 first (see /etc/hosts), so an IPv4-only bind makes the
-				// browser hit ::1:port and get connection refused. PREVENT-PI-004
-				// (loopback-only) means BOTH 127.0.0.1 and ::1. Non-fatal: IPv4-only
-				// hosts or a ::1 already in use just skip the mirror.
+				// v0.8.2: also bind the IPv6 loopback (::1) when bound to the IPv4
+				// loopback. On many systems `localhost` resolves to ::1 first (see
+				// /etc/hosts), so an IPv4-only bind makes the browser hit ::1:port and
+				// get connection refused. PREVENT-PI-004 (loopback-only) means BOTH
+				// 127.0.0.1 and ::1. When bound to a tailscale/0.0.0.0 host, we do NOT
+				// bind ::1 — the tailnet address is what the user reaches. Non-fatal:
+				// IPv4-only hosts or a ::1 already in use just skip the mirror.
 				let v6: ReturnType<typeof createServer> | undefined;
 				const v4Handler = server.listeners("request")[0];
-				if (v4Handler) {
+				if (host === "127.0.0.1" && v4Handler) {
 					v6 = createServer((r, s) =>
 						(v4Handler as (a: IncomingMessage, b: ServerResponse) => void).call(
 							server,
@@ -338,7 +365,7 @@ export async function launchDashboardServer(
 							message: e.message,
 						}),
 					);
-					v6.listen(port, "::1", () => log("ipv6 loopback bound", { port })); // guardrails-allow PREVENT-PI-004: IPv6 loopback (::1) mirror of the localhost dashboard server
+					v6.listen(port, "::1", () => log("ipv6 loopback bound", { port })); // guardrails-allow PREVENT-PI-004: IPv6 loopback (::1) mirror of the dashboard server
 				}
 
 				// Write port.pid

package/extensions/dashboard-server/tailscale.ts

@@ -1,3 +1,5 @@
+import { networkInterfaces } from "node:os";
+
 export function setupTailscaleServe(port = 3000, httpsPort = 443): boolean {
 	// Tailscale serve: exposes localhost dashboard securely.
 	// Only activates when TAILSCALE_ENABLED=1 is set.
@@ -5,3 +7,33 @@ export function setupTailscaleServe(port = 3000, httpsPort = 443): boolean {
 	console.log(`[tailscale] Serve enabled on port ${port} (https:${httpsPort})`);
 	return true;
 }
+
+// CGNAT block used by Tailscale for client addresses (100.64.0.0/10).
+const TAILSCALE_CGNAT_PREFIX = "100.";
+
+/**
+ * Detect the IPv4 address Tailscale has assigned to this host, so the dashboard
+ * server can auto-bind to the tailnet interface and be reachable from other
+ * devices on the tailnet. Returns null when Tailscale is not present.
+ *
+ * The default Tailscale interface name is `tailscale0` on Linux and `Tailscale`
+ * on macOS/Windows. We prefer an address inside Tailscale's CGNAT range
+ * (100.64.0.0/10) when one exists; otherwise we return the first IPv4 address
+ * found on the interface.
+ */
+export function detectTailscaleIP(): string | null {
+	const ifaces = networkInterfaces();
+	const candidates: Array<string> = [];
+	for (const [name, addrs] of Object.entries(ifaces)) {
+		if (name !== "tailscale0" && name !== "Tailscale") continue;
+		if (!addrs) continue;
+		for (const addr of addrs) {
+			if (addr.family === "IPv4") {
+				candidates.push(addr.address);
+			}
+		}
+	}
+	if (candidates.length === 0) return null;
+	const cgnat = candidates.find((ip) => ip.startsWith(TAILSCALE_CGNAT_PREFIX));
+	return cgnat ?? candidates[0];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.19.3",
+  "version": "0.20.0",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/vector-cortex.ts

@@ -134,6 +134,19 @@ export const VC3B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC3B");
  */
 export const VC3C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC3C");
 
+/**
+ * VC4A — dual-tier shard contract (SemanticShardV1 / ExactShardV1 /
+ * ShardManifestV1).
+ * Default ON. `MEGACOMPACT_VC4A=0` disables and is byte-identical to the
+ * predecessor (mode C: exact anchors/current transcript only, no semantic or
+ * exact shard manifest is assembled or emitted, zero
+ * `vector_cortex_shard_manifest_built` / `vector_cortex_protected_span_rejected`
+ * emissions; the predecessor derived pointer and goldens are unchanged). The
+ * real consumers are the shard assembly seam (manifest.ts) and the reader-only
+ * dashboard shards aggregate view.
+ */
+export const VC4A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC4A");
+
 // ---------------------------------------------------------------------------
 // Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
 // Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.

package/src/config.ts

@@ -163,6 +163,7 @@ export {
   VC3A_ENABLED,
   VC3B_ENABLED,
   VC3C_ENABLED,
+  VC4A_ENABLED,
   BREAKER_WINDOW_MS,
   BREAKER_MIN_ATTEMPTS,
   BREAKER_PERF_FAILURES,

package/src/vector-cortex/shards/exact.ts

@@ -0,0 +1,141 @@
+/**
+ * vector-cortex/shards/exact.ts — exact tier partition (VC4A).
+ *
+ * Carves the session's PROTECTED spans — every tool call/result pair, every
+ * anchor, every invalid UTF-8 event — into `ExactShardV1` payloads of the
+ * ORIGINAL bytes, verbatim and never normalized (SHD-UTF8-002: invalid bytes
+ * are exact-only and preserved unchanged — the codec's strict-fatal classifier
+ * means an invalid event has no lossy text to re-encode; we copy `originalBytes`
+ * directly). Each protected span is ATOMIC: a tool call/result pair is one
+ * `ProtectedSpan`, so no exact shard EVER splits a pair, even when the pair
+ * straddles the target-size boundary (SHD-PAIR-001).
+ *
+ * The partition walks the protected spans in canonical stream order (ascending
+ * first-seq), accrues each span's bytes, and closes an exact shard the moment
+ * adding the next atomic span would exceed `targetSize`. A single span over the
+ * budget still occupies exactly one shard — the pair case never splits.
+ *
+ * Triad: A = used with the semantic tier; B = the same exact spans preserved
+ * alongside extractive-derived semantic content; C = exact anchors/current
+ * transcript ONLY (this partition alone, with zero semantic shards).
+ *
+ * Pure/deterministic: hashes + array walking only, no storage, no console, no
+ * network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { createHash } from "node:crypto";
+import type { EventV2 } from "../ledger/types.js";
+import { cumulativeOffsets } from "./semantic.js";
+import type { ExactPartitionInput, ExactPartitionResult, ExactShardCase, ExactShardV1, ProtectedSpan } from "./types.js";
+
+/** Deterministic exact-shard digest over the verbatim original bytes. */
+function sealExactDigest(sessionId: string, bytes: Uint8Array): string {
+  const h = createHash("sha256");
+  h.update(`exact-shard-v1|${sessionId}|`);
+  h.update(bytes);
+  return `sha256:${h.digest("hex")}`;
+}
+
+/** Combine per-span cases into one shard case (invalid dominates, then tool-pair). */
+function combineCase(span: ProtectedSpan, acc: ExactShardCase): ExactShardCase {
+  if (span.case === "invalid-utf8" || acc === "invalid-utf8") return "invalid-utf8";
+  if (span.case === "tool-pair" || acc === "tool-pair") return "tool-pair";
+  return "anchor";
+}
+
+/** Concatenate an ordered group of events' original bytes, verbatim. */
+function concatBytes(events: readonly EventV2[]): Uint8Array {
+  let len = 0;
+  for (const e of events) len += e.originalBytes.length;
+  const out = new Uint8Array(len);
+  let off = 0;
+  for (const e of events) {
+    out.set(e.originalBytes, off);
+    off += e.originalBytes.length;
+  }
+  return out;
+}
+
+/**
+ * Partition the exact tier (task 3). Returns `ok:true` with a list of
+ * `ExactShardV1` whose (sorted) ranges are contiguous, non-overlapping and
+ * jointly cover every protected span exactly once, each preserving the original
+ * bytes verbatim. Defensive failures mirror the semantic tier: a bad target size
+ * rejects with `SHD_INVALID_TARGET_SIZE`; a protected span referencing an event
+ * outside the session's stream, or a mixed-session span, rejects with
+ * `SHD_CROSS_SESSION`.
+ */
+export function partitionExact(input: ExactPartitionInput): ExactPartitionResult {
+  const { sessionId, events, protectedSpans, targetSize } = input;
+  if (!Number.isFinite(targetSize) || targetSize <= 0) {
+    return { ok: false, code: "SHD_INVALID_TARGET_SIZE" };
+  }
+  const stream = new Set(events.map((e) => e.seq.toString()));
+  for (const span of protectedSpans) {
+    if (span.events.some((e) => e.sessionId !== sessionId || !stream.has(e.seq.toString()))) {
+      return { ok: false, code: "SHD_CROSS_SESSION" };
+    }
+  }
+  const offsets = cumulativeOffsets(events);
+  if (protectedSpans.length === 0) return { ok: true, shards: [] };
+
+  // Process protected spans in canonical stream order (ascending first-seq;
+  // equal first-seq preserves input order for determinism).
+  const ordered = [...protectedSpans].sort((a, b) => {
+    const as = a.events.length ? a.events[0].seq : 0n;
+    const bs = b.events.length ? b.events[0].seq : 0n;
+    return as < bs ? -1 : as > bs ? 1 : 0;
+  });
+
+  const shards: ExactShardV1[] = [];
+  let batch: ProtectedSpan[] = [];
+  let batchBytes = 0;
+  let batchCase: ExactShardCase = "anchor";
+
+  const spanBytes = (s: ProtectedSpan): number =>
+    s.events.reduce((n, e) => n + e.originalBytes.length, 0);
+
+  const flush = (): void => {
+    if (batch.length === 0) return;
+    const first = batch[0];
+    const last = batch[batch.length - 1];
+    const firstEv = first.events[0];
+    const lastEv = last.events[last.events.length - 1];
+    const start = offsets.of(firstEv.seq);
+    const end = offsets.of(lastEv.seq);
+    const byteStart = start ? start.byteStart : 0;
+    const byteEnd = end ? end.byteEnd : byteStart;
+    const bytes = concatBytes(batch.flatMap((s) => s.events));
+    shards.push({
+      schema: "exact-shard-v1",
+      sessionId,
+      range: {
+        sessionId,
+        seqStart: firstEv.seq,
+        seqEnd: lastEv.seq,
+        byteStart,
+        byteEnd,
+      },
+      kind: "exact",
+      originalBytes: bytes,
+      digest: sealExactDigest(sessionId, bytes),
+      byteCount: batchBytes,
+      case: batchCase,
+    });
+    batch = [];
+    batchBytes = 0;
+    batchCase = "anchor";
+  };
+
+  for (const span of ordered) {
+    const sz = spanBytes(span);
+    // Close at a complete-span boundary when adding this atomic span would
+    // exceed the budget and we already hold at least one complete span.
+    if (batch.length > 0 && batchBytes + sz > targetSize) flush();
+    batch.push(span);
+    batchBytes += sz;
+    batchCase = combineCase(span, batchCase);
+  }
+  flush();
+
+  return { ok: true, shards };
+}