pi-mega-compact 0.16.2 → 0.17.0

package/extensions/dashboard-server/routes-rag-settings-helpers.ts

@@ -272,6 +272,41 @@ export const SETTINGS: ReadonlyArray<{
 			num("MEGACOMPACT_EMBED_CACHE", "Embed Cache Size", "Embedding cache entries (0 = disabled)", 256, 0, 10000),
 		],
 	},
+	{
+		name: "Vector Cortex",
+		settings: [
+			boolDirect(
+				"MEGACOMPACT_VC0A",
+				"VC0A Baseline Observability",
+				"Structured evaluation observer (MetricEventV1 + latency histogram). OFF = mode C, byte-identical to predecessor.",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_VC0B",
+				"VC0B Replay Correctness",
+				"ReplayCutV2 effective-cut (min of boundary-safe/commit/capture high-water + pair retreat + anchor floor) and M3 effective-cut-v2 migration. OFF = legacy capped replay, byte-identical.",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_VC1A",
+				"VC1A Canonical Byte Events",
+				"EventV2 byte-authority ledger codec (original bytes + SHA-256, strict UTF-8, derived NFC) and canonical validator (EVT_DIGEST_MISMATCH / EVT_UTF8_TAG_INVALID / EVT_DUPLICATE_ID). OFF = mode C, transcript codec unchanged, byte-identical.",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_VC1B",
+				"VC1B Occurrence Ledger + Tool Identity",
+				"Neutral occurrence ledger (LedgerReader/Writer/Admin + CompatJournalV1): per-session monotonic seq, tool result references one earlier call, uniqueness by (eventId,digest) only, and the M2 copy-validate-switch downgrade journal. OFF = mode C, ledger unwritten, byte-identical.",
+				true,
+			),
+			boolDirect(
+				"MEGACOMPACT_VC0C",
+				"VC0C Live Safety Envelope",
+				"TriadResult/Breaker live circuit breaker (60s window, 20 attempts, 30s cooldown, 3 probes, 5min healthy residence) + durable spool before provider invocation; manual reset clears cooldown but never evidence. OFF = mode C, unchanged transcript, byte-identical.",
+				true,
+			),
+		],
+	},
 	{
 		name: "Cost API",
 		settings: [

package/extensions/dashboard-server/routes-vector-cortex.ts

@@ -0,0 +1,332 @@
+/**
+ * dashboard-server/routes-vector-cortex.ts — vector-cortex dashboard routes.
+ *
+ * VC0A ships a single reader-only GET /api/vector-cortex/evaluation that
+ * aggregates redacted evaluation rows (persisted JSONL under the state dir)
+ * via the reader. No writable capability — no reset/mutation here; VC0C adds
+ * breaker health + reset endpoints into this same file, keeping it additive
+ * and within extension limits.
+ *
+ * VC0C (task 5) adds:
+ *   GET  /api/vector-cortex/health          — reader-only aggregate health card
+ *   POST /api/vector-cortex/breakers/reset  — explicit admin capability
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read only), PREVENT-011 (no
+ * `any`), reader-only aggregate (never payloads/prompts/ledger — EVAL-REDACT-002).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import type { RouteContext } from "./routes-core.js";
+import { VC0A_ENABLED, VC0C_ENABLED, VC1B_ENABLED } from "../../src/config.js";
+import { readEvalRows } from "../../src/vector-cortex/eval/persist.js";
+import { summarizeEvalRows } from "../../src/vector-cortex/eval/reader.js";
+import { createVectorCortexSafety } from "../mega-runtime/vector-cortex-safety.js";
+import { createLedgerStore } from "../../src/vector-cortex/ledger/store.js";
+import type {
+  VectorCortexEvaluationSummary,
+  VectorCortexHealthCard,
+  VectorCortexLedgerView,
+  VectorCortexResetResult,
+} from "./api-contracts/vector-cortex.js";
+
+/** Admin capability: an actor-supplied subsystem selector for breaker reset. */
+function readJsonBody(
+  req: IncomingMessage,
+  cb: (
+    result: { ok: true; value: Record<string, unknown> } | { ok: false; error: string },
+  ) => void,
+): void {
+  let body = "";
+  let tooBig = false;
+  req.on("data", (chunk: Buffer) => {
+    if (body.length > 65536) {
+      tooBig = true;
+      return;
+    }
+    body += chunk.toString();
+  });
+  req.on("end", () => {
+    if (tooBig) return cb({ ok: false, error: "body_too_large" });
+    try {
+      const v = body ? JSON.parse(body) : {};
+      if (typeof v !== "object" || v === null || Array.isArray(v)) {
+        return cb({ ok: false, error: "invalid_object" });
+      }
+      cb({ ok: true, value: v as Record<string, unknown> });
+    } catch {
+      cb({ ok: false, error: "invalid_json" });
+    }
+  });
+}
+
+function sendJson(res: ServerResponse, status: number, body: unknown): void {
+  // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+  res.writeHead(status, { "Content-Type": "application/json" });
+  res.end(JSON.stringify(body));
+}
+
+/** Reader-only aggregate GET /api/vector-cortex/evaluation. */
+export function handleVectorCortexEvaluation(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  if (url !== "/api/vector-cortex/evaluation") return false;
+  if (req.method !== "GET") {
+    // Reader-only: no mutation endpoints exist on this path.
+    // guardrails-allow PREVENT-PI-004: loopback dashboard response (local)
+    res.writeHead(405, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ error: "method_not_allowed" }));
+    return true;
+  }
+
+  const enabled = VC0A_ENABLED();
+  const rows = enabled ? readEvalRows(ctx.stateDir) : [];
+  const summary = summarizeEvalRows(rows);
+
+  const body: VectorCortexEvaluationSummary = {
+    enabled,
+    mode: enabled ? "A" : "C",
+    samples: summary.samples,
+    byMode: {
+      A: summary.byMode.A,
+      B: summary.byMode.B,
+      C: summary.byMode.C,
+    },
+    histogram: {
+      edges: [...summary.histogram.edges],
+      cells: [...summary.histogram.cells],
+      overflow: summary.histogram.overflow,
+      total: summary.histogram.total,
+    },
+    rejects: [], // VC0C wires live breaker/reject telemetry here
+    updatedAt: new Date().toISOString(),
+  };
+  sendJson(res, 200, body);
+  return true;
+}
+
+/**
+ * Reader-only GET /api/vector-cortex/health (VC0C).
+ *
+ * Aggregate health card: breaker state (window/probe/backoff), durable spool
+ * frontier/authority/lag, and a worst-state aggregate. Constructs the safety
+ * adapter over the state dir — durable spool state (frontier-frozen, lag) is
+ * read from disk, so it reflects restart. Purely read: no mutation, never
+ * payloads/prompts/ledger.
+ */
+export function handleVectorCortexHealth(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  if (url !== "/api/vector-cortex/health") return false;
+  if (req.method !== "GET") {
+    // Reader-only path: no mutation endpoint lives at /health.
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+
+  const enabled = VC0C_ENABLED();
+  const safety = enabled ? createVectorCortexSafety({ stateDir: ctx.stateDir }) : null;
+  const card = safety ? safety.health() : null;
+
+  const fallback: VectorCortexHealthCard = {
+    enabled: false,
+    mode: "C",
+    state: "CLOSED_A",
+    subsystem: "provider",
+    sinceMs: 0,
+    windowMs: 0,
+    probeCount: 0,
+    backoffDelayMs: 0,
+    frontierFrozen: false,
+    authorityOutage: false,
+    spoolLag: 0,
+    attempts: 0,
+    failures: 0,
+    p95Ms: 0,
+    failureRate: 0,
+    updatedAt: new Date().toISOString(),
+    aggregate: "CLOSED_A",
+    stateSource: "ephemeral",
+  };
+  if (!card) {
+    sendJson(res, 200, fallback);
+    return true;
+  }
+
+  const mode: "A" | "B" | "C" =
+    card.state === "CLOSED_A" ? "A" : card.state === "MANUAL_HALT" ? "C" : "B";
+  const aggregate =
+    card.state === "MANUAL_HALT"
+      ? "MANUAL_HALT"
+      : card.state === "OPEN_C"
+        ? "OPEN_C"
+        : card.state === "OPEN_B"
+          ? "OPEN_B"
+          : card.state === "PROBE_B" || card.state === "PROBE_A"
+            ? "PROBE"
+            : "CLOSED_A";
+
+  const body: VectorCortexHealthCard = {
+    enabled,
+    mode,
+    state: card.state,
+    subsystem: card.subsystem,
+    sinceMs: card.sinceMs,
+    reason: card.reason,
+    windowMs: card.windowMs,
+    probeCount: card.probeCount,
+    backoffDelayMs: card.backoffDelayMs,
+    frontierFrozen: card.frontierFrozen,
+    authorityOutage: card.authorityOutage,
+    spoolLag: card.spoolLag,
+    attempts: card.attempts,
+    failures: card.failures,
+    p95Ms: card.p95Ms,
+    failureRate: card.failureRate,
+    updatedAt: new Date().toISOString(),
+    aggregate,
+    stateSource: card.stateSource,
+  };
+  sendJson(res, 200, body);
+  return true;
+}
+
+/**
+ * Admin POST /api/vector-cortex/breakers/reset (VC0C).
+ *
+ * Explicit admin capability (separate from the reader GET). Clears a breaker's
+ * cooldown — or unwires a MANUAL_HALT — but NEVER evidence (attempts/failures
+ * are retained, per TRIAD_RESILIENCE). The subsystem is actor-supplied via the
+ * JSON body `{ subsystem }`. Every reset is documented in the returned record
+ * (state, retained evidence, updatedAt) — an auditable mutation surface; the
+ * GET path can never mutate.
+ */
+export function handleVectorCortexBreakersReset(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  if (url !== "/api/vector-cortex/breakers/reset") return false;
+  if (req.method !== "POST") {
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+  if (!VC0C_ENABLED()) {
+    sendJson(res, 409, { error: "vco_c_disabled" });
+    return true;
+  }
+
+  readJsonBody(req, (parsed) => {
+    if (!parsed.ok) {
+      sendJson(res, 400, { error: parsed.error });
+      return;
+    }
+    const subsystem = parsed.value.subsystem;
+    if (typeof subsystem !== "string" || subsystem.length === 0) {
+      sendJson(res, 400, { error: "subsystem_required" });
+      return;
+    }
+    try {
+      const safety = createVectorCortexSafety({ stateDir: ctx.stateDir });
+      const rec = safety.reset(subsystem);
+      const body: VectorCortexResetResult = {
+        subsystem: rec.subsystem,
+        state: rec.state,
+        cooldownCleared: true,
+        attempts: rec.attempts,
+        failures: rec.failures,
+        probeCount: rec.probeCount,
+        manualReason: rec.manualReason,
+        updatedAt: rec.updatedAt,
+      };
+      sendJson(res, 200, body);
+    } catch {
+      sendJson(res, 500, { error: "reset_failed" });
+    }
+  });
+  return true;
+}
+
+/**
+ * Reader-only GET /api/vector-cortex/ledger (VC1B).
+ *
+ * Built on the LedgerReader capability: opens the occurrence-v2 ledger for this
+ * repo's state dir and returns the session's occurrence IDENTITY rows
+ * (seq/eventId/kind/digest/toolCallId) plus high-water/count. NEVER ships
+ * sourceBytes or prompt text (reader-only no-ledger-text rule). Optional
+ * `?session=<id>` query selects the session (default "default").
+ */
+export function handleVectorCortexLedger(
+  req: IncomingMessage,
+  res: ServerResponse,
+  ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  const path = url.split("?")[0] ?? url;
+  if (path !== "/api/vector-cortex/ledger") return false;
+  if (req.method !== "GET") {
+    // Reader-only path: no mutation endpoint lives at /ledger.
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+
+  // Parse the optional session query parameter.
+  let session = "default";
+  const qs = url.split("?")[1];
+  if (qs !== undefined) {
+    for (const pair of qs.split("&")) {
+      const [k, v] = pair.split("=");
+      if (k === "session" && v !== undefined && v.length > 0) {
+        session = decodeURIComponent(v);
+      }
+    }
+  }
+
+  const enabled = VC1B_ENABLED();
+  let active = enabled;
+  let highWater = "0";
+  let count = 0;
+  let occurrences: VectorCortexLedgerView["occurrences"] = [];
+
+  if (enabled) {
+    try {
+      const store = createLedgerStore({ stateDir: ctx.stateDir });
+      try {
+        const reader = store.reader();
+        highWater = reader.highWater(session).toString();
+        const rows = reader.readSession(session);
+        count = rows.length;
+        // Identity only: never sourceBytes or prompt text (reader-only rule).
+        occurrences = rows.slice(-500).map((occ) => ({
+          seq: occ.seq.toString(),
+          eventId: occ.eventId,
+          kind: occ.kind,
+          digest: occ.digest,
+          ...(occ.toolCallId !== undefined ? { toolCallId: occ.toolCallId } : {}),
+        }));
+      } finally {
+        store.close();
+      }
+    } catch {
+      // Non-fatal: a missing/corrupt ledger DB degrades to `enabled:false`.
+      active = false;
+    }
+  }
+
+  const body: VectorCortexLedgerView = {
+    enabled: active,
+    session,
+    highWater,
+    count,
+    occurrences,
+    updatedAt: new Date().toISOString(),
+  };
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes.ts

@@ -40,3 +40,9 @@ export { handleRagSettings } from "./routes-rag-settings.js";
 export { handleRagMetrics } from "./routes-rag-metrics.js";
 export { handleModelThresholds } from "./routes-model-thresholds.js";
 export { handleWiki } from "./routes-wiki.js";
+export {
+	handleVectorCortexEvaluation,
+	handleVectorCortexHealth,
+	handleVectorCortexBreakersReset,
+	handleVectorCortexLedger,
+} from "./routes-vector-cortex.js";

package/extensions/dashboard-server/server.ts

@@ -56,6 +56,10 @@ import {
 	handleRagMetrics,
 	handleModelThresholds,
 	handleWiki,
+	handleVectorCortexEvaluation,
+	handleVectorCortexHealth,
+	handleVectorCortexBreakersReset,
+	handleVectorCortexLedger,
 	handleStatic,
 } from "./routes.js";
 
@@ -275,6 +279,10 @@ export async function launchDashboardServer(
 		if (handleRagMetrics(req, res, ctx)) return;
 		if (handleModelThresholds(req, res, ctx)) return;
 		if (handleWiki(req, res, ctx)) return;
+		if (handleVectorCortexEvaluation(req, res, ctx)) return;
+		if (handleVectorCortexHealth(req, res, ctx)) return;
+		if (handleVectorCortexBreakersReset(req, res, ctx)) return;
+		if (handleVectorCortexLedger(req, res, ctx)) return;
 		handleStatic(req, res, ctx);
 	});
 

package/extensions/mega-events/context-handler.ts

@@ -35,6 +35,7 @@ import {
 	type MegaConfig,
 } from "../mega-config.js";
 import { appendMirrorMessages } from "./mirror-append.js";
+import { appendMessagesToLedger } from "../mega-runtime/vector-cortex-ledger.js";
 import { epochIdFor } from "../../src/mirror/epoch.js";
 import { computeLiveTrimCut, liveTrimSummaryMessage } from "../mega-trim.js";
 import { messageContentText } from "./context-handler/messageText.js";
@@ -149,6 +150,21 @@ export function registerContextHandler(
 			}
 		}
 
+		// VC1B (S1): canonical messages -> v2 ledger occurrences. Flag-OFF opens
+		// no DB (byte-identical predecessor); non-fatal. onFailure surfaces
+		// per-append rejections (e.g. EVT_SEQ_REGRESSION on rewind/fork) as
+		// structured warnings rather than swallowing them silently.
+		try {
+			appendMessagesToLedger(
+				runtime.currentStateDir,
+				runtime.rt.sessionId,
+				messages,
+				runtime.logger,
+			);
+		} catch (e) {
+			runtime.logger.warn("vc1b-ledger-append-fail", { error: String(e) });
+		}
+
 		// S52 / v0.16.1: per-model threshold override. The user can tune the
 		// fire point + safety margin PER MODEL (different providers' models range
 		// 8K-1M+ context, so one global tier % is wrong). Falls back to env/default

package/extensions/mega-runtime/vector-cortex-ledger.ts

@@ -0,0 +1,165 @@
+/**
+ * vector-cortex-ledger.ts — VC1B occurrence-ledger writer wiring (S1).
+ *
+ * Wires ONLY the `LedgerWriter` capability into message ingestion: each
+ * canonical message that flows through the compaction pipeline becomes an
+ * occurrence in the v2 ledger. The wiring is writer-only (never reader/admin
+ * here — the dashboard GET owns the reader), matching the capability-gated
+ * CONTRACTS §Store rule. A tool RESULT occurrence references exactly one
+ * earlier call via `toolCallId` (the ledger enforces EVT_TOOL_CALL_MISSING).
+ *
+ * When MEGACOMPACT_VC1B is OFF the seam opens NO ledger DB at all — and even if
+ * a caller already holds a store, its write path is inert (S2 flag-gate in
+ * store.ts) — so the OFF behavior is byte-identical to the pre-VC1B predecessor.
+ *
+ * Local-only (PREVENT-PI-004), no `any` (PREVENT-011), no console.log (events
+ * go through the ledger emit seam); non-fatal — a ledger write failure never
+ * breaks the agent loop.
+ */
+import { createHash } from "node:crypto";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import { VC1B_ENABLED } from "../../src/config/vector-cortex.js";
+import {
+	createLedgerStore,
+	type LedgerWriter,
+} from "../../src/vector-cortex/ledger/store.js";
+
+/** A single canonical message mapped to a ledger occurrence input. */
+export interface LedgerMessageInput {
+	readonly session: string;
+	readonly seq: bigint;
+	readonly eventId: string;
+	readonly kind: string;
+	readonly toolCallId?: string;
+	readonly sourceBytes: Uint8Array;
+}
+
+/** The writer handle handed to ingestion. */
+export interface LedgerWriterHandle {
+	readonly writer: LedgerWriter;
+	readonly close: () => void;
+}
+
+/** Open the ledger writer for ingestion, or null when VC1B is disabled. */
+export function openLedgerWriter(stateDir: string): LedgerWriterHandle | null {
+	if (!VC1B_ENABLED()) return null;
+	const store = createLedgerStore({ stateDir });
+	return { writer: store.writer(), close: () => store.close() };
+}
+
+/**
+ * Append one occurrence for a canonical message. `kind` is the neutral kind;
+ * when `toolCallId` is present the ledger enforces exactly-one reference to an
+ * earlier call. Flag-OFF (null handle) and the writer's own inert path are
+ * no-ops. Non-fatal: never throws to the loop — a failure is surfaced as
+ * `{ ok: false, code }` so the caller can log the reason (Q02).
+ */
+export function appendLedgerMessage(
+	handle: LedgerWriterHandle,
+	input: LedgerMessageInput,
+): { ok: boolean; code?: string } {
+	try {
+		const res = handle.writer.append(input);
+		return res.ok ? { ok: true } : { ok: false, code: res.code };
+	} catch (e) {
+		return { ok: false, code: String(e) };
+	}
+}
+
+/** Structured logger sink for seam append failures (wired by the extension). */
+export interface LedgerAppendLogger {
+	readonly warn: (event: string, fields: Record<string, unknown>) => void;
+}
+
+/**
+ * Append one occurrence per canonical message in `messages` (session-scoped,
+ * sequence = 1-based index). Opens the writer, appends each, closes the handle.
+ * Returns the number of accepted occurrences (0 when disabled OR when every
+ * append was a non-fatal no-op). This is the S1 ingestion seam.
+ *
+ * Monotonic-prefix assumption (Q02): the FIRST accepted append establishes the
+ * per-session high-water, and each subsequent append must carry `seq =
+ * highWater + 1`. Append order here follows `messages` array order with 1-based
+ * indices, so a shorter/mid-removal/reordered `messages` list after a prior
+ * taller run (rewind/fork) can drive `seq` backwards and be rejected with
+ * `EVT_SEQ_REGRESSION`. Such appends fail on the ledger and, without `onFailure`,
+ * are swallowed silently — so the caller should pass `onFailure` to log them.
+ */
+export function appendMessagesToLedger(
+	stateDir: string,
+	sessionId: string,
+	messages: AgentMessage[],
+	onFailure?: Readonly<LedgerAppendLogger>,
+): number {
+	const ledger = openLedgerWriter(stateDir);
+	if (!ledger) return 0;
+	try {
+		let accepted = 0;
+		for (let i = 0; i < messages.length; i++) {
+			const input = messageToLedgerInput(messages[i]!, sessionId, i);
+			const res = appendLedgerMessage(ledger, input);
+			if (res.ok) {
+				accepted++;
+			} else if (onFailure) {
+				onFailure.warn("vc1b-ledger-append-skip", {
+					session: sessionId,
+					seq: input.seq.toString(),
+					eventId: input.eventId,
+					kind: input.kind,
+					reason: res.code ?? "unknown",
+				});
+			}
+		}
+		return accepted;
+	} finally {
+		ledger.close();
+	}
+}
+
+/**
+ * Map an AgentMessage to a ledger occurrence input. `eventId` is the stable
+ * content digest so re-observed identical messages are (event_id,digest)-deduped;
+ * `kind` mirrors the canonical role. `toolCallId` is intentionally NOT set here:
+ * pi discriminates messages by `role` only and the tool-call id lives in
+ * variant-specific `.content`/`.tool_calls` we must not reach into, so the
+ * exactly-one tool back-edge (EVT_TOOL_CALL_MISSING) is exercised at the
+ * store/acceptance level and the full live mapping is deferred to the VC1
+ * producer-wiring sprint.
+ */
+export function messageToLedgerInput(
+	msg: AgentMessage,
+	session: string,
+	indexInSession: number,
+): LedgerMessageInput {
+	const m = msg as { role?: string; id?: string };
+	const text = messageText(msg);
+	const role = m.role ?? "unknown";
+	const digest = `sha256:${createHash("sha256").update(text).digest("hex")}`;
+	return {
+		session,
+		seq: BigInt(indexInSession + 1),
+		eventId: m.id ?? digest,
+		kind: role,
+		sourceBytes: new TextEncoder().encode(text),
+	};
+}
+
+/** Best-effort text extraction from an AgentMessage (string or part array). */
+function messageText(msg: AgentMessage): string {
+	const content = (msg as { content?: unknown }).content;
+	if (content == null) return "";
+	if (typeof content === "string") return content;
+	if (Array.isArray(content)) {
+		return content
+			.map((p) => {
+				const part = p as { type?: string; text?: string };
+				return typeof part.text === "string" ? part.text : "";
+			})
+			.join("\n");
+	}
+	if (typeof content === "object") {
+		const maybe = (content as Record<string, unknown>).text;
+		if (typeof maybe === "string") return maybe;
+	}
+	return "";
+}