pi-mega-compact 0.16.2 → 0.17.0

package/extensions/mega-runtime/vector-cortex-safety.ts

@@ -0,0 +1,181 @@
+/**
+ * vector-cortex-safety.ts — VC0C live safety envelope (VC0C, task 4 + 5).
+ *
+ * Owns the live triad selection BEFORE provider invocation: select mode A
+ * (optimized/learned), then independent mode B (deterministic local spool
+ * deriving directly from authority), then unchanged mode C (continuity, not
+ * semantic completeness — C uses only the exact current transcript and reports
+ * that limitation). A manual reset clears cooldown but NEVER evidence.
+ *
+ * This adapter composes the resilience seams (`resilience/breaker.ts`,
+ * `resilience/spool.ts`, `resilience/emit.ts`) into the health/reset API the
+ * dashboard owns (`GET /api/vector-cortex/health`, admin
+ * `POST /api/vector-cortex/breakers/reset`). It is the pi-runtime-adjacent shell;
+ * all durable state is owned by the src/ resilience modules. Non-fatal: any
+ * breaker/spool noise degrades to mode C (unchanged host transcript), and the
+ * agent loop is never broken.
+ *
+ * Local-only (PREVENT-PI-004); no `any` (PREVENT-011); no console.log — events
+ * go through the injected resilience reporter / Logger.
+ *
+ * LIVENESS HONESTY (VC0C-Q01/Q06): the breaker composed here is PER-PROCESS /
+ * IN-MEMORY and EPHEMERAL — `createVectorCortexSafety` builds a fresh instance on
+ * every dashboard request (GET /health, POST /reset). There is NO persistent
+ * breaker runtime and NO live producer wiring (the real circuit-breaker that runs
+ * around the actual compaction loop is DEFERRED to the producer-wiring sprint,
+ * carried VC0B-I08 -> VC0D). Consequently GET /health always reflects a brand-new
+ * CLOSED_A instance and POST /reset mutates an instance that vanishes when the
+ * request returns: the endpoints are ILLUSTRATIVE until VC0D wires a persistent
+ * instance. The health payload therefore carries `stateSource:"ephemeral"` so the
+ * dashboard/README never present it as a live breaker. Frontier-frozen / spool-lag
+ * are owned by the DURABLE spool but have no live per-session handle here, so they
+ * are surfaced as static/non-live until VC0D (with authorityOutage mirrored from
+ * the host context when supplied).
+ */
+
+import { join } from "node:path";
+import { createResilienceReporter, type ResilienceReporter } from "../../src/vector-cortex/resilience/emit.js";
+import { createBreaker } from "../../src/vector-cortex/resilience/breaker.js";
+import { createSpool, type Spool } from "../../src/vector-cortex/resilience/spool.js";
+import type { BreakerRecord, Mode, TriadResult } from "../../src/vector-cortex/resilience/types.js";
+import { BREAKER_WINDOW_MS, VC0C_ENABLED } from "../../src/config/vector-cortex.js";
+
+/** Daily/health context the safety adapter needs from its host. */
+export interface VectorCortexSafetyContext {
+  /** State directory hosting the durable breaker/spool state. */
+  readonly stateDir: string;
+  /** Monotonic clock for windows/cooldowns (fake-clock testable). */
+  readonly now?: () => number;
+  /** Wall-clock supplier for record timestamps (default ISO now). */
+  readonly wallNow?: () => string;
+  /** True while the mode-A authority (SQLite ledger) is in OUTAGE. */
+  readonly authorityOutage?: () => boolean;
+}
+
+/** Health-card aggregate (dashboard contract). */
+export interface VectorCortexHealthSummary {
+  readonly enabled: boolean;
+  readonly mode: Mode;
+  readonly state: string;
+  readonly subsystem: string;
+  readonly sinceMs: number;
+  readonly reason?: string;
+  readonly windowMs: number;
+  readonly probeCount: number;
+  readonly backoffDelayMs: number;
+  readonly frontierFrozen: boolean;
+  readonly authorityOutage: boolean;
+  readonly spoolLag: number;
+  readonly attempts: number;
+  readonly failures: number;
+  readonly p95Ms: number;
+  readonly failureRate: number;
+  readonly updatedAt: string;
+  /** "ephemeral" => per-process/in-memory breaker; "live" => persistent (VC0D). */
+  readonly stateSource: "ephemeral" | "live";
+}
+
+export interface TriadProvider<T> {
+  readonly session: string;
+  readonly subsystem: string;
+  readonly inputDigest: string;
+  /** Mode A — optimized/learned. */
+  readonly runA: () => T;
+  /** Mode B — deterministic local spool, derives directly from authority. */
+  readonly runB: () => T;
+  /** Mode C — unchanged current transcript (continuity, NOT completeness). */
+  readonly runC: () => T;
+  readonly validate: (v: T) => boolean;
+}
+
+export interface VectorCortexSafety {
+  /**
+   * Select A, then independent B, then unchanged C BEFORE provider invocation.
+   * Demotes on live failure; the selected mode's output is the only thing served
+   * (probe output is never served).
+   */
+  select<T>(provider: TriadProvider<T>): TriadResult<T>;
+  /** Reader-only aggregate for GET /api/vector-cortex/health. */
+  health(): VectorCortexHealthSummary;
+  /** The mode-B durable spool (append before provider; drain on recovery). */
+  readonly spool: Spool;
+  /** Admin reset: clears cooldown, never evidence. Returns the new record. */
+  reset(subsystem: string): BreakerRecord;
+  /** Manual halt with a required reason (authority/digest/causal corruption). */
+  halt(reason: string): BreakerRecord;
+}
+
+/** Build the live safety envelope, wiring the emit seam into the breaker. */
+export function createVectorCortexSafety(
+  ctx: VectorCortexSafetyContext,
+  emit?: (event: string, fields: Record<string, unknown>) => void,
+): VectorCortexSafety {
+  const reporter: ResilienceReporter = createResilienceReporter(emit);
+  const wallNow = ctx.wallNow ?? (() => new Date().toISOString());
+  const breaker = createBreaker({ now: ctx.now, reporter, wallNow });
+  const spool = createSpool({
+    dir: join(ctx.stateDir, "spool"),
+    authorityOutage: ctx.authorityOutage,
+    // Wire the resilience emit seam (incl. vector_cortex_frontier_frozen) into
+    // the mode-B spool so a real authority-frontier freeze is observable.
+    reporter,
+  });
+
+  return {
+    spool,
+    select<T>(provider: TriadProvider<T>): TriadResult<T> {
+      const { subsystem, inputDigest, runA, runB, runC, validate } = provider;
+      // Thin delegation to the breaker: it selects the highest healthy mode
+      // (A, else independent B, else unchanged C) BEFORE provider invocation and
+      // records the attempt/trip. Across invocations A-failure demotes to B and
+      // B-failure demotes to C (continuity — unchanged transcript).
+      return breaker.execute(
+        subsystem,
+        inputDigest,
+        { A: runA, B: runB, C: runC },
+        validate,
+      );
+    },
+
+    health(): VectorCortexHealthSummary {
+      const rec = breaker.snapshot("provider");
+      const enabled = VC0C_ENABLED();
+      // LIVENESS HONESTY (VC0C-Q01/Q06): this breaker is created fresh per request
+      // and is ephemeral — declare it explicitly so the dashboard never presents it
+      // as a live breaker. `authorityOutage` is mirrored from the host context when
+      // supplied (otherwise false). `frontierFrozen` (rec.frozenFrontier) and
+      // `spoolLag` are owned by the DURABLE spool, which has no live per-session
+      // handle here — they stay static/no-op until VC0D wires a persistent breaker
+      // instance; stateSource:"ephemeral" is the signal that they are not live.
+      return {
+        enabled,
+        mode: rec.state === "MANUAL_HALT" ? "C" : rec.state === "CLOSED_A" ? "A" : "B",
+        state: rec.state,
+        subsystem: "provider",
+        sinceMs: rec.transitionedAtMs,
+        reason: rec.manualReason,
+        windowMs: BREAKER_WINDOW_MS,
+        probeCount: rec.probeCount,
+        backoffDelayMs: rec.retryDelayMs,
+        frontierFrozen: rec.frozenFrontier,
+        authorityOutage: ctx.authorityOutage?.() ?? false,
+        spoolLag: 0,
+        attempts: rec.attempts,
+        failures: rec.failures,
+        p95Ms: rec.p95Ms,
+        failureRate: rec.failureRate,
+        updatedAt: wallNow(),
+        stateSource: "ephemeral",
+      };
+    },
+
+    reset(subsystem: string): BreakerRecord {
+      return breaker.reset(subsystem);
+    },
+
+    halt(reason: string): BreakerRecord {
+      return breaker.manualHalt(reason);
+    },
+  };
+}
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.16.2",
+  "version": "0.17.0",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",
@@ -47,6 +47,7 @@
     "clean": "rm -rf dist",
     "prebuild": "npm run clean",
     "build": "tsc -p tsconfig.json",
+    "postbuild": "node scripts/vector-cortex-publish-acceptance.mjs",
     "build:dashboard": "cd extensions/dashboard-client && npx vite build",
     "test:dashboard-smoke": "node scripts/dashboard-tab-smoke.mjs",
     "lint": "tsc --noEmit && node scripts/guardrails-scan.mjs && node scripts/semantic-scan.mjs",

package/src/config/vector-cortex.ts

@@ -0,0 +1,87 @@
+/**
+ * config/vector-cortex.ts — vector-cortex feature flags + breaker/triad constants.
+ *
+ * Every sprint ships one positive `MEGACOMPACT_<SPRINT>` flag, default ON and
+ * `=0`/`_DISABLED` off. Flag-OFF is byte-identical to the predecessor sprint's
+ * behavior (for VC0A: mode C — observer absent, zero evaluation writes).
+ *
+ * The breaker/triad constants (TRIAD_RESILIENCE.md) live here so VC0C consumes
+ * them without re-declaring the ownership boundary. Pi-agnostic, dependency-free.
+ */
+
+/** Positive sprint flag: `=0` or `_DISABLED=true` disables (default ON). */
+function sprintFlag(name: string): boolean {
+  const v = process.env[name];
+  if (v === "0" || v === "false") return false;
+  const disabled = process.env[name + "_DISABLED"];
+  if (disabled === "true" || disabled === "1") return false;
+  return true;
+}
+
+/** VC0A — baseline observability (MetricEventV1 / AnnotationV1). Default ON. */
+export const VC0A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC0A");
+
+/**
+ * VC0B — replay correctness (ReplayCutV2 / ReplayReportV2, M3 effective-cut-v2).
+ * Default ON. `MEGACOMPACT_VC0B=0` disables and is byte-identical to the
+ * predecessor (legacy capped-replay behavior preserved; the v2 cut/replay is
+ * only consulted on the vector-cortex path).
+ */
+export const VC0B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC0B");
+
+/**
+ * VC1A — canonical byte events (EventV2 / EventCodec).
+ * Default ON. `MEGACOMPACT_VC1A=0` disables and is byte-identical to the
+ * predecessor (mode C: ledger absent, current transcript codec unchanged).
+ * The single real consumer is the ledger emit seam (`ledger/emit.ts`): flag OFF
+ * gates zero observability writes.
+ */
+export const VC1A_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC1A");
+
+/**
+ * VC1B — occurrence ledger + tool identity + compat journal (LedgerReader/
+ * Writer/Admin, CompatJournalV1, M2 occurrence-v2 migration).
+ * Default ON. `MEGACOMPACT_VC1B=0` disables and is byte-identical to the
+ * predecessor (mode C: the neutral occurrence ledger is not written, no
+ * journal rows, zero `vector_cortex_occurrence_appended` emissions). The real
+ * consumers are the ledger write integration seam and the compat-journal
+ * switch seam.
+ */
+export const VC1B_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC1B");
+
+/**
+ * VC0C — live safety envelope (TriadResult / Breaker / KillDecision + durable
+ * spool). Default ON. `MEGACOMPACT_VC0C=0` disables and is byte-identical to
+ * the predecessor (mode C: selected before provider invocation, unchanged host
+ * transcript, breaker/spool idle and emitting nothing). The single real
+ * consumer is the resilience emit seam + the safety adapter's triad selection.
+ */
+export const VC0C_ENABLED = (): boolean => sprintFlag("MEGACOMPACT_VC0C");
+
+// ---------------------------------------------------------------------------
+// Breaker state machine constants (TRIAD_RESILIENCE.md §breaker).
+// Rolled numbers for one 60s window; VC0C consumes these at its breaker seam.
+// ---------------------------------------------------------------------------
+
+/** Rolling eligibility window (milliseconds). */
+export const BREAKER_WINDOW_MS = 60_000;
+/** Minimum attempts before a breaker may trip or promote. */
+export const BREAKER_MIN_ATTEMPTS = 20;
+/** Performance trip: ≥ this many failures in window, or ≥ this fraction. */
+export const BREAKER_PERF_FAILURES = 5;
+export const BREAKER_PERF_FAILURE_RATE = 0.1;
+/** Correctness trip trips on the first correctness failure. */
+export const BREAKER_CORRECTNESS_FAILURES = 1;
+/** Cooldown before an open breaker may probe (milliseconds). */
+export const BREAKER_COOLDOWN_MS = 30_000;
+/** Consecutive successful probes required to advance a state. */
+export const BREAKER_PROBE_COUNT = 3;
+/** Exponential retry base: 30s * 2^attempt, capped, ±10% jitter. */
+export const BREAKER_RETRY_BASE_MS = 30_000;
+export const BREAKER_RETRY_CAP_MS = 15 * 60_000;
+export const BREAKER_RETRY_JITTER = 0.1;
+/** Promotion hysteresis: failure rate must be < this and p95 within budget. */
+export const BREAKER_HYSTERESIS_FAILURE_RATE = 0.02;
+export const BREAKER_HYSTERESIS_BUDGET_P95_MS = 50;
+/** Minimum healthy residence before a further promotion (milliseconds). */
+export const BREAKER_MIN_HEALTHY_RESIDENCE_MS = 5 * 60_000;

package/src/config.ts

@@ -143,3 +143,30 @@ export const RAG_HYDE_ENABLED = (): boolean =>
 
 /** Spec 1: vbrainstorm visual design migration for the dashboard. */
 export const NEW_UI = (): boolean => ragEnabled("MEGACOMPACT_NEW_UI");
+
+// ---------------------------------------------------------------------------
+// Vector-cortex flags + breaker constants (VC0A+). Positive sprint flags,
+// default ON, `=0`/`_DISABLED` off. Re-exported from src/config/vector-cortex.ts
+// so root consumers share one source of truth.
+// ---------------------------------------------------------------------------
+
+export {
+  VC0A_ENABLED,
+  VC0B_ENABLED,
+  VC1A_ENABLED,
+  VC0C_ENABLED,
+  VC1B_ENABLED,
+  BREAKER_WINDOW_MS,
+  BREAKER_MIN_ATTEMPTS,
+  BREAKER_PERF_FAILURES,
+  BREAKER_PERF_FAILURE_RATE,
+  BREAKER_CORRECTNESS_FAILURES,
+  BREAKER_COOLDOWN_MS,
+  BREAKER_PROBE_COUNT,
+  BREAKER_RETRY_BASE_MS,
+  BREAKER_RETRY_CAP_MS,
+  BREAKER_RETRY_JITTER,
+  BREAKER_HYSTERESIS_FAILURE_RATE,
+  BREAKER_HYSTERESIS_BUDGET_P95_MS,
+  BREAKER_MIN_HEALTHY_RESIDENCE_MS,
+} from "./config/vector-cortex.js";

package/src/httpEmbedder.ts

@@ -26,7 +26,8 @@
  */
 
 import type { Embedder, Vector } from "./embedder.js";
-import { l2Normalize } from "./embedder.js";
+import { l2Normalize, TrigramEmbedder } from "./embedder.js";
+import { Logger } from "./log.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
 import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
 
@@ -40,6 +41,94 @@ export interface HttpEmbedderOptions {
   dim?: number;
 }
 
+// ── Oversized-input chunking + graceful fallback (BowTiedDevil 500 report) ────
+//
+// The embedding server enforces a physical batch size (llama.cpp n_ctx_slot,
+// commonly 2048 tokens); a single over-large prompt is rejected with HTTP 500
+// ("input is too large to process"). Previously embed() sent the WHOLE region
+// as one prompt and threw on the 500, propagating an unhandled rejection up
+// through VectorStore.addCheckpoint → engine and crashing the checkpoint write.
+//
+// Two layered defenses now live inside embed():
+//   1. CHUNKING — text whose estimated token count exceeds the configured batch
+//      limit is split into <=limit chunks (paragraph/sentence/word boundaries,
+//      never mid-word when avoidable), each chunk embedded, and the per-chunk
+//      vectors mean-pooled (weighted by chunk tokens) + L2-renormalized. This
+//      changes the vector for oversized docs — which previously crashed — and
+//      is the standard pooling approximation for long inputs.
+//   2. FALLBACK — if the server is unreachable / returns an error / returns an
+//      unparseable body, embed() catches and falls back to the local
+//      TrigramEmbedder (logging one structured warn, never the text), so a
+//      server outage degrades semantic dedup instead of crashing the agent loop
+//      (repo non-fatal-store / fails-closed convention).
+//
+// Token count is ESTIMATED, not tokenized: chars / MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN
+// (default 4, a conservative English/code average). Both knobs are env-tunable.
+
+/** Default physical batch the typical local server accepts (llama.cpp n_ctx_slot). */
+const DEFAULT_BATCH_TOKENS = 2048;
+/** Conservative chars-per-token average for English + code. */
+const DEFAULT_CHARS_PER_TOKEN = 4;
+
+function envInt(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined) return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+
+/** Estimated token count of `text` (chars / charsPerToken, rounded up). Exported for tests. */
+export function estimateTokens(text: string, charsPerToken: number): number {
+  return Math.ceil(text.length / charsPerToken);
+}
+
+/**
+ * Split `text` into chunks each estimated <= maxTokens. Prefers paragraph, then
+ * sentence, then word boundaries; falls back to a hard slice only when a single
+ * word/run still exceeds the limit (never splits a word unless unavoidable).
+ */
+export function chunkText(text: string, maxTokens: number, charsPerToken: number): string[] {
+  const maxChars = maxTokens * charsPerToken;
+  if (text.length <= maxChars) return [text];
+  const chunks: string[] = [];
+  let start = 0;
+  while (start < text.length) {
+    let end = Math.min(start + maxChars, text.length);
+    if (end < text.length) {
+      // Back off to the last nice boundary within the window: paragraph, then
+      // sentence, then whitespace. Require the boundary in the back half so we
+      // don't pathologically emit tiny chunks.
+      const windowStart = start + Math.floor(maxChars / 2);
+      const para = text.lastIndexOf("\n\n", end);
+      const sentence = text.lastIndexOf(". ", end);
+      const space = text.lastIndexOf(" ", end);
+      if (para >= windowStart) end = para + 2;
+      else if (sentence >= windowStart) end = sentence + 2;
+      else if (space >= windowStart) end = space + 1;
+      // else: no good boundary in the back half — hard-slice at maxChars.
+    }
+    const piece = text.slice(start, end);
+    if (piece.trim().length > 0) chunks.push(piece);
+    start = end;
+  }
+  return chunks;
+}
+
+/** Weighted mean-pool of equal-dim vectors (weights = chunk token estimates), then L2-renormalize. */
+export function meanPool(vectors: readonly Vector[], weights: readonly number[]): Vector {
+  const dim = vectors[0]?.length ?? 0;
+  const acc = new Array<number>(dim).fill(0);
+  let total = 0;
+  for (let i = 0; i < vectors.length; i++) {
+    const w = weights[i] ?? 0;
+    total += w;
+    const v = vectors[i];
+    for (let d = 0; d < dim; d++) acc[d] += v[d] * w;
+  }
+  if (total > 0) for (let d = 0; d < dim; d++) acc[d] /= total;
+  return l2Normalize(acc);
+}
+
 // Inline worker script: resolves a hostname via dns.lookup in a child process
 // (dns.lookup is callback-async; the child has its own event loop). Used to
 // verify that a hostname in the embedding URL resolves to loopback ONLY.
@@ -217,12 +306,20 @@ export class HttpEmbedder implements Embedder {
   private readonly apiKey?: string;
   private readonly headers: Record<string, string>;
   private resolvedDim: number;
+  private readonly batchTokens: number;
+  private readonly charsPerToken: number;
+  private readonly fallback: TrigramEmbedder;
+  private readonly logger: Logger;
 
   constructor(opts: HttpEmbedderOptions) {
     this.url = opts.url;
     this.apiKey = opts.apiKey;
     this.headers = opts.headers ?? {};
     this.resolvedDim = opts.dim ?? 0; // resolved after the first embed
+    this.batchTokens = envInt("MEGACOMPACT_EMBEDDING_BATCH_TOKENS", DEFAULT_BATCH_TOKENS);
+    this.charsPerToken = envInt("MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN);
+    this.fallback = new TrigramEmbedder();
+    this.logger = new Logger();
   }
 
   get dim(): number {
@@ -235,7 +332,43 @@ export class HttpEmbedder implements Embedder {
     return new URL("/api/chat", this.url).href;
   }
 
+  /**
+   * Embed `text`. Oversized inputs are chunked (each chunk <= the configured
+   * server batch) and mean-pooled; a server failure falls back to the local
+   * TrigramEmbedder. NEVER throws — a misbehaving server must degrade semantic
+   * dedup, not crash the checkpoint write path (VectorStore.addCheckpoint has
+   * no try/catch around embed()).
+   */
   embed(text: string): Vector {
+    const est = estimateTokens(text, this.charsPerToken);
+    const chunks =
+      est > this.batchTokens ? chunkText(text, this.batchTokens, this.charsPerToken) : [text];
+    try {
+      if (chunks.length === 1) {
+        return this.embedOne(chunks[0]);
+      }
+      const vectors: Vector[] = [];
+      const weights: number[] = [];
+      for (const c of chunks) {
+        vectors.push(this.embedOne(c));
+        weights.push(estimateTokens(c, this.charsPerToken));
+      }
+      return meanPool(vectors, weights);
+    } catch (e) {
+      // Graceful fallback — degrade to the local embedder rather than crash.
+      // Never log the text itself (privacy / PREVENT-PI-004).
+      this.logger.warn("embedder_http_fallback", {
+        reason: e instanceof Error ? e.message : String(e),
+        chunks: chunks.length,
+        chars: text.length,
+        estTokens: est,
+      });
+      return this.fallback.embed(text);
+    }
+  }
+
+  /** Single request/response round-trip for one chunk. Throws on any failure. */
+  private embedOne(text: string): Vector {
     const ollama = isOllamaEndpoint(this.url);
     const body = ollama
       ? JSON.stringify({ model: process.env.MEGACOMPACT_OLLAMA_MODEL || "nomic-embed-text", prompt: text })

package/src/vector-cortex/eval/annotations.ts

@@ -0,0 +1,70 @@
+/**
+ * vector-cortex/eval/annotations.ts — redaction metadata + JSONL serialization.
+ *
+ * Payload bytes, prompts, and exact ledger text are NEVER serialized. Before a
+ * JSONL line is written, each such field is replaced by a SHA-256 digest + byte
+ * count + redaction kind (EVAL-REDACT-002: prompt bytes never appear in JSONL).
+ * The redaction journal is itself the AnnotationV1 output.
+ *
+ * Uses node:crypto for the digest (pure local math, PREVENT-PI-004 safe).
+ */
+
+import { createHash } from "node:crypto";
+import type { AnnotationV1 } from "./types.js";
+
+export type RedactKind = "payload" | "prompt" | "ledger";
+
+/** A raw content field awaiting redaction before serialization. */
+export interface RawContent {
+  /** Field name in the original annotation. */
+  readonly field: string;
+  /** Kind of content — determines the redaction reason. */
+  readonly kind: RedactKind;
+  /** Original bytes; digested, never serialized. */
+  readonly bytes: Uint8Array;
+}
+
+/** Canonical serialization of an annotation: digest/count metadata only. */
+export interface SerializedAnnotation {
+  readonly jsonl: string;
+  readonly annotation: AnnotationV1;
+}
+
+/** SHA-256 over bytes, unpadded base64 (conformance canonical binary form). */
+export function digestBytes(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("base64");
+}
+
+/** Replace every raw content field with its digest/count metadata. */
+export function redactAnnotation(
+  itemId: string,
+  contents: ReadonlyArray<RawContent>,
+): AnnotationV1 {
+  const redactions = contents.map((c) => ({
+    field: c.field,
+    digest: digestBytes(c.bytes),
+    bytes: c.bytes.byteLength,
+    kind: c.kind,
+  }));
+  return {
+    itemId,
+    redactions,
+    redactedCount: redactions.length,
+  };
+}
+
+/**
+ * Serialize one set of raw contents to a single redacted JSONL line. Returns
+ * the annotation with its digest metadata and the JSONL text; the raw bytes
+ * never appear in the returned string.
+ */
+export function serializeRedactedJsonl(
+  itemId: string,
+  contents: ReadonlyArray<RawContent>,
+): SerializedAnnotation {
+  const annotation = redactAnnotation(itemId, contents);
+  return {
+    annotation,
+    jsonl: `${JSON.stringify(annotation)}\n`,
+  };
+}

package/src/vector-cortex/eval/metrics.ts

@@ -0,0 +1,113 @@
+/**
+ * vector-cortex/eval/metrics.ts — canonical metric ordering + fixed histogram.
+ *
+ * 1. Stable sort rows by `(session, seq, event)` — session string byte order,
+ *    then integer seq ascending, then event-name byte order as tiebreak
+ *    (EVAL-ORDER-003: equal seq rows use event-name order).
+ * 2. Bucket latency values at exactly 1/5/10/25/50/100/250ms with inclusive
+ *    boundaries (EVAL-BUCKET-001); values past 250ms kept in a separate
+ *    overflow bucket. Histogram totals are permutation-stable.
+ *
+ * Pure/no side effects (PREVENT-PI-004).
+ */
+
+import { LATENCY_BUCKETS } from "./types.js";
+import type { MetricEventV1, EvalReject } from "./types.js";
+
+/** Inclusive-boundary bucket edges, with a sentinel past the largest edge. */
+const EDGES: readonly number[] = LATENCY_BUCKETS;
+const OVERFLOW_INDEX = EDGES.length;
+
+/** Histogram cell counts aligned to EDGES, plus a trailing overflow cell. */
+export function emptyHistogram(): number[] {
+  return new Array<number>(EDGES.length + 1).fill(0);
+}
+
+/**
+ * Per-session histogram of `value` (when the sample is in `unit`), plus a
+ * non-latency count kept separate so totals are permutation-stable.
+ */
+export interface LatencyHistogram {
+  /** Cells: one per bucket edge, then overflow. Sum == counted latency samples. */
+  cells: number[];
+  /** Bucket label for each cell (overflow label is "250+" passed via >250). */
+  edges: readonly number[];
+  /** Number of latency samples that landed past the largest edge. */
+  overflow: number;
+  /** Total latency samples bucketed (cells sum). */
+  total: number;
+}
+
+export interface MetricsResult {
+  /** Rows in canonical `(session, seq, event)` order. */
+  readonly rows: readonly MetricEventV1[];
+  /** Per-session latency histograms (mode A observed). */
+  readonly histogram: LatencyHistogram;
+  /** Rejections encountered while validating streaming input. */
+  readonly rejects: readonly EvalReject[];
+}
+
+/** True when a sample carries a latency (`unit === "ms"`) datum. */
+export function isLatencySample(m: MetricEventV1): boolean {
+  return m.unit === "ms";
+}
+
+/** Bucket a latency ms value into a cell index (0..EDGES.length, last=overflow). */
+export function bucketIndex(value: number): number {
+  if (!Number.isFinite(value) || value < 0) return OVERFLOW_INDEX;
+  for (let i = 0; i < EDGES.length; i++) {
+    if (value <= EDGES[i]) return i; // inclusive upper boundary
+  }
+  return OVERFLOW_INDEX;
+}
+
+/**
+ * Bucket latency sample values across all rows into the fixed histogram.
+ * Overflow is kept separate (counted in `overflow` and the trailing cell).
+ */
+export function bucketHistogram(rows: readonly MetricEventV1[]): LatencyHistogram {
+  const cells = emptyHistogram();
+  let overflow = 0;
+  for (const m of rows) {
+    if (!isLatencySample(m)) continue;
+    const idx = bucketIndex(m.value);
+    cells[idx] += 1;
+    if (idx === EDGES.length) overflow += 1;
+  }
+  return {
+    cells,
+    edges: [...EDGES],
+    overflow,
+    total: cells.reduce((a, b) => a + b, 0),
+  };
+}
+
+/**
+ * Stable sort MetricEventV1 rows by `(session, seq, event)`:
+ *  - session: UTF-8 / JS string byte order;
+ *  - seq: ascending numeric;
+ *  - event: ascending string byte order (tiebreak for equal seq).
+ * Equal content stays separate occurrences (never deduplicated here).
+ */
+export function sortCanonical(rows: readonly MetricEventV1[]): MetricEventV1[] {
+  const out = [...rows];
+  out.sort((a, b) => {
+    const s = a.session < b.session ? -1 : a.session > b.session ? 1 : 0;
+    if (s !== 0) return s;
+    if (a.seq !== b.seq) return a.seq - b.seq;
+    return a.event < b.event ? -1 : a.event > b.event ? 1 : 0;
+  });
+  return out;
+}
+
+/**
+ * Assisted sort plus histogram, returning both under one result.
+ * The observer (mode A) calls this before JSONL serialization.
+ */
+export function buildMetrics(rows: readonly MetricEventV1[]): MetricsResult {
+  return {
+    rows: sortCanonical(rows),
+    histogram: bucketHistogram(rows),
+    rejects: [],
+  };
+}