pi-mega-compact 0.20.7 → 0.20.9

package/src/vector-cortex/heal/restore-readers.ts

@@ -0,0 +1,145 @@
+/**
+ * vector-cortex/heal/restore-readers.ts — the two EXACT source readers (VC6B).
+ *
+ * Split out of `restore.ts` so the orchestrator stays a short, readable policy
+ * file and each reader can be reasoned about on its own (PRACTICES: split at the
+ * 300-line soft limit, delegate-shell + impl).
+ *
+ * Both readers share one discipline: they return bytes ONLY after recomputing
+ * SHA-256 over the bytes they actually hold and comparing it to the digest the
+ * REQUEST pinned. Neither reader trusts the digest recorded alongside its own
+ * source — an `ExactShardV1.digest` is metadata that lives in the same file as
+ * the bytes, so a swapped file carries a matching (wrong) pair. Only the
+ * caller-supplied digest is authoritative, and only a fresh hash of the fetched
+ * bytes can be checked against it.
+ *
+ * Pure/deterministic/local: `node:crypto` only (a Node built-in, not a network
+ * call), no storage, no console (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { createHash } from "node:crypto";
+
+import type { EventV2, ExactShardV1, ShardRange } from "./restore-types.js";
+
+/**
+ * SHA-256 over bytes as LOWERCASE HEX with NO `sha256:` prefix — the
+ * `ExactShardV1.digest` / `ReconstructionSpan.digest` / `RestoreSpanRequest`
+ * convention. `EventV2.bytesDigest` prefixes this same value with `sha256:`.
+ */
+export function sha256Hex(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/**
+ * Strip the `sha256:` prefix from an `EventV2.bytesDigest` so it can be compared
+ * against a bare hex digest. A value that does not carry the prefix is returned
+ * unchanged rather than mangled — the comparison then simply fails, which is the
+ * correct outcome for a malformed record.
+ */
+export function bareHex(digest: string): string {
+  return digest.startsWith("sha256:") ? digest.slice("sha256:".length) : digest;
+}
+
+/**
+ * Structural range equality. All five components must match: two spans over the
+ * same seq window but different byte offsets are DIFFERENT spans (a re-encoded
+ * or re-offset stream is not the stream that was requested).
+ */
+export function rangeEquals(a: ShardRange, b: ShardRange): boolean {
+  return (
+    a.sessionId === b.sessionId &&
+    a.seqStart === b.seqStart &&
+    a.seqEnd === b.seqEnd &&
+    a.byteStart === b.byteStart &&
+    a.byteEnd === b.byteEnd
+  );
+}
+
+/** What a reader attempt produced: bytes, a stated failure, or simply nothing. */
+export type ReadOutcome =
+  /** Bytes fetched AND verified against the request digest. */
+  | { readonly kind: "bytes"; readonly bytes: Uint8Array }
+  /** A source existed but its bytes did not hash to the pinned digest. */
+  | { readonly kind: "digest-mismatch" }
+  /** No source of this tier covers the span; try the next tier. */
+  | { readonly kind: "absent" };
+
+/**
+ * Mode A — indexed exact-shard read.
+ *
+ * Selection is by RANGE identity plus the shard's own recorded digest, which
+ * narrows the candidate set the way a real index lookup would. The accepted
+ * candidate is then re-hashed from its `originalBytes`: this second check is
+ * what catches a shard whose bytes were swapped AFTER the index lookup resolved
+ * (the sprint's unique failure injection). A candidate that matches by range but
+ * fails either digest check reports `digest-mismatch` — it does NOT silently fall
+ * through to the ledger, because a corrupt exact shard is a fact worth surfacing.
+ */
+export function readExactShard(
+  shards: readonly ExactShardV1[],
+  range: ShardRange,
+  digest: string,
+): ReadOutcome {
+  const byRange = shards.filter((s) => rangeEquals(s.range, range));
+  if (byRange.length === 0) return { kind: "absent" };
+
+  const candidate = byRange.find((s) => s.digest === digest);
+  if (candidate === undefined) return { kind: "digest-mismatch" };
+
+  // Defense in depth: the shard's recorded digest is metadata, the bytes are the
+  // authority. Re-hash what we actually hold.
+  const bytes = candidate.originalBytes;
+  if (sha256Hex(bytes) !== digest) return { kind: "digest-mismatch" };
+  return { kind: "bytes", bytes };
+}
+
+/**
+ * Mode B — ledger range scan.
+ *
+ * An INDEPENDENT path from mode A: no shard index is consulted. The occurrence
+ * records covering `[seqStart..seqEnd]` are selected, sorted ascending by seq
+ * (the caller's array order is untrusted — a scan that concatenated records in
+ * arrival order would produce plausible-looking but wrong bytes), each record's
+ * own `bytesDigest` is verified, and the concatenation is hashed against the
+ * span digest.
+ *
+ * Both checks matter. Per-record verification localizes corruption to a single
+ * occurrence; the span-level hash catches a scan that is individually valid but
+ * collectively wrong — a missing record in the middle of the range, or a range
+ * that covers different content than the requester believed.
+ */
+export function readLedgerSpan(
+  events: readonly EventV2[],
+  range: ShardRange,
+  digest: string,
+): ReadOutcome {
+  const covering = events
+    .filter(
+      (e) =>
+        e.sessionId === range.sessionId &&
+        e.seq >= range.seqStart &&
+        e.seq <= range.seqEnd,
+    )
+    .sort((a, b) => (a.seq < b.seq ? -1 : a.seq > b.seq ? 1 : 0));
+
+  if (covering.length === 0) return { kind: "absent" };
+
+  // Per-record byte authority (VC1A): a record whose own digest disagrees with
+  // its bytes is corrupt, and a corrupt record can never contribute to a restore.
+  for (const e of covering) {
+    if (sha256Hex(e.originalBytes) !== bareHex(e.bytesDigest)) {
+      return { kind: "digest-mismatch" };
+    }
+  }
+
+  const total = covering.reduce((sum, e) => sum + e.originalBytes.length, 0);
+  const bytes = new Uint8Array(total);
+  let offset = 0;
+  for (const e of covering) {
+    bytes.set(e.originalBytes, offset);
+    offset += e.originalBytes.length;
+  }
+
+  if (sha256Hex(bytes) !== digest) return { kind: "digest-mismatch" };
+  return { kind: "bytes", bytes };
+}

package/src/vector-cortex/heal/restore-types.ts

@@ -0,0 +1,185 @@
+/**
+ * vector-cortex/heal/restore-types.ts — VC6B exact source restoration contract.
+ *
+ * VC6A optimized WHICH EDGES the closure plan walks. VC6B answers the next
+ * question: when the plan needs a node whose bytes are no longer in the live
+ * window, WHERE do those bytes come from? The answer is deliberately narrow —
+ * an EXACT source, or nothing.
+ *
+ * THE CARDINAL RULE. Restored bytes are ONLY ever read from an exact source:
+ *   1. an `ExactShardV1` whose range and digest both match the request, or
+ *   2. a scan of the `EventV2` occurrence ledger over the requested seq range.
+ * Bytes are NEVER inferred from an embedding, a semantic shard, a RAPTOR
+ * summary, or any other derived/lossy representation. A semantic tier can tell
+ * you what a span was ABOUT; it cannot tell you what the span WAS. Attempting to
+ * "restore" from a derived source would silently fabricate transcript history,
+ * so VC6B has no code path that can do it: `RestoreReader` exposes exactly the
+ * two exact sources and nothing else.
+ *
+ * THE VERIFICATION RULE. Every restored span must hash to the SHA-256 digest the
+ * REQUEST pinned, checked immediately before insertion. A source that matches by
+ * range but not by hash is rejected (`HEAL_RESTORE_DIGEST_MISMATCH`) — never
+ * "close enough". This is what makes a swapped shard file, a truncated read, or a
+ * corrupted ledger record fail loudly instead of poisoning the reconstruction.
+ *
+ * DIGEST PINNING (three fields exist; do not confuse them).
+ *   - `ReconstructionSpan.digest` and `ExactShardV1.digest` are SHA-256 in
+ *     LOWERCASE HEX with NO prefix.
+ *   - `EventV2.bytesDigest` is `sha256:<hex>`, WITH the prefix.
+ * `RestoreSpanRequest.digest` works at SPAN level and uses the FORMER: bare
+ * lowercase hex, matching `ExactShardV1.digest` / `ReconstructionSpan.digest`.
+ * The prefixed `EventV2.bytesDigest` is used ONLY for per-event verification
+ * inside the ledger-scan path and is never the request-level digest. Mixing the
+ * two would make every ledger restoration fail (or, worse, make a prefix-stripped
+ * comparison accidentally succeed against the wrong granularity).
+ *
+ * BOUNDS. A restore request is attacker-shaped input: it names spans and byte
+ * ranges. `RESTORE_LIMIT_SPANS` / `RESTORE_LIMIT_BYTES` bound it BEFORE any
+ * reader is consulted, so an oversized request cannot be used to make the
+ * restorer walk the whole ledger (HEAL-LIMIT-002).
+ *
+ * Pure types + registered conformance IDs: no storage, no console, no network
+ * (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import type { ExactShardV1, ShardRange } from "../shards/types.js";
+import type { EventV2 } from "../ledger/types.js";
+
+/**
+ * One requested span: the node it restores, the byte/seq window that IDENTIFIES
+ * it, and the digest that AUTHENTICATES it.
+ *
+ * `range` is identity — it says which slice of the canonical stream to look for.
+ * `digest` is authentication — it says which bytes are acceptable. Both are
+ * required: identity alone would accept a shard file that was swapped in place,
+ * and a digest alone would require scanning every source.
+ */
+export interface RestoreSpanRequest {
+  /** The closure node these bytes belong to (echoed back in the result). */
+  readonly nodeId: string;
+  /** The span's identity: session + inclusive seq bounds + half-open bytes. */
+  readonly range: ShardRange;
+  /**
+   * SHA-256 of the span's original bytes, LOWERCASE HEX, NO `sha256:` prefix
+   * (matches `ExactShardV1.digest` / `ReconstructionSpan.digest`).
+   */
+  readonly digest: string;
+}
+
+/** A batch of span restorations for one session. */
+export interface RestoreRequestV1 {
+  readonly schema: "restore-request-v1";
+  readonly sessionId: string;
+  /** Requested spans, restored in this order (the result preserves it). */
+  readonly spans: readonly RestoreSpanRequest[];
+}
+
+/**
+ * One successfully restored span. `source` records WHICH exact tier answered so
+ * the triad mode can be derived and so an operator can see whether the shard
+ * index is doing its job or every read is falling through to a ledger scan.
+ */
+export interface RestoreSpanResult {
+  readonly nodeId: string;
+  /** Which exact source produced the bytes (never a derived/semantic tier). */
+  readonly source: "exact-shard" | "ledger-scan";
+  /** The EXACT original bytes, verbatim — invalid UTF-8 included, unnormalized. */
+  readonly bytes: Uint8Array;
+  /** The verified SHA-256 (bare lowercase hex) — equals the request's digest. */
+  readonly digest: string;
+}
+
+/**
+ * The restoration outcome for one request.
+ *
+ * `mode` mirrors TRIAD_RESILIENCE:
+ *   A — every span came from an indexed exact shard (the fast, normal path);
+ *   B — every span was restored, but at least one required a ledger range scan
+ *       (an INDEPENDENT code path: no shard index involved, bytes rebuilt by
+ *       concatenating verified occurrence records);
+ *   C — at least one span could not be restored from ANY exact source. Mode C
+ *       OMITS the span and DISCLOSES the loss (`semanticLossStated`) rather than
+ *       substituting derived text.
+ */
+export interface RestoreResultV1 {
+  readonly schema: "restore-result-v1";
+  readonly sessionId: string;
+  readonly mode: "A" | "B" | "C";
+  /** Restored spans in request order (only digest-verified spans appear here). */
+  readonly restored: readonly RestoreSpanResult[];
+  /** Node ids that could not be restored (identity only — never bytes). */
+  readonly missing: readonly string[];
+  /** Set in mode C: the caller MUST be told the old context is gone. */
+  readonly semanticLossStated: boolean;
+  /** Deduplicated failure codes in deterministic order. */
+  readonly codes: readonly RestoreFailureCode[];
+}
+
+/** VC6B failure codes (registered HEAL codes). */
+export type RestoreFailureCode =
+  /** The request exceeds the span-count or aggregate-byte bound. */
+  | "HEAL_RESTORE_LIMIT"
+  /** A source's bytes do not hash to the pinned digest — nothing is inserted. */
+  | "HEAL_RESTORE_DIGEST_MISMATCH"
+  /** Neither an exact shard nor the ledger covers the requested span. */
+  | "HEAL_RESTORE_SOURCE_MISSING"
+  /** A restored span does not correspond to a requested span/digest. */
+  | "HEAL_RESTORE_RANGE_MISMATCH";
+
+/**
+ * Post-restoration verdict. `verifyRestored` re-derives every digest from the
+ * bytes actually carried in the result, so a result object that was mutated
+ * after `restoreSources` returned still fails before insertion.
+ */
+export type RestoreVerification =
+  | { readonly ok: true }
+  | { readonly ok: false; readonly codes: readonly RestoreFailureCode[] };
+
+/**
+ * The ONLY sources a restoration may read. Deliberately exhaustive: there is no
+ * embedding, semantic shard, or summary field here, so "restore from a derived
+ * source" is not merely forbidden by policy — it is unrepresentable.
+ */
+export interface RestoreReader {
+  /** Indexed exact shards (mode A). */
+  readonly exactShards: readonly ExactShardV1[];
+  /** Raw occurrence records for the ledger range scan (mode B). */
+  readonly ledgerEvents: readonly EventV2[];
+}
+
+/**
+ * Maximum spans in one request. A request naming more spans than this is
+ * rejected outright — the bound exists so a single call cannot be turned into an
+ * unbounded traversal of the ledger.
+ */
+export const RESTORE_LIMIT_SPANS = 64;
+
+/**
+ * Maximum aggregate requested bytes (4 MiB), summed from the REQUEST's ranges
+ * (`byteEnd - byteStart`) so the bound is enforceable without reading anything.
+ */
+export const RESTORE_LIMIT_BYTES = 4 * 1024 * 1024;
+
+/**
+ * Registered VC6B conformance ID range (HEAL-016..030), continuing VC6A's
+ * HEAL-001..015. The acceptance test reads these rows from the v2 manifest and
+ * asserts each returns its manifest `ok`/`code`.
+ */
+export const RESTORE_IDS: readonly string[] = Array.from(
+  { length: 15 },
+  (_v, i) => `HEAL-${String(i + 16).padStart(3, "0")}`,
+);
+
+/** Named VC6B conformance assertions (the sprint's headline rows). */
+export const RESTORE_NAMED_IDS = [
+  "HEAL-SPAN-001",
+  "HEAL-LIMIT-002",
+  "HEAL-DIGEST-003",
+] as const;
+
+/** The two structured events the VC6B reporter emits. */
+export type RestoreEventName =
+  | "vector_cortex_source_restored"
+  | "vector_cortex_restore_digest_rejected";
+
+export type { EventV2, ExactShardV1, ShardRange };

package/src/vector-cortex/heal/restore.ts

@@ -0,0 +1,165 @@
+/**
+ * vector-cortex/heal/restore.ts — VC6B exact source restoration orchestrator.
+ *
+ * Turns a `RestoreRequestV1` into a `RestoreResultV1` by consulting, per span,
+ * the two EXACT sources in `restore-readers.ts` — an indexed exact shard first,
+ * then a ledger range scan — and inserting bytes only after they hash to the
+ * digest the request pinned.
+ *
+ * ORDER OF OPERATIONS IS THE CONTRACT.
+ *
+ *   1. BOUNDS, BEFORE ANY READER TOUCH. The span count and the aggregate
+ *      requested byte span are computed PURELY FROM THE REQUEST (`byteEnd -
+ *      byteStart`, which needs no source at all) and checked first. On breach we
+ *      return immediately, having never read `reader.exactShards` or
+ *      `reader.ledgerEvents`. This is HEAL-LIMIT-002: an oversized request must
+ *      not be able to make the restorer walk the ledger even once, so the check
+ *      cannot be "inside the loop, before the read" — it must be before the loop
+ *      exists. The acceptance corpus proves it by passing EMPTY readers with an
+ *      over-limit request: a reader-touching implementation would still return
+ *      "missing" rather than "limit".
+ *
+ *   2. EXACT SHARD, then LEDGER, then MISSING. Sources are tried strongest-first.
+ *      A digest mismatch anywhere is recorded and the span is NOT restored — the
+ *      restorer never downgrades to "closest available bytes".
+ *
+ *   3. MODE from what actually happened, not from what was attempted. A = every
+ *      span came from a shard; B = all restored, at least one via ledger scan;
+ *      C = something is missing, and mode C STATES its semantic loss rather than
+ *      hiding an incomplete restoration behind a successful-looking result.
+ *
+ * PURE. No storage, no console, no clock, no network — `node:crypto` (a Node
+ * built-in) is the only dependency beyond types (PREVENT-PI-004 / PREVENT-011).
+ * The reporter seam in `restore-emit.ts` is flag-gated; THIS arithmetic is not,
+ * so flag-off is byte-identical.
+ */
+
+import type {
+  RestoreFailureCode,
+  RestoreReader,
+  RestoreRequestV1,
+  RestoreResultV1,
+  RestoreSpanRequest,
+  RestoreSpanResult,
+} from "./restore-types.js";
+import { RESTORE_LIMIT_BYTES, RESTORE_LIMIT_SPANS } from "./restore-types.js";
+import { readExactShard, readLedgerSpan } from "./restore-readers.js";
+
+/**
+ * Fixed code ordering so a result's `codes` array is deterministic regardless of
+ * which span failed first. Deterministic output is what lets the conformance
+ * corpus pin an exact expected value.
+ */
+const CODE_ORDER: readonly RestoreFailureCode[] = [
+  "HEAL_RESTORE_LIMIT",
+  "HEAL_RESTORE_DIGEST_MISMATCH",
+  "HEAL_RESTORE_SOURCE_MISSING",
+  "HEAL_RESTORE_RANGE_MISMATCH",
+] as const;
+
+/** Deduplicate + sort codes into the fixed priority order. */
+export function orderCodes(
+  codes: readonly RestoreFailureCode[],
+): readonly RestoreFailureCode[] {
+  const seen = new Set(codes);
+  return CODE_ORDER.filter((c) => seen.has(c));
+}
+
+/**
+ * Total bytes the request ASKS for, derived from the ranges alone. Negative or
+ * inverted ranges contribute 0 rather than reducing the total — an inverted span
+ * must never be usable to smuggle a large request under the bound.
+ */
+function requestedBytes(spans: readonly RestoreSpanRequest[]): number {
+  let total = 0;
+  for (const s of spans) {
+    const len = s.range.byteEnd - s.range.byteStart;
+    total += len > 0 ? len : 0;
+  }
+  return total;
+}
+
+/** The immediate over-limit result: nothing read, everything missing. */
+function limitExceeded(request: RestoreRequestV1): RestoreResultV1 {
+  return {
+    schema: "restore-result-v1",
+    sessionId: request.sessionId,
+    mode: "C",
+    restored: [],
+    missing: request.spans.map((s) => s.nodeId),
+    semanticLossStated: true,
+    codes: ["HEAL_RESTORE_LIMIT"],
+  };
+}
+
+/**
+ * Restore the exact original bytes for every requested span.
+ *
+ * Never throws: a request naming spans no source covers yields a mode-C result
+ * that discloses the loss, not an exception (PRACTICES: non-fatal — a failed
+ * restoration must degrade the prompt, never break the agent loop).
+ */
+export function restoreSources(
+  request: RestoreRequestV1,
+  reader: RestoreReader,
+): RestoreResultV1 {
+  // (1) Bounds first — computed from the request only. `reader` is not touched
+  // on this path, which is exactly what HEAL-LIMIT-002 asserts.
+  if (
+    request.spans.length > RESTORE_LIMIT_SPANS ||
+    requestedBytes(request.spans) > RESTORE_LIMIT_BYTES
+  ) {
+    return limitExceeded(request);
+  }
+
+  const restored: RestoreSpanResult[] = [];
+  const missing: string[] = [];
+  const codes: RestoreFailureCode[] = [];
+  let usedLedger = false;
+
+  // (2) Per span, strongest source first, in request order.
+  for (const span of request.spans) {
+    const exact = readExactShard(reader.exactShards, span.range, span.digest);
+    if (exact.kind === "bytes") {
+      restored.push({
+        nodeId: span.nodeId,
+        source: "exact-shard",
+        bytes: exact.bytes,
+        digest: span.digest,
+      });
+      continue;
+    }
+    if (exact.kind === "digest-mismatch") codes.push("HEAL_RESTORE_DIGEST_MISMATCH");
+
+    const ledger = readLedgerSpan(reader.ledgerEvents, span.range, span.digest);
+    if (ledger.kind === "bytes") {
+      usedLedger = true;
+      restored.push({
+        nodeId: span.nodeId,
+        source: "ledger-scan",
+        bytes: ledger.bytes,
+        digest: span.digest,
+      });
+      continue;
+    }
+    if (ledger.kind === "digest-mismatch") codes.push("HEAL_RESTORE_DIGEST_MISMATCH");
+
+    // (3) Neither exact source answered. Identity only — never bytes.
+    missing.push(span.nodeId);
+    codes.push("HEAL_RESTORE_SOURCE_MISSING");
+  }
+
+  const complete = missing.length === 0 && restored.length === request.spans.length;
+  const mode: RestoreResultV1["mode"] = !complete ? "C" : usedLedger ? "B" : "A";
+
+  return {
+    schema: "restore-result-v1",
+    sessionId: request.sessionId,
+    mode,
+    restored,
+    missing,
+    // Mode C omits spans, so it MUST disclose that the old context is gone.
+    semanticLossStated: mode === "C",
+    codes: orderCodes(codes),
+  };
+}