pi-mega-compact 0.19.3 → 0.20.1

package/src/vector-cortex/residual/stream.ts

@@ -0,0 +1,135 @@
+/**
+ * vector-cortex/residual/stream.ts — canonical PROTECTED STREAM serialization
+ * for the VC4B residual codec.
+ *
+ * The protected stream is `header + all block scales/coefficient arrays +
+ * corrections` (RESIDUAL_CODEC §erasure parity), serialized in exactly this
+ * canonical order so encode and decode are byte-symmetric:
+ *
+ *   header (46 bytes)  magic `VCR1` | u32 LE originalLength | 32-byte SHA-256
+ *                      | u16 LE blockSize | u16 LE k | u16 LE m
+ *   blocks             for each block in ascending index:
+ *                        float32 LE scale | 4096 * int16 LE coefficient
+ *   corrections        varint blockCount, then per non-empty block
+ *                        u32 LE blockIndex | varint count
+ *                        | count * (u16 LE offset, u8 original)
+ *
+ * Pure serialization: no storage, no console, no network (PREVENT-PI-004 /
+ * PREVENT-011).
+ */
+
+import {
+  parseCorrections,
+  serializeCorrections,
+} from "./quantize.js";
+import {
+  RESIDUAL_BLOCK_SIZE,
+  RESIDUAL_HEADER_BYTES,
+  RESIDUAL_MAGIC,
+  RS_DATA_SHARDS,
+  RS_PARITY_SHARDS,
+  type QuantizedBlockV1,
+  type ResidualCodecV1,
+  type ResidualHeaderV1,
+} from "./types.js";
+
+/** Per-block serialized size: float32 scale + n int16 coefficients. */
+export function blockBytes(blockSize: number = RESIDUAL_BLOCK_SIZE): number {
+  return 4 + blockSize * 2;
+}
+
+/** Serialize the canonical 46-byte header. */
+export function serializeHeader(header: ResidualHeaderV1): Uint8Array {
+  const out = new Uint8Array(RESIDUAL_HEADER_BYTES);
+  const view = new DataView(out.buffer);
+  for (let i = 0; i < 4; i++) out[i] = RESIDUAL_MAGIC.charCodeAt(i);
+  view.setUint32(4, header.originalLength, true);
+  const digest = Buffer.from(header.payloadDigest, "hex");
+  out.set(digest.subarray(0, 32), 8);
+  view.setUint16(40, header.blockSize, true);
+  view.setUint16(42, header.dataShards, true);
+  view.setUint16(44, header.parityShards, true);
+  return out;
+}
+
+/** Parse the canonical header. Returns null on magic/geometry mismatch. */
+export function parseHeader(bytes: Uint8Array): ResidualHeaderV1 | null {
+  if (bytes.length < RESIDUAL_HEADER_BYTES) return null;
+  for (let i = 0; i < 4; i++) {
+    if (bytes[i] !== RESIDUAL_MAGIC.charCodeAt(i)) return null;
+  }
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const originalLength = view.getUint32(4, true);
+  const payloadDigest = Buffer.from(bytes.subarray(8, 40)).toString("hex");
+  const blockSize = view.getUint16(40, true);
+  const dataShards = view.getUint16(42, true);
+  const parityShards = view.getUint16(44, true);
+  if (blockSize !== RESIDUAL_BLOCK_SIZE) return null;
+  if (dataShards !== RS_DATA_SHARDS || parityShards !== RS_PARITY_SHARDS) return null;
+  return {
+    magic: RESIDUAL_MAGIC,
+    originalLength,
+    payloadDigest,
+    blockSize: RESIDUAL_BLOCK_SIZE,
+    dataShards: RS_DATA_SHARDS,
+    parityShards: RS_PARITY_SHARDS,
+  };
+}
+
+/** Serialize the full protected stream for one encoded artifact. */
+export function serializeStream(codec: ResidualCodecV1): Uint8Array {
+  const head = serializeHeader(codec.header);
+  const perBlock = blockBytes(codec.header.blockSize);
+  const corrections = serializeCorrections(codec.corrections);
+  const out = new Uint8Array(
+    head.length + codec.blocks.length * perBlock + corrections.length,
+  );
+  out.set(head, 0);
+  const view = new DataView(out.buffer);
+  let pos = head.length;
+  for (const block of codec.blocks) {
+    view.setFloat32(pos, block.scale, true);
+    pos += 4;
+    for (let i = 0; i < block.coefficients.length; i++) {
+      view.setInt16(pos, block.coefficients[i]!, true);
+      pos += 2;
+    }
+  }
+  out.set(corrections, pos);
+  return out;
+}
+
+/** Parse a protected stream back into the codec artifact. Null on malformed. */
+export function parseStream(stream: Uint8Array): ResidualCodecV1 | null {
+  const header = parseHeader(stream);
+  if (!header) return null;
+  const perBlock = blockBytes(header.blockSize);
+  const blockCount = Math.ceil(header.originalLength / header.blockSize);
+  const bodyEnd = RESIDUAL_HEADER_BYTES + blockCount * perBlock;
+  if (stream.length < bodyEnd) return null;
+  const view = new DataView(stream.buffer, stream.byteOffset, stream.byteLength);
+  const blocks: QuantizedBlockV1[] = [];
+  let pos = RESIDUAL_HEADER_BYTES;
+  for (let b = 0; b < blockCount; b++) {
+    const scale = view.getFloat32(pos, true);
+    pos += 4;
+    const coefficients = new Int16Array(header.blockSize);
+    for (let i = 0; i < header.blockSize; i++) {
+      coefficients[i] = view.getInt16(pos, true);
+      pos += 2;
+    }
+    blocks.push({ scale, coefficients });
+  }
+  const parsed = parseCorrections(stream, pos);
+  if (!parsed) return null;
+  // Every correction must name a block that exists.
+  for (const block of parsed.blocks) {
+    if (block.blockIndex >= blockCount) return null;
+  }
+  return {
+    schema: "residual-codec-v1",
+    header,
+    blocks,
+    corrections: parsed.blocks,
+  };
+}

package/src/vector-cortex/residual/types.ts

@@ -0,0 +1,236 @@
+/**
+ * vector-cortex/residual/types.ts — reversible residual payload codec (VC4B).
+ *
+ * Owns `ResidualCodecV1` / `ParityShardV1` — the contract of the sprint failure
+ * triad:
+ *
+ *   A = admitted residual (DCT + int16 + exact corrections) + RS(9,6) parity;
+ *   B = exact compressed bytes (forced when the >95% accounting rejects A);
+ *   C = ledger bytes (forced when A/B decode fails).
+ *
+ * Semantic vectors NEVER claim to recover exact text. Exact bytes come only from
+ * exact payload shards (VC4A `ExactShardV1`) or this REVERSIBLE codec: a block
+ * DCT-II analysis, int16 quantization, and a block-scoped exact correction
+ * stream that makes post-decode byte error exactly zero for admitted artifacts.
+ * Numeric erasure parity protects the codec bytes; it is not a substitute for
+ * the exact payload (RESIDUAL_CODEC.md).
+ *
+ * Consumes only reviewer-accepted predecessor contracts (VC1A EventV2 bytes via
+ * VC4A shard ranges) and the common contracts. Pure types + registered
+ * conformance IDs: no storage, no console, no network (PREVENT-PI-004 /
+ * PREVENT-011).
+ */
+
+/** Fixed transform block length (RESIDUAL_CODEC §byte scope: 4096). */
+export const RESIDUAL_BLOCK_SIZE = 4096;
+
+/** Reed–Solomon shard geometry: k=6 data shards, m=3 parity shards. */
+export const RS_DATA_SHARDS = 6;
+export const RS_PARITY_SHARDS = 3;
+export const RS_TOTAL_SHARDS = RS_DATA_SHARDS + RS_PARITY_SHARDS;
+
+/** GF(2^8) primitive polynomial for the parity field (0x11d). */
+export const GF_PRIMITIVE_POLYNOMIAL = 0x11d;
+
+/** Header magic bytes `VCR1`. */
+export const RESIDUAL_MAGIC = "VCR1";
+
+/**
+ * Admission ratio: residual is admitted only when its FULL encoded size is at
+ * most `floor(0.95 * exactCompressedSize)` (RESIDUAL_CODEC §admission). The
+ * accounting counts every persisted byte — header, scales, coefficients,
+ * corrections, shard metadata, all 9 shards, and digests.
+ */
+export const ADMISSION_NUMERATOR = 95;
+export const ADMISSION_DENOMINATOR = 100;
+
+/**
+ * Canonical header of an encoded residual artifact. Serialized (little-endian)
+ * as: magic `VCR1` (4 bytes), u32 original length, 32-byte SHA-256 of the
+ * ORIGINAL payload, u16 block size, u16 data shard count `k`, u16 parity count
+ * `m` — 46 bytes total.
+ */
+export interface ResidualHeaderV1 {
+  readonly magic: typeof RESIDUAL_MAGIC;
+  /** Original (pre-padding) payload length in bytes. */
+  readonly originalLength: number;
+  /** SHA-256 of the original payload bytes, lowercase hex (64 chars). */
+  readonly payloadDigest: string;
+  readonly blockSize: typeof RESIDUAL_BLOCK_SIZE;
+  readonly dataShards: typeof RS_DATA_SHARDS;
+  readonly parityShards: typeof RS_PARITY_SHARDS;
+}
+
+/** Serialized header byte length: 4 + 4 + 32 + 2 + 2 + 2. */
+export const RESIDUAL_HEADER_BYTES = 46;
+
+/**
+ * One transform block's quantized coefficients. `scale` is the per-block float32
+ * scale `max(abs(c))/32767` (exactly 0 for an all-zero block); `coefficients`
+ * are the int16 quantized DCT-II coefficients in ascending frequency order.
+ */
+export interface QuantizedBlockV1 {
+  /** Per-block float32 LE scale. Zero for an all-zero coefficient block. */
+  readonly scale: number;
+  /** Int16 coefficients, length exactly `RESIDUAL_BLOCK_SIZE`. */
+  readonly coefficients: Int16Array;
+}
+
+/**
+ * One exact byte correction inside a block: the reconstruction differed from
+ * the original at `offset`, and `original` is the authoritative byte.
+ */
+export interface CorrectionV1 {
+  /** Offset within the block, 0..4095. */
+  readonly offset: number;
+  /** The original (authoritative) byte value, 0..255. */
+  readonly original: number;
+}
+
+/**
+ * The block-scoped exact correction stream. Blocks appear in ascending
+ * `blockIndex`; each block's corrections are sorted by ascending offset with no
+ * duplicate offset. Omitted blocks have an implicit count of zero.
+ */
+export interface BlockCorrectionsV1 {
+  readonly blockIndex: number;
+  readonly corrections: readonly CorrectionV1[];
+}
+
+/**
+ * A fully encoded residual artifact before shard splitting: the header plus the
+ * per-block quantization and the exact correction stream. Serializing this in
+ * canonical order produces the PROTECTED STREAM that the parity layer splits.
+ */
+export interface ResidualCodecV1 {
+  readonly schema: "residual-codec-v1";
+  readonly header: ResidualHeaderV1;
+  readonly blocks: readonly QuantizedBlockV1[];
+  readonly corrections: readonly BlockCorrectionsV1[];
+}
+
+/**
+ * One Reed–Solomon shard over the protected stream. Shards 0..5 are the
+ * systematic data shards (the protected stream split into `k=6` equal,
+ * zero-padded pieces); shards 6..8 are the parity shards, in that order. Every
+ * shard carries its own SHA-256 so an unknown corruption is DETECTED (and can
+ * then be marked as a known erasure) even though it is never blindly corrected.
+ */
+export interface ParityShardV1 {
+  readonly schema: "parity-shard-v1";
+  /** Shard index 0..8 (0..5 data, 6..8 parity). */
+  readonly index: number;
+  readonly kind: "data" | "parity";
+  /** Shard payload bytes; every shard has the same length. */
+  readonly bytes: Uint8Array;
+  /** SHA-256 of `bytes`, lowercase hex. */
+  readonly digest: string;
+  /** Length of the UNPADDED protected stream (needed to truncate on recovery). */
+  readonly streamLength: number;
+}
+
+/** Byte accounting handed forward to VC4C (spec §next handoff). */
+export interface ResidualAccountingV1 {
+  /** Every persisted residual byte: header + scales + coefficients + corrections + shards + digests. */
+  readonly encodedSize: number;
+  /** Size of the competing exact compressed representation. */
+  readonly exactCompressedSize: number;
+  /** `floor(0.95 * exactCompressedSize)` — the inclusive admission ceiling. */
+  readonly admissionCeiling: number;
+  /** Number of correction entries across every block (density numerator). */
+  readonly correctionCount: number;
+  /** Number of transform blocks. */
+  readonly blockCount: number;
+}
+
+/** Failure codes the residual codec / parity layer can return. */
+export type ResidualFailureCode =
+  | "RES_QUANTIZE_RANGE"
+  | "RES_TOO_MANY_ERASURES"
+  | "RES_DUPLICATE_SHARD_INDEX"
+  | "RES_SHARD_DIGEST_MISMATCH"
+  | "RES_SHARD_LENGTH_MISMATCH"
+  | "RES_SINGULAR_MATRIX"
+  | "RES_PAYLOAD_DIGEST_MISMATCH"
+  | "RES_HEADER_INVALID"
+  | "RES_CORRECTION_DUPLICATE_OFFSET"
+  | "RES_CORRECTION_RANGE"
+  | "RES_NOT_ADMITTED";
+
+/** Encode result: an admitted artifact, or the exact reason it was not admitted. */
+export type ResidualEncodeResult =
+  | {
+      ok: true;
+      /** Mode A: the residual was admitted. */
+      admitted: true;
+      codec: ResidualCodecV1;
+      shards: readonly ParityShardV1[];
+      accounting: ResidualAccountingV1;
+    }
+  | {
+      ok: true;
+      /** Mode B: encode succeeded but accounting rejected it; store exact bytes. */
+      admitted: false;
+      code: "RES_NOT_ADMITTED";
+      accounting: ResidualAccountingV1;
+    }
+  | { ok: false; code: ResidualFailureCode };
+
+/** Decode result: the exact original bytes, or the exact failure code. */
+export type ResidualDecodeResult =
+  | { ok: true; bytes: Uint8Array }
+  | { ok: false; code: ResidualFailureCode };
+
+/** Parity reconstruction result over a (possibly partial) shard set. */
+export type ParityRecoveryResult =
+  | { ok: true; stream: Uint8Array; recoveredIndices: readonly number[] }
+  | { ok: false; code: ResidualFailureCode };
+
+/** The two structured events the VC4B reporter emits. */
+export type ResidualEventName =
+  | "vector_cortex_residual_admitted"
+  | "vector_cortex_parity_recovery_failed";
+
+/** Injected emit callback — same (event, fields) shape as the other VC seams. */
+export type ResidualEmitter = (
+  event: ResidualEventName,
+  fields: Record<string, unknown>,
+) => void;
+
+/** Typed, best-effort reporter bound to the two residual event names. */
+export interface ResidualReporter {
+  readonly residualAdmitted: (fields: Record<string, unknown>) => void;
+  readonly parityRecoveryFailed: (fields: Record<string, unknown>) => void;
+}
+
+/**
+ * Aggregate-only residual metrics exposed to the dashboard. NEVER payload:
+ * counts, byte totals and ratios only (SECURITY_PRIVACY — the exact ledger is
+ * not training data and residual payloads are never rendered).
+ */
+export interface ResidualMetricsV1 {
+  readonly encodeAttempts: number;
+  readonly admittedCount: number;
+  readonly rejectedCount: number;
+  readonly recoveryFailures: number;
+  readonly encodedByteTotal: number;
+  readonly exactByteTotal: number;
+}
+
+/**
+ * Registered RES conformance ID range (RES-001..050). The acceptance test reads
+ * these rows from the v2 manifest and asserts each returns its manifest
+ * `ok`/`code`. The three named assertions (RES-DCT-001 / RES-RS-002 /
+ * RES-ADMIT-003) live alongside them.
+ */
+export const RES_IDS: readonly string[] = Array.from(
+  { length: 50 },
+  (_v, i) => `RES-${String(i + 1).padStart(3, "0")}`,
+);
+
+/** Named RES conformance assertions (the sprint's headline rows). */
+export const RES_NAMED_IDS = [
+  "RES-DCT-001",
+  "RES-RS-002",
+  "RES-ADMIT-003",
+] as const;

package/src/vector-cortex/shards/exact.ts

@@ -0,0 +1,141 @@
+/**
+ * vector-cortex/shards/exact.ts — exact tier partition (VC4A).
+ *
+ * Carves the session's PROTECTED spans — every tool call/result pair, every
+ * anchor, every invalid UTF-8 event — into `ExactShardV1` payloads of the
+ * ORIGINAL bytes, verbatim and never normalized (SHD-UTF8-002: invalid bytes
+ * are exact-only and preserved unchanged — the codec's strict-fatal classifier
+ * means an invalid event has no lossy text to re-encode; we copy `originalBytes`
+ * directly). Each protected span is ATOMIC: a tool call/result pair is one
+ * `ProtectedSpan`, so no exact shard EVER splits a pair, even when the pair
+ * straddles the target-size boundary (SHD-PAIR-001).
+ *
+ * The partition walks the protected spans in canonical stream order (ascending
+ * first-seq), accrues each span's bytes, and closes an exact shard the moment
+ * adding the next atomic span would exceed `targetSize`. A single span over the
+ * budget still occupies exactly one shard — the pair case never splits.
+ *
+ * Triad: A = used with the semantic tier; B = the same exact spans preserved
+ * alongside extractive-derived semantic content; C = exact anchors/current
+ * transcript ONLY (this partition alone, with zero semantic shards).
+ *
+ * Pure/deterministic: hashes + array walking only, no storage, no console, no
+ * network (PREVENT-PI-004 / PREVENT-011).
+ */
+import { createHash } from "node:crypto";
+import type { EventV2 } from "../ledger/types.js";
+import { cumulativeOffsets } from "./semantic.js";
+import type { ExactPartitionInput, ExactPartitionResult, ExactShardCase, ExactShardV1, ProtectedSpan } from "./types.js";
+
+/** Deterministic exact-shard digest over the verbatim original bytes. */
+function sealExactDigest(sessionId: string, bytes: Uint8Array): string {
+  const h = createHash("sha256");
+  h.update(`exact-shard-v1|${sessionId}|`);
+  h.update(bytes);
+  return `sha256:${h.digest("hex")}`;
+}
+
+/** Combine per-span cases into one shard case (invalid dominates, then tool-pair). */
+function combineCase(span: ProtectedSpan, acc: ExactShardCase): ExactShardCase {
+  if (span.case === "invalid-utf8" || acc === "invalid-utf8") return "invalid-utf8";
+  if (span.case === "tool-pair" || acc === "tool-pair") return "tool-pair";
+  return "anchor";
+}
+
+/** Concatenate an ordered group of events' original bytes, verbatim. */
+function concatBytes(events: readonly EventV2[]): Uint8Array {
+  let len = 0;
+  for (const e of events) len += e.originalBytes.length;
+  const out = new Uint8Array(len);
+  let off = 0;
+  for (const e of events) {
+    out.set(e.originalBytes, off);
+    off += e.originalBytes.length;
+  }
+  return out;
+}
+
+/**
+ * Partition the exact tier (task 3). Returns `ok:true` with a list of
+ * `ExactShardV1` whose (sorted) ranges are contiguous, non-overlapping and
+ * jointly cover every protected span exactly once, each preserving the original
+ * bytes verbatim. Defensive failures mirror the semantic tier: a bad target size
+ * rejects with `SHD_INVALID_TARGET_SIZE`; a protected span referencing an event
+ * outside the session's stream, or a mixed-session span, rejects with
+ * `SHD_CROSS_SESSION`.
+ */
+export function partitionExact(input: ExactPartitionInput): ExactPartitionResult {
+  const { sessionId, events, protectedSpans, targetSize } = input;
+  if (!Number.isFinite(targetSize) || targetSize <= 0) {
+    return { ok: false, code: "SHD_INVALID_TARGET_SIZE" };
+  }
+  const stream = new Set(events.map((e) => e.seq.toString()));
+  for (const span of protectedSpans) {
+    if (span.events.some((e) => e.sessionId !== sessionId || !stream.has(e.seq.toString()))) {
+      return { ok: false, code: "SHD_CROSS_SESSION" };
+    }
+  }
+  const offsets = cumulativeOffsets(events);
+  if (protectedSpans.length === 0) return { ok: true, shards: [] };
+
+  // Process protected spans in canonical stream order (ascending first-seq;
+  // equal first-seq preserves input order for determinism).
+  const ordered = [...protectedSpans].sort((a, b) => {
+    const as = a.events.length ? a.events[0].seq : 0n;
+    const bs = b.events.length ? b.events[0].seq : 0n;
+    return as < bs ? -1 : as > bs ? 1 : 0;
+  });
+
+  const shards: ExactShardV1[] = [];
+  let batch: ProtectedSpan[] = [];
+  let batchBytes = 0;
+  let batchCase: ExactShardCase = "anchor";
+
+  const spanBytes = (s: ProtectedSpan): number =>
+    s.events.reduce((n, e) => n + e.originalBytes.length, 0);
+
+  const flush = (): void => {
+    if (batch.length === 0) return;
+    const first = batch[0];
+    const last = batch[batch.length - 1];
+    const firstEv = first.events[0];
+    const lastEv = last.events[last.events.length - 1];
+    const start = offsets.of(firstEv.seq);
+    const end = offsets.of(lastEv.seq);
+    const byteStart = start ? start.byteStart : 0;
+    const byteEnd = end ? end.byteEnd : byteStart;
+    const bytes = concatBytes(batch.flatMap((s) => s.events));
+    shards.push({
+      schema: "exact-shard-v1",
+      sessionId,
+      range: {
+        sessionId,
+        seqStart: firstEv.seq,
+        seqEnd: lastEv.seq,
+        byteStart,
+        byteEnd,
+      },
+      kind: "exact",
+      originalBytes: bytes,
+      digest: sealExactDigest(sessionId, bytes),
+      byteCount: batchBytes,
+      case: batchCase,
+    });
+    batch = [];
+    batchBytes = 0;
+    batchCase = "anchor";
+  };
+
+  for (const span of ordered) {
+    const sz = spanBytes(span);
+    // Close at a complete-span boundary when adding this atomic span would
+    // exceed the budget and we already hold at least one complete span.
+    if (batch.length > 0 && batchBytes + sz > targetSize) flush();
+    batch.push(span);
+    batchBytes += sz;
+    batchCase = combineCase(span, batchCase);
+  }
+  flush();
+
+  return { ok: true, shards };
+}