pi-mega-compact 0.19.3 → 0.20.1

package/src/vector-cortex/residual/codec.ts

@@ -0,0 +1,292 @@
+/**
+ * vector-cortex/residual/codec.ts — reversible residual payload codec (VC4B).
+ *
+ * Encode: split `EventV2.originalBytes` into 4096-byte blocks (zero-padding only
+ * the final block), map bytes to `x=(byte-127.5)/127.5`, take the orthonormal
+ * DCT-II, quantize to int16 with a per-block float32 scale, reconstruct, and
+ * append a block-scoped EXACT correction stream wherever the reconstruction
+ * differs. Post-decode byte error for an admitted artifact is therefore exactly
+ * ZERO — the digest is verified before admission, never assumed.
+ *
+ * Admission (RESIDUAL_CODEC §admission): residual is admitted only when the FULL
+ * encoded size — header, scales, coefficients, corrections, shard index/length
+ * metadata, all 9 shards and their digests — is `<= floor(0.95 *
+ * exactCompressedSize)` AND the complete decode + digest check succeeds.
+ * Otherwise the caller stores the exact compressed payload (mode B). Coefficient
+ * bytes alone are never compared.
+ *
+ * Emits `vector_cortex_residual_admitted` / `vector_cortex_parity_recovery_failed`
+ * through the flag-gated reporter and exposes AGGREGATE-ONLY residual metrics
+ * (counts/byte totals — never payload).
+ *
+ * Guardrails: local hashing only, no storage, no console, no network
+ * (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { createHash } from "node:crypto";
+import { VC4B_ENABLED } from "../../config/vector-cortex.js";
+import {
+  bytesToSignal,
+  forwardDct,
+  inverseDct,
+  signalToBytes,
+  splitBlocks,
+} from "./dct.js";
+import {
+  applyCorrections,
+  dequantizeBlock,
+  diffBlock,
+  quantizeBlock,
+} from "./quantize.js";
+import { encodeShards, recoverStream, sha256Hex } from "./parity.js";
+import { parseStream, serializeStream } from "./stream.js";
+import {
+  ADMISSION_DENOMINATOR,
+  ADMISSION_NUMERATOR,
+  RESIDUAL_BLOCK_SIZE,
+  RESIDUAL_MAGIC,
+  RS_DATA_SHARDS,
+  RS_PARITY_SHARDS,
+  type BlockCorrectionsV1,
+  type ParityShardV1,
+  type QuantizedBlockV1,
+  type ResidualAccountingV1,
+  type ResidualCodecV1,
+  type ResidualDecodeResult,
+  type ResidualEmitter,
+  type ResidualEncodeResult,
+  type ResidualMetricsV1,
+  type ResidualReporter,
+} from "./types.js";
+
+/** Per-shard persisted metadata: u8 index + u32 LE length + 32-byte digest. */
+const SHARD_METADATA_BYTES = 1 + 4 + 32;
+
+/**
+ * The inclusive admission ceiling `floor(0.95 * exactCompressedSize)`, computed
+ * in integer arithmetic so the boundary is exact (a fractional 0.95 multiply
+ * would make the "one byte above rejects" case depend on float rounding).
+ */
+export function admissionCeiling(exactCompressedSize: number): number {
+  return Math.floor(
+    (exactCompressedSize * ADMISSION_NUMERATOR) / ADMISSION_DENOMINATOR,
+  );
+}
+
+/** Total persisted bytes of the shard set (payload + per-shard metadata). */
+export function shardSetBytes(shards: readonly ParityShardV1[]): number {
+  return shards.reduce(
+    (n, s) => n + s.bytes.length + SHARD_METADATA_BYTES,
+    0,
+  );
+}
+
+/** Build the codec artifact (transform + quantize + exact corrections). */
+export function buildArtifact(
+  payload: Uint8Array,
+): { ok: true; codec: ResidualCodecV1 } | { ok: false; code: "RES_QUANTIZE_RANGE" } {
+  const digest = sha256Hex(payload);
+  const rawBlocks = splitBlocks(payload, RESIDUAL_BLOCK_SIZE);
+  const blocks: QuantizedBlockV1[] = [];
+  const corrections: BlockCorrectionsV1[] = [];
+
+  for (let b = 0; b < rawBlocks.length; b++) {
+    const original = rawBlocks[b]!;
+    const quantized = quantizeBlock(forwardDct(bytesToSignal(original)));
+    if (!quantized.ok) return { ok: false, code: "RES_QUANTIZE_RANGE" };
+    blocks.push(quantized.block);
+    // Reconstruct and diff: any residual byte error becomes an exact correction.
+    const reconstructed = signalToBytes(
+      inverseDct(dequantizeBlock(quantized.block)),
+    );
+    const diff = diffBlock(original, reconstructed);
+    if (diff.length > 0) corrections.push({ blockIndex: b, corrections: diff });
+  }
+
+  return {
+    ok: true,
+    codec: {
+      schema: "residual-codec-v1",
+      header: {
+        magic: RESIDUAL_MAGIC,
+        originalLength: payload.length,
+        payloadDigest: digest,
+        blockSize: RESIDUAL_BLOCK_SIZE,
+        dataShards: RS_DATA_SHARDS,
+        parityShards: RS_PARITY_SHARDS,
+      },
+      blocks,
+      corrections,
+    },
+  };
+}
+
+/**
+ * Encode a payload and decide admission against the competing exact compressed
+ * size. Admission requires BOTH the <=95% byte accounting AND a full decode whose
+ * digest matches the original payload.
+ */
+export function encodeResidual(
+  payload: Uint8Array,
+  exactCompressedSize: number,
+  emit?: ResidualEmitter,
+): ResidualEncodeResult {
+  const built = buildArtifact(payload);
+  if (!built.ok) return { ok: false, code: built.code };
+  const codec = built.codec;
+
+  const stream = serializeStream(codec);
+  const shards = encodeShards(stream);
+  const encodedSize = stream.length + shardSetBytes(shards);
+  const correctionCount = codec.corrections.reduce(
+    (n, b) => n + b.corrections.length,
+    0,
+  );
+  const accounting: ResidualAccountingV1 = {
+    encodedSize,
+    exactCompressedSize,
+    admissionCeiling: admissionCeiling(exactCompressedSize),
+    correctionCount,
+    blockCount: codec.blocks.length,
+  };
+
+  const reporter = createResidualReporter(emit);
+  if (encodedSize > accounting.admissionCeiling) {
+    return { ok: true, admitted: false, code: "RES_NOT_ADMITTED", accounting };
+  }
+
+  // Never admit without proving the full decode round-trips to the exact bytes.
+  const verified = decodeResidual(shards, emit);
+  if (!verified.ok) return { ok: false, code: verified.code };
+  if (sha256Hex(verified.bytes) !== codec.header.payloadDigest) {
+    return { ok: false, code: "RES_PAYLOAD_DIGEST_MISMATCH" };
+  }
+
+  reporter.residualAdmitted({
+    encodedSize,
+    exactCompressedSize,
+    admissionCeiling: accounting.admissionCeiling,
+    blockCount: accounting.blockCount,
+    correctionCount,
+  });
+  return { ok: true, admitted: true, codec, shards, accounting };
+}
+
+/**
+ * Decode from a (possibly partial / partially corrupt) shard set: recover the
+ * protected stream, invert the transform, apply the exact corrections, truncate
+ * to the original length, and verify the payload digest.
+ */
+export function decodeResidual(
+  shards: readonly ParityShardV1[],
+  emit?: ResidualEmitter,
+): ResidualDecodeResult {
+  const reporter = createResidualReporter(emit);
+  const recovered = recoverStream(shards);
+  if (!recovered.ok) {
+    reporter.parityRecoveryFailed({
+      code: recovered.code,
+      shardCount: shards.length,
+    });
+    return { ok: false, code: recovered.code };
+  }
+  const codec = parseStream(recovered.stream);
+  if (!codec) {
+    reporter.parityRecoveryFailed({
+      code: "RES_HEADER_INVALID",
+      shardCount: shards.length,
+    });
+    return { ok: false, code: "RES_HEADER_INVALID" };
+  }
+  return decodeArtifact(codec);
+}
+
+/** Decode a parsed artifact directly (no parity layer). */
+export function decodeArtifact(codec: ResidualCodecV1): ResidualDecodeResult {
+  const blockSize = codec.header.blockSize;
+  const out = new Uint8Array(codec.blocks.length * blockSize);
+  const byIndex = new Map<number, readonly { offset: number; original: number }[]>();
+  for (const b of codec.corrections) byIndex.set(b.blockIndex, b.corrections);
+
+  for (let b = 0; b < codec.blocks.length; b++) {
+    const reconstructed = signalToBytes(
+      inverseDct(dequantizeBlock(codec.blocks[b]!)),
+    );
+    const applied = applyCorrections(reconstructed, byIndex.get(b) ?? []);
+    if (!applied.ok) return { ok: false, code: applied.code };
+    out.set(reconstructed, b * blockSize);
+  }
+
+  const bytes = out.subarray(0, codec.header.originalLength);
+  if (sha256Hex(bytes) !== codec.header.payloadDigest) {
+    return { ok: false, code: "RES_PAYLOAD_DIGEST_MISMATCH" };
+  }
+  // Return an independent copy so the caller cannot alias the working buffer.
+  return { ok: true, bytes: Uint8Array.from(bytes) };
+}
+
+/** SHA-256 of arbitrary bytes (re-exported so callers need one import). */
+export function payloadDigest(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+// ── aggregate-only metrics + reporter ───────────────────────────────────────
+
+/**
+ * Accumulate AGGREGATE-ONLY residual metrics (counts/byte totals). Never
+ * payload, never prompt text: the dashboard reads this shape and nothing else.
+ */
+export function accumulateMetrics(
+  previous: ResidualMetricsV1,
+  result: ResidualEncodeResult,
+): ResidualMetricsV1 {
+  if (!result.ok) {
+    return { ...previous, encodeAttempts: previous.encodeAttempts + 1 };
+  }
+  const base = {
+    ...previous,
+    encodeAttempts: previous.encodeAttempts + 1,
+    encodedByteTotal: previous.encodedByteTotal + result.accounting.encodedSize,
+    exactByteTotal:
+      previous.exactByteTotal + result.accounting.exactCompressedSize,
+  };
+  return result.admitted
+    ? { ...base, admittedCount: base.admittedCount + 1 }
+    : { ...base, rejectedCount: base.rejectedCount + 1 };
+}
+
+/** A zeroed metrics accumulator. */
+export function emptyMetrics(): ResidualMetricsV1 {
+  return {
+    encodeAttempts: 0,
+    admittedCount: 0,
+    rejectedCount: 0,
+    recoveryFailures: 0,
+    encodedByteTotal: 0,
+    exactByteTotal: 0,
+  };
+}
+
+/** Build the flag-gated typed reporter (mirrors the VC4A shard reporter). */
+export function createResidualReporter(emit?: ResidualEmitter): ResidualReporter {
+  const fire = (
+    event: Parameters<ResidualEmitter>[0],
+    fields: Record<string, unknown>,
+  ): void => {
+    if (!VC4B_ENABLED()) return;
+    if (!emit) return;
+    try {
+      emit(event, fields);
+    } catch {
+      /* non-fatal observability — never break the agent loop */
+    }
+  };
+  return {
+    residualAdmitted(fields) {
+      fire("vector_cortex_residual_admitted", fields);
+    },
+    parityRecoveryFailed(fields) {
+      fire("vector_cortex_parity_recovery_failed", fields);
+    },
+  };
+}

package/src/vector-cortex/residual/dct.ts

@@ -0,0 +1,166 @@
+/**
+ * vector-cortex/residual/dct.ts — orthonormal DCT-II basis V1 (VC4B).
+ *
+ * Basis V1 is the orthonormal DCT-II matrix generated ANALYTICALLY and never
+ * learned or stored (RESIDUAL_CODEC §byte scope):
+ *
+ *   alpha(0) = sqrt(1/n),  alpha(k>0) = sqrt(2/n)
+ *   C[k][i]  = alpha(k) * cos(pi * (2i + 1) * k / (2n))
+ *
+ * Forward coefficients are the dot product with each basis row (the orthonormal
+ * least-squares solve); the inverse is the transpose applied to the coefficient
+ * vector. Byte mapping is `x = (byte - 127.5) / 127.5`; the inverse maps back
+ * with round-to-nearest-even and clamps to 0..255.
+ *
+ * The full 4096x4096 matrix is ~134 MB of float64, so it is NEVER materialized.
+ * Instead the cosine argument `pi*(2i+1)*k/(2n)` is reduced modulo `2n` against
+ * a precomputed half-period table of length `2n` — this is EXACTLY the same set
+ * of cosine values the matrix would hold (cos is sampled only at the `2n`
+ * distinct arguments `pi*j/(2n)`, j = 0..2n-1), so the transform is bit-stable
+ * and table-generation is O(n) rather than O(n^2).
+ *
+ * Pure numeric transform: no storage, no console, no network (PREVENT-PI-004 /
+ * PREVENT-011).
+ */
+
+import { RESIDUAL_BLOCK_SIZE } from "./types.js";
+
+/** Byte-to-signal mapping midpoint / half-range (RESIDUAL_CODEC §transform). */
+const BYTE_MIDPOINT = 127.5;
+
+/**
+ * Cosine table cache keyed by block length. `table[j] = cos(pi * j / (2n))` for
+ * j = 0..2n-1; every DCT argument `pi*(2i+1)*k/(2n)` reduces to one of these
+ * entries (with a sign) because cos has period `2*pi` = index period `4n`.
+ */
+const cosTables = new Map<number, Float64Array>();
+
+/** Build (or fetch) the `4n`-entry cosine table for block length `n`. */
+function cosTable(n: number): Float64Array {
+  const cached = cosTables.get(n);
+  if (cached) return cached;
+  const period = 4 * n;
+  const table = new Float64Array(period);
+  for (let j = 0; j < period; j++) {
+    table[j] = Math.cos((Math.PI * j) / (2 * n));
+  }
+  cosTables.set(n, table);
+  return table;
+}
+
+/** Orthonormal DCT-II row scale: alpha(0)=sqrt(1/n), else sqrt(2/n). */
+export function alpha(k: number, n: number): number {
+  return k === 0 ? Math.sqrt(1 / n) : Math.sqrt(2 / n);
+}
+
+/**
+ * Map a padded byte block to the signal domain: `x = (byte - 127.5) / 127.5`.
+ * The input must be exactly `n` bytes (the caller zero-pads the final block).
+ */
+export function bytesToSignal(block: Uint8Array): Float64Array {
+  const out = new Float64Array(block.length);
+  for (let i = 0; i < block.length; i++) {
+    out[i] = (block[i]! - BYTE_MIDPOINT) / BYTE_MIDPOINT;
+  }
+  return out;
+}
+
+/**
+ * Map a reconstructed signal back to bytes: invert the affine map, round to
+ * nearest with ties-to-even, and clamp to 0..255 (RESIDUAL_CODEC §transform).
+ */
+export function signalToBytes(signal: Float64Array): Uint8Array {
+  const out = new Uint8Array(signal.length);
+  for (let i = 0; i < signal.length; i++) {
+    const v = signal[i]! * BYTE_MIDPOINT + BYTE_MIDPOINT;
+    const r = roundHalfToEven(v);
+    out[i] = r < 0 ? 0 : r > 255 ? 255 : r;
+  }
+  return out;
+}
+
+/**
+ * Round to nearest, ties to even (banker's rounding). `Math.round` rounds ties
+ * toward +Infinity, which is NOT the rule RESIDUAL_CODEC mandates.
+ */
+export function roundHalfToEven(v: number): number {
+  const floor = Math.floor(v);
+  const diff = v - floor;
+  if (diff > 0.5) return floor + 1;
+  if (diff < 0.5) return floor;
+  // Exact tie: pick the even neighbour.
+  return floor % 2 === 0 ? floor : floor + 1;
+}
+
+/**
+ * Forward orthonormal DCT-II over one block of length `n` (default 4096).
+ * `coefficients[k] = alpha(k) * sum_i x[i] * cos(pi*(2i+1)*k/(2n))`, emitted in
+ * ascending frequency order `k = 0..n-1` (the exact RESIDUAL_CODEC coefficient
+ * order).
+ */
+export function forwardDct(signal: Float64Array): Float64Array {
+  const n = signal.length;
+  const table = cosTable(n);
+  const period = 4 * n;
+  const out = new Float64Array(n);
+  for (let k = 0; k < n; k++) {
+    let sum = 0;
+    // Argument index advances by 2k each step, starting at k (i=0 gives
+    // (2*0+1)*k = k). Reduce modulo the 4n-entry period.
+    let idx = k % period;
+    const step = (2 * k) % period;
+    for (let i = 0; i < n; i++) {
+      sum += signal[i]! * table[idx]!;
+      idx += step;
+      if (idx >= period) idx -= period;
+    }
+    out[k] = alpha(k, n) * sum;
+  }
+  return out;
+}
+
+/**
+ * Inverse orthonormal DCT (the transpose of the forward matrix):
+ * `x[i] = sum_k alpha(k) * c[k] * cos(pi*(2i+1)*k/(2n))`.
+ */
+export function inverseDct(coefficients: Float64Array): Float64Array {
+  const n = coefficients.length;
+  const table = cosTable(n);
+  const period = 4 * n;
+  // Pre-scale each coefficient by its row alpha so the inner loop is a plain
+  // dot product (identical arithmetic to scaling inside the loop for k, and it
+  // keeps the per-i accumulation order stable).
+  const scaled = new Float64Array(n);
+  for (let k = 0; k < n; k++) scaled[k] = alpha(k, n) * coefficients[k]!;
+  const out = new Float64Array(n);
+  for (let i = 0; i < n; i++) {
+    let sum = 0;
+    const step = (2 * i + 1) % period;
+    let idx = 0;
+    for (let k = 0; k < n; k++) {
+      sum += scaled[k]! * table[idx]!;
+      idx += step;
+      if (idx >= period) idx -= period;
+    }
+    out[i] = sum;
+  }
+  return out;
+}
+
+/**
+ * Split a payload into fixed 4096-byte blocks, zero-padding ONLY the final
+ * block. The original length is retained by the caller (the header) so the
+ * padding is dropped on decode.
+ */
+export function splitBlocks(
+  payload: Uint8Array,
+  blockSize: number = RESIDUAL_BLOCK_SIZE,
+): Uint8Array[] {
+  const blocks: Uint8Array[] = [];
+  for (let off = 0; off < payload.length; off += blockSize) {
+    const block = new Uint8Array(blockSize);
+    block.set(payload.subarray(off, Math.min(off + blockSize, payload.length)));
+    blocks.push(block);
+  }
+  return blocks;
+}

package/src/vector-cortex/residual/fixture-payload.ts

@@ -0,0 +1,90 @@
+/**
+ * vector-cortex/residual/fixture-payload.ts — deterministic payload
+ * materialization for the VC4B conformance corpus.
+ *
+ * The residual fixtures describe their payloads GENERATIVELY (`kind` + `length`
+ * + `seed`) rather than embedding megabytes of base64, so this module is the
+ * single normative generator both the fixture producer and the acceptance test
+ * agree on. Every generator is pure and deterministic — the same descriptor
+ * always yields byte-identical output, which is what makes the committed
+ * fixtures meaningful.
+ *
+ * Pure byte generation: no storage, no console, no network (PREVENT-PI-004 /
+ * PREVENT-011).
+ */
+
+/** A generative payload descriptor as carried by a residual fixture. */
+export interface PayloadDescriptor {
+  readonly kind: string;
+  readonly length?: number;
+  readonly seed?: number;
+  readonly value?: number;
+  readonly outlierOffset?: number;
+  readonly bytesBase64?: string;
+}
+
+/** Linear congruential generator (numerical-recipes constants), deterministic. */
+function lcgBytes(length: number, seed: number): Uint8Array {
+  const out = new Uint8Array(length);
+  let state = seed >>> 0;
+  for (let i = 0; i < length; i++) {
+    state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
+    out[i] = (state >>> 24) & 0xff;
+  }
+  return out;
+}
+
+/**
+ * Materialize a fixture payload descriptor into exact bytes.
+ *
+ *   empty        zero bytes
+ *   zeros        `length` 0x00 bytes
+ *   constant     `length` copies of `value`
+ *   sequence     `i % 256`
+ *   lcg          deterministic pseudorandom stream from `seed`
+ *   text         repeating printable ASCII (valid UTF-8)
+ *   invalid-utf8 a stream containing lone continuation/overlong bytes
+ *   dc-outlier   all 0xff except one 0x00 at `outlierOffset` (forces the exact
+ *                correction stream: a DC-dominant block whose coarse scale
+ *                cannot reproduce the outlier's neighbourhood)
+ *   alternating  0x00/0xff alternation (maximum Nyquist energy)
+ */
+export function materializePayload(d: PayloadDescriptor): Uint8Array {
+  const length = d.length ?? 0;
+  switch (d.kind) {
+    case "empty":
+      return new Uint8Array(0);
+    case "zeros":
+      return new Uint8Array(length);
+    case "constant":
+      return new Uint8Array(length).fill((d.value ?? 0) & 0xff);
+    case "sequence":
+      return Uint8Array.from({ length }, (_v, i) => i % 256);
+    case "lcg":
+      return lcgBytes(length, d.seed ?? 1);
+    case "text": {
+      const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789 ";
+      return Uint8Array.from({ length }, (_v, i) =>
+        alphabet.charCodeAt(i % alphabet.length),
+      );
+    }
+    case "invalid-utf8": {
+      // Deliberately invalid: lone 0x80 continuation, 0xff (never valid), and
+      // the overlong 0xc0 0xaf encoding of "/". Never normalized by the codec.
+      const pattern = [0xff, 0x80, 0xc0, 0xaf, 0x00, 0xfe];
+      return Uint8Array.from({ length }, (_v, i) => pattern[i % pattern.length]!);
+    }
+    case "dc-outlier": {
+      const out = new Uint8Array(length).fill(255);
+      const at = d.outlierOffset ?? 0;
+      if (at < length) out[at] = 0;
+      return out;
+    }
+    case "alternating":
+      return Uint8Array.from({ length }, (_v, i) => (i % 2 === 0 ? 0 : 255));
+    case "literal":
+      return new Uint8Array(Buffer.from(d.bytesBase64 ?? "", "base64"));
+    default:
+      throw new Error(`unknown residual payload kind: ${d.kind}`);
+  }
+}