pi-mega-compact 0.17.0 → 0.18.0

package/src/vector-cortex/conformance/runner.ts

@@ -0,0 +1,165 @@
+/**
+ * vector-cortex/conformance/runner.ts — v2 conformance runner (VC1C).
+ *
+ * Dispatches a conformance case STRICTLY by domain/version (no inference, no
+ * partial output on unknown) and compares both the canonical success bytes and
+ * the expected failure code. Owns `DowngradeReport`.
+ *
+ * The runner is triad-shaped (mode A client):
+ *   A = this manifest runner over the real algorithms;
+ *   B = an independent exact fixture reader (invertible digest recheck);
+ *   C = reject any unknown domain/version WITHOUT partial output.
+ *
+ * Downgrade export: `exporter` produces a new legacy copy that never edits
+ * authority data; the resulting `DowngradeReport` is deterministic — a second
+ * run yields a byte-identical report digest (CONF-DOWN-003).
+ *
+ * No network, no side effects on authority (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { CONF_FAIL, type FixtureManifestEntry } from "./manifest.js";
+import {
+  createConformanceReporter,
+  type ConformanceReporter,
+} from "./emit.js";
+
+/** A registered conformance algorithm handler for a (domain, tuple) key. */
+export interface ConformanceHandler {
+  /**
+   * Run the case. `fixture` is the raw parsed fixture object. Returns ok with a
+   * canonical output digest (the success bytes), or a failure code.
+   */
+  run(
+    entry: FixtureManifestEntry,
+    fixture: unknown,
+  ): { ok: true; outputBytes: Uint8Array; outputDigest: string } | { ok: false; code: string };
+}
+
+/**
+ * The deterministic downgrade-export report. `reportDigest` is a SHA-256 over
+ * the canonical JSON of the report body, so a repeated export is byte-identical.
+ */
+export interface DowngradeReport {
+  readonly schema: "downgrade-report-v1";
+  readonly exportedCopyId: string;
+  readonly copiedCount: number;
+  readonly unrepresentableIds: readonly string[];
+  readonly reportDigest: string;
+}
+
+/** A minimal exporter: produces a new legacy copy and its deterministic report. */
+export interface DowngradeExporter {
+  /** Run the export once. Must never edit the authority data it reads. */
+  exportOnce(): DowngradeReport;
+}
+
+/**
+ * Result of dispatching a conformance case: success bytes (mode A/B) or the
+ * exact expected failure code (mode C rejects unknown without partial output).
+ */
+export type CaseResult =
+  | { ok: true; outputDigest: string; algorithm: string }
+  | { ok: false; code: string; algorithm: string };
+
+const UNKNOWN_DOMAIN = "CONF_UNKNOWN_DOMAIN";
+const UNKNOWN_VERSION = "CONF_UNKNOWN_VERSION";
+/**
+ * The runner-pinned expectation mismatch code: a dispatched handler returned an
+ * outcome that contradicts the manifest entry's frozen expectation (the wrong
+ * failure code, or a success/failure parity error). This is the runner's own
+ * cross-check (task 5: "compare both success bytes and expected failure code"),
+ * distinct from the domain's own failure codes.
+ */
+const EXPECTATION_MISMATCH = "CONF_EXPECTATION_MISMATCH";
+
+export { UNKNOWN_DOMAIN, UNKNOWN_VERSION, EXPECTATION_MISMATCH };
+
+/**
+ * Run one conformance case by dispatching strictly on `(domain, algorithmTuple)`.
+ * If no handler is registered for the exact domain/algorithms, returns
+ * `CONF_UNKNOWN_DOMAIN`/`CONF_UNKNOWN_VERSION` WITHOUT partial output.
+ *
+ * After dispatch the runner cross-checks the handler's outcome against the
+ * manifest entry's frozen expectation (task 5):
+ *   - failure expected (`entry.failureCode` set): the handler must return
+ *     `ok:false` with that exact code, else `CONF_EXPECTATION_MISMATCH`;
+ *   - success expected (`entry.expected === "ok"`): the handler must return
+ *     `ok:true`, and when the manifest pins an `expectedOutputDigest` the
+ *     handler's `outputDigest` must match it, else `CONF_DIGEST_DRIFT`.
+ */
+export function runConformanceCase(
+  entry: FixtureManifestEntry,
+  handlers: ReadonlyMap<string, ConformanceHandler>,
+  fixture: unknown,
+  reporter: ConformanceReporter = createConformanceReporter(),
+): CaseResult {
+  const key = `${entry.domain}::${entry.algorithmTuple.join(";")}`;
+  const handler = handlers.get(key);
+  let result: CaseResult;
+  if (!handler) {
+    if (entry.domain === "") {
+      result = { ok: false, code: UNKNOWN_DOMAIN, algorithm: entry.algorithm };
+    } else {
+      result = { ok: false, code: UNKNOWN_VERSION, algorithm: entry.algorithm };
+    }
+  } else {
+    const out = handler.run(entry, fixture);
+    if (out.ok) {
+      // Success expected? The manifest entry pins the expected success bytes.
+      if (entry.expected !== "ok") {
+        result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
+      } else if (
+        entry.expectedOutputDigest !== undefined &&
+        out.outputDigest !== entry.expectedOutputDigest
+      ) {
+        result = { ok: false, code: CONF_FAIL.DIGEST_DRIFT, algorithm: entry.algorithm };
+      } else {
+        result = { ok: true, outputDigest: out.outputDigest, algorithm: entry.algorithm };
+      }
+    } else {
+      // Failure expected? The manifest entry pins the exact failure code.
+      if (entry.failureCode !== undefined && out.code === entry.failureCode) {
+        result = { ok: false, code: out.code, algorithm: entry.algorithm };
+      } else {
+        result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
+      }
+    }
+  }
+  reporter.caseChecked({
+    id: entry.id,
+    domain: entry.domain,
+    algorithm: entry.algorithm,
+    ok: result.ok,
+    ...(result.ok ? { outputDigest: result.outputDigest } : { code: result.code }),
+  });
+  return result;
+}
+
+/**
+ * Build a dispatch key for a (domain, algorithmTuple) pair. Callers register
+ * handlers under this key.
+ */
+export function handlerKey(
+  domain: string,
+  algorithmTuple: readonly string[],
+): string {
+  return `${domain}::${algorithmTuple.join(";")}`;
+}
+
+/**
+ * Run a downgrade export and recompute its report digest deterministically.
+ * A second invocation produces a byte-identical report (CONF-DOWN-003).
+ */
+export function runDowngradeExport(
+  exporter: DowngradeExporter,
+  reporter: ConformanceReporter = createConformanceReporter(),
+): DowngradeReport {
+  const report = exporter.exportOnce();
+  reporter.downgradeWritten({
+    exportedCopyId: report.exportedCopyId,
+    copiedCount: report.copiedCount,
+    unrepresentableIds: report.unrepresentableIds,
+    reportDigest: report.reportDigest,
+  });
+  return report;
+}

package/src/vector-cortex/conformance/triadB-reader.ts

@@ -0,0 +1,110 @@
+/**
+ * conformance/triadB-reader.ts — TRIAD-B independent exact reader (VC1C).
+ *
+ * Deliberately shares NO code and NO cached-seed instance with the runner
+ * modules (l1-minhash-v2.ts / l1-lsh-v2.ts). It re-derives the minhash-v2
+ * signature and 64 bucket keys from first principles, using only the PUBLISHED
+ * (a,b) seed pairs passed in (read by the caller from
+ * conformance/.../minhash/seeds-v2.json — the table authored independently by
+ * scripts/gen-fixtures/minhash.mjs, the same process that produced the frozen
+ * fixture bytes). Because B shares no subroutine with the runner, a systematic
+ * error in the runner's splitmix64 / seed-derivation / signature arithmetic
+ * cannot satisfy both triad A and triad B.
+ *
+ * Pure compute, no deps, no storage, no network (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { createHash } from "node:crypto";
+
+const MASK64 = 0xffffffffffffffffn;
+const P = (1n << 61n) - 1n; // 2^61 - 1
+const SLOTS = 256;
+const U64 = 8;
+const BANDS = 64;
+
+/** A published (a, b) seed pair as decimal strings (exactness beyond 2^53). */
+export interface PublishedSeedPair {
+  readonly a: string;
+  readonly b: string;
+}
+
+/** 64-bit FNV-1a over UTF-8 bytes or a raw byte buffer (own, independent loop). */
+function fnv1a64(input: string | Uint8Array): bigint {
+  let h = 0xcbf29ce484222325n;
+  for (const b of Buffer.from(input)) {
+    h = ((h ^ BigInt(b)) * 0x100000001b3n) & MASK64;
+  }
+  return h;
+}
+
+/**
+ * Re-derive the 2048-byte minhash-v2 signature and (optionally) the 64 LSH
+ * bucket keys for `text` within `session`, from published seed pairs only —
+ * never the runner module.
+ */
+export function independentReaderV2(
+  text: string,
+  session: string,
+  seedPairs: readonly PublishedSeedPair[],
+  withBuckets: boolean,
+): { bytes: Uint8Array; digest: string; buckets: string[] } {
+  // Normalization: ANSI-strip, NFC, case-fold, CRLF->LF, whitespace collapse,
+  // 32K cap — mirrored from the frozen VC1C spec.
+  let norm = (text || "")
+    .replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
+    .normalize("NFC")
+    .toLowerCase()
+    .replace(/\r\n?/g, "\n")
+    .replace(/\s+/g, " ")
+    .trim();
+  if (norm.length > 32768) norm = norm.slice(0, 32768);
+
+  // Five-code-point shingles (deduped, capped at 50_000).
+  const grams: bigint[] = [];
+  if (norm.length > 0) {
+    const cps = Array.from(norm);
+    const seen = new Set<bigint>();
+    const short = cps.length < 5;
+    for (let i = 0; i < (short ? 1 : cps.length - 4); i++) {
+      const h = short ? fnv1a64(norm) : fnv1a64(cps.slice(i, i + 5).join(""));
+      if (!seen.has(h)) {
+        seen.add(h);
+        grams.push(h);
+      }
+      if (!short && seen.size >= 50000) break;
+    }
+  }
+
+  // Signature: slot i = min over shingles of (a_i*x + b_i) mod p (exact BigInt).
+  const sig = new Array<bigint>(SLOTS).fill(P);
+  for (let i = 0; i < SLOTS && grams.length > 0; i++) {
+    const pair = seedPairs[i] as PublishedSeedPair;
+    const a = BigInt(pair.a);
+    const b = BigInt(pair.b);
+    let min = P;
+    for (const x of grams) {
+      const h = (a * x + b) % P;
+      if (h < min) min = h;
+    }
+    sig[i] = min;
+  }
+
+  // Own 256 x u64 little-endian encoder.
+  const buf = Buffer.allocUnsafe(SLOTS * U64);
+  for (let i = 0; i < SLOTS; i++) buf.writeBigUInt64LE(sig[i] as bigint, i * U64);
+  const bytes = new Uint8Array(buf);
+  const digest = createHash("sha256").update(bytes).digest("hex");
+
+  // 64 buckets: FNV-1a over `<session>|v2|` + each 32-byte band (LE hex).
+  const buckets: string[] = [];
+  if (withBuckets) {
+    const prefix = Buffer.from(`${session}|v2|`, "utf8");
+    for (let band = 0; band < BANDS; band++) {
+      const bandBytes = Buffer.from(bytes.slice(band * 32, band * 32 + 32));
+      const out = Buffer.allocUnsafe(U64);
+      out.writeBigUInt64LE(fnv1a64(Buffer.concat([prefix, bandBytes])), 0);
+      buckets.push(`b${band}:${out.toString("hex")}`);
+    }
+  }
+  return { bytes, digest, buckets };
+}

package/src/vector-cortex/encoder/asset.ts

@@ -0,0 +1,155 @@
+/**
+ * vector-cortex/encoder/asset.ts — VC2A asset verification (task 2).
+ *
+ * Verifies a ModelManifestV1 before any allocation: SHA-256 the ONNX and
+ * tokenizer against the manifest digests, require opset 17, batch exactly 1 and
+ * maximum 512 tokens, and confirm the current platform is in the supported
+ * matrix. On ANY of these the caller demotes to mode B (asset-free trigram) —
+ * never a remote fetch (PREVENT-PI-004). A truncated/unreadable asset during
+ * the digest read demotes with ENC_ASSET_UNREADABLE.
+ *
+ * Pi-agnostic. Filesystem reads only, zero network (PREVENT-PI-004).
+ */
+
+import { createHash } from "node:crypto";
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import {
+  ENC_FAIL,
+  ENCODER_BATCH,
+  ENCODER_MAX_TOKENS,
+  ENCODER_OPSET,
+  ENCODER_SUPPORTED_PLATFORMS,
+  type EncoderPlatform,
+  type ModelManifestV1,
+} from "./types.js";
+
+export type AssetVerifyResult =
+  | { ok: true; embeddedBytes: number; maxTokens: number; onnxDigest: string; tokenizerDigest: string }
+  | { ok: false; code: string };
+
+/** True when `p` is a single basename: non-empty, no path separators, no "..",
+ *  no leading dot-segment traversal. Keeps manifest-controlled asset paths
+ *  confined to the asset directory (no path-traversal via join()). */
+function isBasename(p: string): boolean {
+  if (!p || p.length === 0 || p.includes("/") || p.includes("\\")) return false;
+  if (p === "." || p === "..") return false;
+  return true;
+}
+
+/** Digest the on-disk bytes of one asset file; "" on unreadable (truncated). */
+function digestFile(path: string): string | null {
+  try {
+    const buf = readFileSync(path);
+    return createHash("sha256").update(buf).digest("hex");
+  } catch {
+    // Truncated / unreadable during the digest read -> ENC_ASSET_UNREADABLE.
+    return null;
+  }
+}
+
+/**
+ * Detect the current platform (MODEL_ASSET supported matrix). Unrecognized
+ * hosts return null so verification demotes to mode B (unsupported platform).
+ */
+export function detectPlatform(host = process.platform, arch = process.arch): EncoderPlatform | null {
+  if (!host || !arch) return null;
+  // Normalize "win32"/"win32"/"linux"/"darwin" + "x64"/"arm64".
+  const plat = host === "win32" ? "win32" : host === "darwin" ? "darwin" : host === "linux" ? "linux" : "";
+  const a = arch === "x64" ? "x64" : arch === "arm64" ? "arm64" : "";
+  if (!plat || !a) return null;
+  const candidate = `${plat}-${a}` as EncoderPlatform;
+  return (ENCODER_SUPPORTED_PLATFORMS as readonly string[]).includes(candidate)
+    ? candidate
+    : null;
+}
+
+function isManifest(m: unknown): m is ModelManifestV1 {
+  const o = m as ModelManifestV1;
+  return (
+    !!o &&
+    typeof o === "object" &&
+    o.schema === "model-manifest-v1" &&
+    typeof o.opset === "number" &&
+    typeof o.batch === "number" &&
+    typeof o.maxTokens === "number" &&
+    typeof o.platform === "string" &&
+    !!o.onnx &&
+    !!o.tokenizer &&
+    typeof o.onnx.path === "string" &&
+    typeof o.onnx.sha256 === "string" &&
+    typeof o.tokenizer.path === "string" &&
+    typeof o.tokenizer.sha256 === "string"
+  );
+}
+
+/**
+ * Verify the asset manifest + digest + constraints BEFORE allocation.
+ *
+ *   - manifest parse/shape failure      -> ENC_MANIFEST_INVALID -> mode B
+ *   - unsupported platform              -> ENC_PLATFORM_UNSUPPORTED -> mode B
+ *   - opset != 17                       -> ENC_OPSET_INVALID -> mode B
+ *   - batch != 1                        -> ENC_BATCH_INVALID -> mode B
+ *   - maxTokens > 512                   -> ENC_TOKENS_EXCEEDED -> mode B
+ *   - on-disk digest != manifest digest -> ENC_DIGEST_MISMATCH (one-byte mutation)
+ *   - unreadable/truncated file         -> ENC_ASSET_UNREADABLE
+ *
+ * Returns ok only when EVERY constraint passes and both files hash to the
+ * declared digests (the "only batch1/max512 verified assets reach inference"
+ * invariant). The ok result surfaces the verified manifest's `maxTokens`
+ * (<= 512) so the runtime can enforce the per-manifest token capacity at
+ * inference (Q03), not just the global 512 ceiling.
+ */
+export function verifyEncoderAsset(
+  assetDir: string,
+  manifest: unknown,
+  platform: EncoderPlatform | null = detectPlatform(),
+): AssetVerifyResult {
+  if (typeof manifest !== "object" || manifest === null || !isManifest(manifest)) {
+    return { ok: false, code: ENC_FAIL.MANIFEST_INVALID };
+  }
+  if (!platform) return { ok: false, code: ENC_FAIL.PLATFORM_UNSUPPORTED };
+  // The manifest's declared platform must match the runtime host (per-platform
+  // asset pinning): a bundle cross-shipped to the wrong arch is not qualified.
+  if (manifest.platform !== platform) return { ok: false, code: ENC_FAIL.PLATFORM_UNSUPPORTED };
+  if (manifest.opset !== ENCODER_OPSET) return { ok: false, code: ENC_FAIL.OPSET_INVALID };
+  if (manifest.batch !== ENCODER_BATCH) return { ok: false, code: ENC_FAIL.BATCH_INVALID };
+  if (manifest.maxTokens > ENCODER_MAX_TOKENS) return { ok: false, code: ENC_FAIL.TOKENS_EXCEEDED };
+
+  // Constrain asset paths to basenames (no separators / no '..') so a forged
+  // manifest cannot read digests from arbitrary paths off the asset dir.
+  if (!isBasename(manifest.onnx.path) || !isBasename(manifest.tokenizer.path)) {
+    return { ok: false, code: ENC_FAIL.MANIFEST_INVALID };
+  }
+
+  const onnxPath = join(assetDir, manifest.onnx.path);
+  const onnxDigest = digestFile(onnxPath);
+  if (onnxDigest === null) return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
+  if (onnxDigest !== manifest.onnx.sha256) return { ok: false, code: ENC_FAIL.DIGEST_MISMATCH };
+
+  const tokPath = join(assetDir, manifest.tokenizer.path);
+  const tokDigest = digestFile(tokPath);
+  if (tokDigest === null) return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
+  if (tokDigest !== manifest.tokenizer.sha256) return { ok: false, code: ENC_FAIL.DIGEST_MISMATCH };
+
+  let embeddedBytes = 0;
+  try {
+    embeddedBytes = statSync(onnxPath).size + statSync(tokPath).size;
+  } catch {
+    return { ok: false, code: ENC_FAIL.ASSET_UNREADABLE };
+  }
+  return { ok: true, embeddedBytes, maxTokens: manifest.maxTokens, onnxDigest, tokenizerDigest: tokDigest };
+}
+
+/**
+ * Read + shape-check a committed ModelManifestV1 from an asset directory.
+ * Returns the parsed manifest or null when the file is absent/malformed.
+ */
+export function readEncoderManifest(assetDir: string): ModelManifestV1 | null {
+  try {
+    const raw = JSON.parse(readFileSync(join(assetDir, "manifest.json"), "utf8")) as unknown;
+    return isManifest(raw) ? raw : null;
+  } catch {
+    return null;
+  }
+}

package/src/vector-cortex/encoder/emit.ts

@@ -0,0 +1,51 @@
+/**
+ * vector-cortex/encoder/emit.ts — VC2A observability seam.
+ *
+ * Emits the two VC2A structured events, gated on `MEGACOMPACT_VC2A` (mode C
+ * parity: flag OFF => zero emissions). Every event is a JSON line with `ts` +
+ * `event` (ENGINEERING_PRACTICES §8); the emitters are never fatal on consumer
+ * failure (non-fatal observability, never breaks the agent loop).
+ *
+ *   vector_cortex_encoder_asset_verified   — a qualified manifest+digest load
+ *   vector_cortex_encoder_runtime_demoted  — a demotion to mode B or C
+ *
+ * No network, no side effects beyond the supplied emit callback
+ * (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { VC2A_ENABLED } from "../../config/vector-cortex.js";
+
+export type EncoderEmit = (event: string, fields: Record<string, unknown>) => void;
+
+/** The two-event reporter surface consumed by the runtime seams. */
+export interface EncoderReporter {
+  readonly assetVerified: (fields: Record<string, unknown>) => void;
+  readonly runtimeDemoted: (fields: Record<string, unknown>) => void;
+}
+
+/** A flag-gated no-op reporter (zero emissions, default when none injected). */
+export const NOOP_ENCODER_REPORTER: EncoderReporter = {
+  assetVerified: () => {},
+  runtimeDemoted: () => {},
+};
+
+/**
+ * Flag-gated emit: no-op when VC2A is off or no emitter is supplied. The
+ * returned reporter is itself flag-gated (`VC2A_ENABLED`), so wiring it into a
+ * runtime seam yields zero emissions when `MEGACOMPACT_VC2A=0` (byte-identical
+ * to the predecessor).
+ */
+export function createEncoderReporter(emit?: EncoderEmit): EncoderReporter {
+  const fire = (event: string, fields: Record<string, unknown>): void => {
+    if (!VC2A_ENABLED()) return;
+    try {
+      emit?.(event, { ...fields, ts: new Date().toISOString() });
+    } catch {
+      /* non-fatal observability */
+    }
+  };
+  return {
+    assetVerified: (fields) => fire("vector_cortex_encoder_asset_verified", fields),
+    runtimeDemoted: (fields) => fire("vector_cortex_encoder_runtime_demoted", fields),
+  };
+}