pi-mega-compact 0.16.2 → 0.17.0

package/src/vector-cortex/ledger/event-codec.ts

@@ -0,0 +1,79 @@
+/**
+ * vector-cortex/ledger/event-codec.ts — EventV2 byte-authority codec, Mode A
+ * (VC1A, TRIAD_RESILIENCE).
+ *
+ * A retains `originalBytes` + a SHA-256 `bytesDigest` and classifies strict
+ * UTF-8 (success) vs invalid-byte content WITHOUT replacement decoding:
+ * `TextDecoder(..., {fatal:true})` either yields text or throws — invalid input
+ * is represented only as `{valid:false, base64}` and is NEVER lossy-replaced.
+ * `canonicalNfc` is computed ONLY as a derived field for valid UTF-8 and is
+ * NEVER used for identity, digest, or byte reconstruction — `decode` returns the
+ * authoritative `originalBytes` exactly.
+ *
+ * Mode B (`./event-codecB.ts`) is a genuinely independent implementation of the
+ * same byte rule; A and B share no subroutine and must agree byte-for-byte.
+ *
+ * Pure/deterministic — no console, no network, no side effects (PREVENT-PI-004).
+ */
+
+import nodeCrypto from "node:crypto";
+import type { BytesDigest, EventCodec, EventEncodeInput, EventV2 } from "./types.js";
+
+/** Mode-A internal SHA-256 helper (NOT shared with mode B). */
+function digestSha256A(bytes: Uint8Array): BytesDigest {
+  const hex = nodeCrypto.createHash("sha256").update(bytes).digest("hex");
+  return `sha256:${hex}`;
+}
+
+/** Mode-A internal base64 (NOT shared with mode B). */
+function base64EncodeA(bytes: Uint8Array): string {
+  return Buffer.from(bytes).toString("base64");
+}
+
+/**
+ * Strict UTF-8 classification with FATAL decoding (never replaces invalid bytes
+ * with U+FFFD). Returns the decoded text, or `{valid:false, base64}`.
+ */
+export function classifyUtf8(bytes: Uint8Array): { valid: true; text: string } | { valid: false; base64: string } {
+  try {
+    const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+    return { valid: true, text };
+  } catch {
+    return { valid: false, base64: base64EncodeA(bytes) };
+  }
+}
+
+/** Build the Mode-A byte-authority codec. */
+export function createEventCodec(): EventCodec {
+  return {
+    encode(input: EventEncodeInput): EventV2 {
+      const bytes = input.bytes;
+      const bytesDigest = digestSha256A(bytes);
+      const utf8 = classifyUtf8(bytes);
+      const event: EventV2 = {
+        schema: "event-v2",
+        sessionId: input.sessionId,
+        seq: input.seq,
+        eventId: input.eventId,
+        role: input.role,
+        kind: input.kind,
+        originalBytes: bytes,
+        bytesDigest,
+        utf8,
+        occurredAtMs: input.occurredAtMs,
+      };
+      if (input.toolCallId !== undefined) event.toolCallId = input.toolCallId;
+      // canonicalNfc is a DERIVED search key — valid UTF-8 only, never identity.
+      if (utf8.valid) event.canonicalNfc = utf8.text.normalize("NFC");
+      return event;
+    },
+
+    decode(event: EventV2): Uint8Array {
+      return event.originalBytes;
+    },
+
+    classifyUtf8,
+  };
+}
+
+export type { BytesDigest, EventV2 };

package/src/vector-cortex/ledger/event-codecB.ts

@@ -0,0 +1,87 @@
+/**
+ * vector-cortex/ledger/event-codecB.ts — EventV2 byte-authority codec, Mode B
+ * (VC1A, TRIAD_RESILIENCE).
+ *
+ * B is a GENUINELY independent raw-byte record implementation of the SAME byte
+ * rule as Mode A (`./event-codec.ts`). It shares NO subroutine with A: its own
+ * SHA-256 helper, its own base64 helper, its own strict-UTF-8 classifier, and
+ * its own raw record — the lesson from VC0B-I09 (a mode-B tested against code
+ * identical to A is a tautology). Because the digest of given bytes is a pure
+ * function, two independent implementations MUST agree byte-for-byte; the
+ * acceptance test asserts `bytesDigest` parity across the fixture corpus.
+ *
+ * B retains `originalBytes` + `bytesDigest` and classifies strict UTF-8 with no
+ * lossy replacement; `canonicalNfc` is derived for valid UTF-8 but never used
+ * for identity, digest, or reconstruction.
+ *
+ * Pure/deterministic — no console, no network, no side effects (PREVENT-PI-004).
+ */
+
+import nodeCrypto from "node:crypto";
+import type { EventV2 } from "./types.js";
+
+/** Mode-B internal SHA-256 helper (independent of A's — no shared subroutine). */
+function digestSha256B(bytes: Uint8Array): string {
+  const d = nodeCrypto.createHash("sha256");
+  d.update(bytes);
+  const hex = d.digest("hex");
+  return `sha256:${hex}`;
+}
+
+/** Mode-B internal base64 (independent of A's). */
+function base64EncodeB(bytes: Uint8Array): string {
+  let out = "";
+  const buf = Buffer.from(bytes);
+  for (let i = 0; i < buf.length; i += 3) {
+    const b0 = buf[i];
+    const b1 = i + 1 < buf.length ? buf[i + 1] : -1;
+    const b2 = i + 2 < buf.length ? buf[i + 2] : -1;
+    out += CHARS[(b0 >> 2) & 63];
+    out += CHARS[((b0 & 3) << 4) | (b1 < 0 ? 0 : (b1 >> 4) & 15)];
+    if (b1 >= 0) out += CHARS[((b1 & 15) << 2) | (b2 < 0 ? 0 : (b2 >> 6) & 3)];
+    if (b2 >= 0) out += CHARS[b2 & 63];
+  }
+  // Padding (independent reconstruction of standard base64).
+  const pad = (3 - (buf.length % 3)) % 3;
+  for (let i = 0; i < pad; i++) out += "=";
+  return out;
+}
+const CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+/** Mode-B strict UTF-8 classifier (fatal decode; no replacement). */
+function classifyUtf8B(bytes: Uint8Array): { valid: true; text: string } | { valid: false; base64: string } {
+  try {
+    const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+    return { valid: true, text };
+  } catch {
+    return { valid: false, base64: base64EncodeB(bytes) };
+  }
+}
+
+/** Mode-B independent raw byte record of an EventV2 occurrence. */
+export interface RawByteRecord {
+  readonly originalBytes: Uint8Array;
+  readonly bytesDigest: string;
+  readonly utf8: { valid: true; text: string } | { valid: false; base64: string };
+  readonly canonicalNfc?: string;
+}
+
+/** Independently digest + classify raw bytes into a B record. */
+export function recordRawBytesB(bytes: Uint8Array): RawByteRecord {
+  const bytesDigest = digestSha256B(bytes);
+  const utf8 = classifyUtf8B(bytes);
+  const record: RawByteRecord = { originalBytes: bytes, bytesDigest, utf8 };
+  if (utf8.valid) {
+    return { originalBytes: bytes, bytesDigest, utf8, canonicalNfc: utf8.text.normalize("NFC") };
+  }
+  return record;
+}
+
+/**
+ * Independent digest check: recompute the SHA-256 over an EventV2's
+ * `originalBytes` via B's OWN algorithm and compare to the stored digest.
+ * Returns the recomputed digest string (parity-checked by the caller).
+ */
+export function digestCheckB(event: EventV2): string {
+  return digestSha256B(event.originalBytes);
+}

package/src/vector-cortex/ledger/sqlite.ts

@@ -0,0 +1,265 @@
+/**
+ * vector-cortex/ledger/sqlite.ts — occurrence-v2 SQLite store, Mode A (VC1B).
+ *
+ * A self-contained, isolated SQLite store (`node:sqlite` `DatabaseSync`) over its
+ * OWN database file — NOT the host `sqlite.db` — so the neutral v2 ledger is a
+ * true independent authority, not a host table. Holds the canonical occurrence
+ * rows plus per-session monotonic seq and the durable contiguous high-water.
+ *
+ * The defect M2 fixes: the host `raw_transcript` table keys on
+ * `(content_hash, session_id)`, so repeated equal bytes at a later turn are
+ * silently dropped. Here the ledger keys occurrences by `(event_id, digest)`
+ * ONLY — equal bytes at a distinct seq/event-id are preserved as separate
+ * occurrences (M2-DUP-001).
+ *
+ * Append enforces, per occurrence: (1) strictly monotonic per-session seq
+ * (EVT_SEQ_REGRESSION otherwise); (2) a tool RESULT names exactly one earlier
+ * call (EVT_TOOL_CALL_MISSING otherwise); (3) exact `(event_id,digest)`
+ * re-appends are acknowledged idempotently, never duplicated.
+ *
+ * Mode A of the VC1B triad: transactional SQLite append. Mode B (`./spool.ts`)
+ * is an independent fsync spool; mode C leaves the host transcript unchanged.
+ *
+ * PREVENT-002: every query is parameterized. PREVENT-011: no `any`.
+ * PREVENT-PI-004: local filesystem only, no network. No console.log.
+ */
+
+import { DatabaseSync } from "node:sqlite";
+import { existsSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { createHash } from "node:crypto";
+import type { LedgerAppendResult, LedgerOccurrence } from "./store.js";
+
+/** Failure codes (re-export of the store contract union, kept local for rows). */
+export type SqliteAppendCode = "EVT_TOOL_CALL_MISSING" | "EVT_SEQ_REGRESSION";
+
+/** DB row shape for occurrence_v2 (snake_case columns). */
+interface OccurrenceDBRow {
+  session: string;
+  seq: number;
+  event_id: string;
+  digest: string;
+  kind: string;
+  tool_call_id: string | null;
+  source_bytes: Uint8Array | null;
+}
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS occurrence_v2 (
+  session       TEXT    NOT NULL,
+  seq           INTEGER NOT NULL,
+  event_id      TEXT    NOT NULL,
+  digest        TEXT    NOT NULL,
+  kind          TEXT    NOT NULL,
+  tool_call_id  TEXT,
+  source_bytes  BLOB,
+  PRIMARY KEY (session, seq),
+  UNIQUE (session, event_id, digest)
+) STRICT;
+
+CREATE TABLE IF NOT EXISTS ledger_high_water (
+  session TEXT PRIMARY KEY,
+  seq     INTEGER NOT NULL
+) STRICT;
+`;
+
+/** Default digest for a row when the caller omits one (sha256 over bytes). */
+export function ledgerDigest(bytes: Uint8Array): string {
+  const hex = createHash("sha256").update(bytes).digest("hex");
+  return `sha256:${hex}`;
+}
+
+/** Open (or reuse) the isolated occurrence-v2 ledger DB handle. */
+export function openOccurrenceStore(dbPath: string): DatabaseSync {
+  const dir = dirname(dbPath);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const db = new DatabaseSync(dbPath);
+  db.exec("PRAGMA journal_mode = WAL");
+  db.exec("PRAGMA foreign_keys = ON");
+  db.exec(SCHEMA);
+  return db;
+}
+
+function rowToOccurrence(row: OccurrenceDBRow): LedgerOccurrence {
+  return {
+    session: row.session,
+    seq: BigInt(row.seq),
+    eventId: row.event_id,
+    digest: row.digest,
+    kind: row.kind,
+    toolCallId: row.tool_call_id ?? undefined,
+    sourceBytes: row.source_bytes ? Buffer.from(row.source_bytes) : new Uint8Array(0),
+  };
+}
+
+/** Current durable contiguous high-water for a session (0 when none). */
+export function ledgerHighWater(db: DatabaseSync, session: string): bigint {
+  const row = db
+    .prepare(`SELECT seq FROM ledger_high_water WHERE session = @session`)
+    .get({ "@session": session }) as { seq: number } | undefined;
+  return row ? BigInt(row.seq) : 0n;
+}
+
+/** Read all accepted occurrences for a session in ascending (seq,eventId). */
+export function readSessionOccurrences(db: DatabaseSync, session: string): LedgerOccurrence[] {
+  const rows = db
+    .prepare(
+      `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
+       FROM occurrence_v2 WHERE session = @session
+       ORDER BY seq ASC, event_id ASC`,
+    )
+    .all({ "@session": session }) as unknown as OccurrenceDBRow[];
+  return rows.map(rowToOccurrence);
+}
+
+/** Occurrences at or above `fromSeq` (inclusive), ascending. */
+export function readFromSeq(db: DatabaseSync, session: string, fromSeq: bigint): LedgerOccurrence[] {
+  const rows = db
+    .prepare(
+      `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
+       FROM occurrence_v2 WHERE session = @session AND seq >= @from
+       ORDER BY seq ASC, event_id ASC`,
+    )
+    .all({ "@session": session, "@from": Number(fromSeq) }) as unknown as OccurrenceDBRow[];
+  return rows.map(rowToOccurrence);
+}
+
+/** Count accepted occurrences for a session. */
+export function countOccurrences(db: DatabaseSync, session: string): number {
+  const row = db
+    .prepare(`SELECT COUNT(*) AS cnt FROM occurrence_v2 WHERE session = @session`)
+    .get({ "@session": session }) as { cnt: number };
+  return row.cnt;
+}
+
+/** Whether a `(event_id, digest)` pair is already accepted in a session. */
+export function hasOccurrence(db: DatabaseSync, session: string, eventId: string, digest: string): boolean {
+  const row = db
+    .prepare(
+      `SELECT 1 AS hit FROM occurrence_v2
+       WHERE session = @session AND event_id = @event_id AND digest = @digest LIMIT 1`,
+    )
+    .get({ "@session": session, "@event_id": eventId, "@digest": digest }) as { hit: number } | undefined;
+  return row !== undefined;
+}
+
+/** The stored occurrence matching a `(event_id, digest)` pair, or undefined. */
+function findOccurrence(
+  db: DatabaseSync,
+  session: string,
+  eventId: string,
+  digest: string,
+): LedgerOccurrence | undefined {
+  const row = db
+    .prepare(
+      `SELECT session, seq, event_id, digest, kind, tool_call_id, source_bytes
+       FROM occurrence_v2
+       WHERE session = @session AND event_id = @event_id AND digest = @digest LIMIT 1`,
+    )
+    .get({ "@session": session, "@event_id": eventId, "@digest": digest }) as OccurrenceDBRow | undefined;
+  return row ? rowToOccurrence(row) : undefined;
+}
+
+/** Whether a prior call row with the given eventId exists in the session. */
+function hasCallRef(db: DatabaseSync, session: string, eventId: string): boolean {
+  const row = db
+    .prepare(
+      `SELECT 1 AS hit FROM occurrence_v2
+       WHERE session = @session AND event_id = @event_id LIMIT 1`,
+    )
+    .get({ "@session": session, "@event_id": eventId }) as { hit: number } | undefined;
+  return row !== undefined;
+}
+
+/**
+ * Append one occurrence within a transaction. Enforces monotonic seq, tool-call
+ * reference completeness, and `(event_id,digest)` idempotent ack. Returns the
+ * accepted row or a deterministic failure code.
+ */
+export function appendOccurrence(
+  db: DatabaseSync,
+  input: {
+    readonly session: string;
+    readonly seq: bigint;
+    readonly eventId: string;
+    readonly kind: string;
+    readonly toolCallId?: string;
+    readonly sourceBytes: Uint8Array;
+    readonly digest?: string;
+  },
+): LedgerAppendResult {
+  const digest = input.digest ?? ledgerDigest(input.sourceBytes);
+  const occurrence: LedgerOccurrence = {
+    session: input.session,
+    seq: input.seq,
+    eventId: input.eventId,
+    digest,
+    kind: input.kind,
+    toolCallId: input.toolCallId,
+    sourceBytes: input.sourceBytes,
+  };
+
+  // (3) exact (event_id,digest) re-append is acknowledged idempotently, WITHOUT
+  // inserting a new row. Return the EXISTING stored occurrence (its real seq) —
+  // not the caller's input — so a journal record written from this result carries
+  // the existing seq, which always corresponds to a real occurrence_v2 row (Q01:
+  // never a phantom journal row at a seq with no backing occurrence).
+  const existing = findOccurrence(db, input.session, input.eventId, digest);
+  if (existing) {
+    return { ok: true, occurrence: existing };
+  }
+
+  // (1) monotonic seq: must be exactly the next expected seq (highWater+1).
+  const hw = ledgerHighWater(db, input.session);
+  if (input.seq !== hw + 1n) {
+    return { ok: false, code: "EVT_SEQ_REGRESSION", rejected: occurrence };
+  }
+
+  // (2) a tool RESULT names exactly one earlier call in the same session.
+  if (input.toolCallId !== undefined && input.toolCallId !== "") {
+    if (!hasCallRef(db, input.session, input.toolCallId)) {
+      return { ok: false, code: "EVT_TOOL_CALL_MISSING", rejected: occurrence };
+    }
+  }
+
+  // Accept: insert the row and advance the per-session high-water atomically.
+  db.exec("SAVEPOINT mc_ledger");
+  try {
+    db.prepare(
+      `INSERT INTO occurrence_v2 (session, seq, event_id, digest, kind, tool_call_id, source_bytes)
+       VALUES (@session, @seq, @event_id, @digest, @kind, @tool_call_id, @source_bytes)`,
+    ).run({
+      "@session": input.session,
+      "@seq": Number(input.seq),
+      "@event_id": input.eventId,
+      "@digest": digest,
+      "@kind": input.kind,
+      "@tool_call_id": input.toolCallId ?? null,
+      "@source_bytes": Buffer.from(input.sourceBytes),
+    });
+    db.prepare(
+      `INSERT INTO ledger_high_water (session, seq) VALUES (@session, @seq)
+       ON CONFLICT(session) DO UPDATE SET seq = excluded.seq`,
+    ).run({ "@session": input.session, "@seq": Number(input.seq) });
+    db.exec("RELEASE mc_ledger");
+  } catch (e) {
+    db.exec("ROLLBACK TO mc_ledger");
+    db.exec("RELEASE mc_ledger");
+    throw e;
+  }
+  return { ok: true, occurrence };
+}
+
+/** Append a batch; each occurrence reports its own result. */
+export function appendOccurrenceBatch(
+  db: DatabaseSync,
+  inputs: ReadonlyArray<Parameters<typeof appendOccurrence>[1]>,
+): LedgerAppendResult[] {
+  return inputs.map((input) => appendOccurrence(db, input));
+}
+
+/** Diagnostic: total accepted occurrences across all sessions. */
+export function countAllOccurrences(db: DatabaseSync): number {
+  const row = db.prepare(`SELECT COUNT(*) AS cnt FROM occurrence_v2`).get() as { cnt: number };
+  return row.cnt;
+}