pi-daddy 0.27.4 → 0.32.0

package/src/governance/record.ts

@@ -0,0 +1,294 @@
+/**
+ * One record envelope for every append-only store (ADR-0076 PR 3d).
+ *
+ * A line is `{ v, seq, prev, at, kind, id, body, digest }`: `prev` is the SHA-256 of the previous line's
+ * bytes (null for the first), `digest` the SHA-256 of the canonical JSON of the record without `digest`.
+ * Writes go under the existing file lock and read only the tail, so an append is O(tail), not O(file).
+ *
+ * Damage policy (operator decision, 2026-09-21): the reader returns every intact record before the first
+ * damaged line plus one marker naming that line; the writer refuses `LEDGER_DAMAGED` until `repairLedger`
+ * truncates the torn tail with the caller's explicit consent. Governance fails closed (no spawn goes
+ * unrecorded), history stays readable, repair is a deliberate act that shows what it drops.
+ *
+ * Stated limit: the WRITER inspects only the last line, so a record tampered in the middle of the file does not
+ * stop appends; it shows as damage on every READ (report, dashboard, timeline), and a repair there would drop
+ * every intact record after it. That is why repair previews by default and why the preview names records it
+ * would drop that still parse. Whole-file verification is the reader's job, not the writer's.
+ */
+import { createHash } from "node:crypto";
+import { randomUUID } from "node:crypto";
+import { appendFile, mkdir, open, readFile, stat, truncate } from "node:fs/promises";
+import { dirname } from "node:path";
+import { GovernanceRefusal, refusal } from "../kernel/refusals.ts";
+import { withFileLock } from "./file-lock.ts";
+
+export const RECORD_FORMAT = 1 as const;
+export const RECORD_KINDS = [
+  "binding",
+  "capability",
+  "lifecycle",
+  "approval",
+  "lease",
+  "check",
+  "fact",
+  "work",
+  "control",
+  "experiment",
+  "activity",
+  "advice",
+] as const;
+export type RecordKind = (typeof RECORD_KINDS)[number];
+export const LEDGER_DAMAGED = "LEDGER_DAMAGED" as const;
+
+export interface RecordEnvelope<B = unknown> {
+  v: typeof RECORD_FORMAT;
+  seq: number;
+  prev: string | null;
+  at: string;
+  kind: RecordKind;
+  id: string;
+  body: B;
+  /** Present when the record was copied from a pre-format store; the original is left untouched. */
+  imported?: { path: string; line: number };
+  digest: string;
+}
+
+export interface RecordDamage {
+  line: number;
+  reason: string;
+}
+
+export interface ReadRecords<B = unknown> {
+  records: RecordEnvelope<B>[];
+  damage: RecordDamage | null;
+}
+
+/** Deterministic JSON: sorted object keys at every level, arrays in order. No `undefined` survives. */
+export function canonicalJson(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map((v) => canonicalJson(v === undefined ? null : v)).join(",")}]`;
+  const entries = Object.entries(value as Record<string, unknown>)
+    .filter(([, v]) => v !== undefined)
+    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`;
+}
+
+const sha256 = (text: string) => createHash("sha256").update(text, "utf8").digest("hex");
+
+export function recordDigest(record: object): string {
+  const { digest: _drop, ...rest } = record as Record<string, unknown>;
+  return sha256(canonicalJson(rest));
+}
+
+const HEX64 = /^[0-9a-f]{64}$/;
+/** Damage reasons that mean "this is not a record file at all" rather than "a record file with a torn tail". */
+export const PRE_FORMAT_REASONS: ReadonlySet<string> = new Set([
+  "unknown format version",
+  "not an object",
+  "invalid seq",
+  "invalid prev",
+  "invalid timestamp",
+  "unknown kind",
+  "invalid id",
+  "missing body",
+  "invalid digest",
+]);
+
+/** Parse one line; returns a reason string when it is not a well-formed record of this format. */
+function parseLine(line: string): { record: RecordEnvelope } | { reason: string } {
+  let value: unknown;
+  try {
+    value = JSON.parse(line);
+  } catch {
+    return { reason: "unterminated or unparsable line" };
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) return { reason: "not an object" };
+  const r = value as Record<string, unknown>;
+  if (r.v !== RECORD_FORMAT) return { reason: "unknown format version" };
+  if (typeof r.seq !== "number" || !Number.isInteger(r.seq) || r.seq < 1) return { reason: "invalid seq" };
+  if (!(r.prev === null || (typeof r.prev === "string" && HEX64.test(r.prev)))) return { reason: "invalid prev" };
+  if (typeof r.at !== "string" || Number.isNaN(Date.parse(r.at))) return { reason: "invalid timestamp" };
+  if (typeof r.kind !== "string" || !(RECORD_KINDS as readonly string[]).includes(r.kind))
+    return { reason: "unknown kind" };
+  if (typeof r.id !== "string" || r.id.length === 0 || r.id.length > 128) return { reason: "invalid id" };
+  if (!("body" in r)) return { reason: "missing body" };
+  if (typeof r.digest !== "string" || !HEX64.test(r.digest)) return { reason: "invalid digest" };
+  const record = r as unknown as RecordEnvelope;
+  if (recordDigest(record) !== record.digest) return { reason: "digest mismatch" };
+  return { record };
+}
+
+/** Read every intact record before the first damaged line. Never throws on content; damage is data. */
+export function readRecords<B = unknown>(text: string): ReadRecords<B> {
+  const records: RecordEnvelope<B>[] = [];
+  if (text.length === 0) return { records, damage: null };
+  const terminated = text.endsWith("\n");
+  const lines = text.split("\n");
+  if (terminated) lines.pop();
+  let prevLine: string | null = null;
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    const lineNumber = index + 1;
+    const last = index === lines.length - 1;
+    if (last && !terminated)
+      return { records, damage: { line: lineNumber, reason: "unterminated or unparsable line" } };
+    const parsed = parseLine(line);
+    if ("reason" in parsed) return { records, damage: { line: lineNumber, reason: parsed.reason } };
+    const { record } = parsed;
+    if (record.seq !== records.length + 1) return { records, damage: { line: lineNumber, reason: "sequence gap" } };
+    const expectedPrev = prevLine === null ? null : sha256(prevLine);
+    if (record.prev !== expectedPrev) return { records, damage: { line: lineNumber, reason: "prev hash mismatch" } };
+    records.push(record as RecordEnvelope<B>);
+    prevLine = line;
+  }
+  return { records, damage: null };
+}
+
+export async function readRecordsFile<B = unknown>(path: string): Promise<ReadRecords<B>> {
+  try {
+    return readRecords<B>(await readFile(path, "utf8"));
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return { records: [], damage: null };
+    throw error;
+  }
+}
+
+/** The last complete line and its sequence, reading only the file's tail. Damage is reported, not skipped. */
+async function tail(path: string): Promise<{ seq: number; prevLine: string | null } | { damage: RecordDamage }> {
+  let size: number;
+  try {
+    size = (await stat(path)).size;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return { seq: 0, prevLine: null };
+    throw error;
+  }
+  if (size === 0) return { seq: 0, prevLine: null };
+  const window = Math.min(size, 256 * 1024);
+  const handle = await open(path, "r");
+  try {
+    const buffer = Buffer.alloc(window);
+    await handle.read(buffer, 0, window, size - window);
+    const text = buffer.toString("utf8");
+    if (!text.endsWith("\n")) return { damage: { line: -1, reason: "unterminated or unparsable line" } };
+    const body = text.slice(0, -1);
+    const cut = body.lastIndexOf("\n");
+    const lastLine = cut === -1 ? body : body.slice(cut + 1);
+    if (cut === -1 && window < size) {
+      // The last record is larger than the window: read the whole file once rather than call it damage.
+      const whole = (await readFile(path, "utf8")).slice(0, -1);
+      const wholeCut = whole.lastIndexOf("\n");
+      const wholeLast = wholeCut === -1 ? whole : whole.slice(wholeCut + 1);
+      const parsedWhole = parseLine(wholeLast);
+      if ("reason" in parsedWhole) return { damage: { line: -1, reason: parsedWhole.reason } };
+      return { seq: parsedWhole.record.seq, prevLine: wholeLast };
+    }
+    const parsed = parseLine(lastLine);
+    if ("reason" in parsed) return { damage: { line: -1, reason: parsed.reason } };
+    return { seq: parsed.record.seq, prevLine: lastLine };
+  } finally {
+    await handle.close();
+  }
+}
+
+export interface AppendOptions {
+  id?: string;
+  at?: string;
+  imported?: { path: string; line: number };
+}
+
+/** Append one record under the file lock. Refuses LEDGER_DAMAGED when the tail is torn or invalid. */
+export async function appendRecord<B>(
+  path: string,
+  kind: RecordKind,
+  body: B,
+  options: AppendOptions = {},
+): Promise<RecordEnvelope<B>> {
+  await mkdir(dirname(path), { recursive: true });
+  return withFileLock(path, "ledger", async () => {
+    const last = await tail(path);
+    if ("damage" in last) {
+      const where = last.damage.line === -1 ? "its last line" : `line ${last.damage.line}`;
+      const preFormat = PRE_FORMAT_REASONS.has(last.damage.reason);
+      throw new GovernanceRefusal(
+        refusal(
+          LEDGER_DAMAGED,
+          preFormat
+            ? `ledger ${path} predates the record format (${last.damage.reason}); do NOT repair it — run ` +
+                `\`pi-daddy ledger import ${path} <target>\` or point PI_DADDY_LEDGER at the imported project ledger`
+            : `ledger ${path} is damaged at ${where} (${last.damage.reason}); run ` +
+                `\`pi-daddy ledger repair ${path}\` to preview the torn tail, then retry with --yes`,
+          { path, reason: last.damage.reason, preFormat },
+        ),
+      );
+    }
+    const draft: Omit<RecordEnvelope<B>, "digest"> = {
+      v: RECORD_FORMAT,
+      seq: last.seq + 1,
+      prev: last.prevLine === null ? null : sha256(last.prevLine),
+      at: options.at ?? new Date().toISOString(),
+      kind,
+      id: options.id ?? randomUUID(),
+      body,
+      ...(options.imported ? { imported: options.imported } : {}),
+    };
+    // Digest the serialised round-trip, not the in-memory value: a body with `toJSON` (a Date, a class) would
+    // otherwise be written as one thing and digested as another, and the line would read back as damage.
+    const plain = JSON.parse(JSON.stringify(draft)) as Omit<RecordEnvelope<B>, "digest">;
+    const record: RecordEnvelope<B> = { ...plain, digest: recordDigest(plain) };
+    await appendFile(path, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+    return record;
+  });
+}
+
+export interface RepairResult {
+  /** The raw lines that were (or would be) dropped, in file order. */
+  dropped: string[];
+  /** Where the intact prefix ends, 1-based line count. */
+  keptLines: number;
+  /** How many dropped lines still parse as JSON objects: a tampered middle, not a torn tail. */
+  droppedParseable: number;
+  /** True when the file is not a record file at all (a pre-format ledger); nothing is ever truncated then. */
+  preFormat: boolean;
+  /** True when there was no file. */
+  missing: boolean;
+}
+
+/**
+ * Drop everything from the first damaged line on. `apply: false` previews. The repair never rewrites an
+ * intact record and never invents one; it only truncates, and only after the caller decided to.
+ */
+export async function repairLedger(path: string, options: { apply: boolean }): Promise<RepairResult> {
+  return withFileLock(path, "ledger repair", async () => {
+    let text: string;
+    try {
+      text = await readFile(path, "utf8");
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT")
+        return { dropped: [], keptLines: 0, droppedParseable: 0, preFormat: false, missing: true };
+      throw error;
+    }
+    const read = readRecords(text);
+    if (read.damage === null)
+      return { dropped: [], keptLines: read.records.length, droppedParseable: 0, preFormat: false, missing: false };
+    const lines = text.split("\n");
+    if (text.endsWith("\n")) lines.pop();
+    const dropped = lines.slice(read.damage.line - 1);
+    const droppedParseable = dropped.filter((line) => {
+      try {
+        const v: unknown = JSON.parse(line);
+        return !!v && typeof v === "object" && !Array.isArray(v);
+      } catch {
+        return false;
+      }
+    }).length;
+    // A file whose FIRST line is not a record is a pre-format ledger. Repairing it would delete it whole; the
+    // only right action is to import it, so `apply` is refused here regardless of what the caller asked.
+    const preFormat = read.records.length === 0 && PRE_FORMAT_REASONS.has(read.damage.reason);
+    if (options.apply && !preFormat) {
+      const keep = lines.slice(0, read.damage.line - 1);
+      const bytes = Buffer.byteLength(keep.length > 0 ? `${keep.join("\n")}\n` : "", "utf8");
+      await truncate(path, bytes);
+    }
+    return { dropped, keptLines: read.damage.line - 1, droppedParseable, preFormat, missing: false };
+  });
+}

package/src/governance/retention-contract.ts

@@ -0,0 +1,201 @@
+import { Compile } from "typebox/compile";
+import { parseRetentionJson } from "./retention-json.ts";
+import type { ExecutionRetentionManifest } from "./execution-retention.ts";
+
+const string = (maxLength = 512) => ({
+  type: "string",
+  minLength: 1,
+  maxLength,
+  pattern: "^[^\\u0000-\\u001f\\u007f]+$",
+});
+const nullable = (schema: object) => ({ anyOf: [schema, { type: "null" }] });
+const literal = (value: unknown) => ({ const: value });
+const closed = (properties: Record<string, object>) => ({
+  type: "object",
+  properties,
+  required: Object.keys(properties),
+  additionalProperties: false,
+});
+const hash = { type: "string", pattern: "^[a-f0-9]{64}$" };
+const integer = { type: "integer", minimum: 0, maximum: Number.MAX_SAFE_INTEGER };
+const uuid = { type: "string", pattern: "^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$" };
+export const RETENTION_CONTENT_KINDS = [
+  "stdout",
+  "stderr",
+  "paneSnapshot",
+  "checkReceipt",
+  "result",
+  "session",
+] as const;
+export const RETENTION_SCHEMA = freeze({
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "https://pi-daddy.local/contracts/execution-retention/v2/manifest.schema.json",
+  ...closed({
+    version: literal("2.0"),
+    archiveId: uuid,
+    identity: closed({
+      executionId: string(),
+      parentExecutionId: nullable(string()),
+      childId: string(),
+      toolCallId: nullable(string()),
+      executor: { enum: ["process", "herdr", "check"] },
+      taskDigest: nullable(hash),
+      definitionDigest: nullable(hash),
+      configurationDigest: hash,
+      workspaceId: nullable(string()),
+    }),
+    native: closed({
+      pid: nullable({ type: "integer", minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
+      paneId: nullable(string(4096)),
+      agentName: nullable(string(4096)),
+      tabId: nullable(string(4096)),
+      sessionId: nullable(uuid),
+      sessionPath: nullable(string(4096)),
+      branchLeafId: nullable(string(128)),
+    }),
+    nativeSession: closed({
+      source: { enum: [null, "herdr-id", "herdr-path", "pi-session-file", "pi-session-manager"] },
+      status: { enum: ["missing", "verified", "invalid", "changed", "truncated", "unsupported"] },
+      sessionId: nullable(uuid),
+      sessionPath: nullable(string(4096)),
+      parentSessionPath: nullable(string(4096)),
+      branchLeafId: nullable(string(128)),
+      branchState: { enum: ["unknown", "observed"] },
+      lastPersistedEntryId: nullable(string(128)),
+      sha256: nullable(hash),
+      reason: nullable(string()),
+    }),
+    state: { enum: ["running", "terminal"] },
+    outcome: nullable(
+      closed({
+        code: nullable({ type: "integer", minimum: -2147483648, maximum: 2147483647 }),
+        signal: nullable(string(64)),
+        timedOut: { type: "boolean" },
+        aborted: { type: "boolean" },
+        truncated: { type: "boolean" },
+        failed: { type: "boolean" },
+      }),
+    ),
+    content: closed(
+      Object.fromEntries(
+        RETENTION_CONTENT_KINDS.map((kind) => [
+          kind,
+          {
+            oneOf: [
+              closed({ status: literal("missing"), path: literal(null), sha256: literal(null), bytes: literal(null) }),
+              closed({
+                status: literal("retained"),
+                path: { type: "string", pattern: `^${kind}-[a-f0-9]{64}\\.bin$` },
+                sha256: hash,
+                bytes: { ...integer, maximum: 1024 * 1024 },
+              }),
+            ],
+          },
+        ]),
+      ),
+    ),
+    coverage: closed({
+      complete: literal(false),
+      losses: { type: "array", items: string(), maxItems: 64, uniqueItems: true },
+    }),
+    acceptance: literal("not-assessed"),
+  }),
+  allOf: [
+    {
+      if: { properties: { state: literal("running") } },
+      then: { properties: { outcome: literal(null) } },
+      else: { properties: { outcome: { type: "object" } } },
+    },
+    {
+      if: { properties: { nativeSession: { properties: { branchState: literal("unknown") } } } },
+      then: {
+        properties: {
+          nativeSession: { properties: { branchLeafId: literal(null) } },
+          native: { properties: { branchLeafId: literal(null) } },
+        },
+      },
+    },
+  ],
+});
+const validator = Compile(RETENTION_SCHEMA);
+function freeze<T>(value: T): T {
+  if (value && typeof value === "object") {
+    for (const child of Object.values(value)) freeze(child);
+    Object.freeze(value);
+  }
+  return value;
+}
+
+/** Closed, detached versioned wire builder; never authenticates the supplied observation. */
+export function buildExecutionRetentionManifest(value: unknown): ExecutionRetentionManifest {
+  // Reject non-JSON objects/accessors instead of invoking serialization hooks while collecting evidence.
+  let nodes = 0;
+  const inspect = (x: unknown, depth: number): void => {
+    if (++nodes > 4096 || depth > 16) throw new TypeError("retention manifest exceeds bounds");
+    if (
+      x === null ||
+      typeof x === "string" ||
+      typeof x === "boolean" ||
+      (typeof x === "number" && Number.isSafeInteger(x))
+    )
+      return;
+    if (
+      !x ||
+      typeof x !== "object" ||
+      (!Array.isArray(x) && Object.getPrototypeOf(x) !== Object.prototype && Object.getPrototypeOf(x) !== null)
+    )
+      throw new TypeError("retention manifest must be JSON data");
+    if (
+      Array.isArray(x) &&
+      (Object.getPrototypeOf(x) !== Array.prototype ||
+        x.length > 256 ||
+        Reflect.ownKeys(x).length !== x.length + 1 ||
+        Array.from({ length: x.length }, (_, i) => String(i)).some((key) => !Object.hasOwn(x, key)))
+    )
+      throw new TypeError("retention arrays must be dense plain JSON arrays");
+    for (const key of Reflect.ownKeys(x)) {
+      if (Array.isArray(x) && key === "length") continue;
+      const d = Object.getOwnPropertyDescriptor(x, key)!;
+      if (typeof key !== "string" || !d.enumerable || !("value" in d))
+        throw new TypeError("retention manifest must be plain JSON data");
+      inspect(d.value, depth + 1);
+    }
+  };
+  inspect(value, 0);
+  if (!validator.Check(value)) throw new TypeError("invalid execution-retention 2.0 manifest");
+  const m = JSON.parse(JSON.stringify(value)) as ExecutionRetentionManifest;
+  if (
+    m.native.sessionId !== m.nativeSession.sessionId ||
+    m.native.sessionPath !== m.nativeSession.sessionPath ||
+    m.native.branchLeafId !== m.nativeSession.branchLeafId
+  ) {
+    throw new TypeError("native session projection mismatch");
+  }
+  for (const kind of RETENTION_CONTENT_KINDS) {
+    const ref = m.content[kind];
+    if (ref.status === "retained" && ref.path !== `${kind}-${ref.sha256}.bin`)
+      throw new TypeError("retention content identity mismatch");
+  }
+  if (
+    m.nativeSession.branchState === "observed" &&
+    (m.nativeSession.source !== "pi-session-manager" || m.nativeSession.status !== "verified")
+  )
+    throw new TypeError("unverified active native branch");
+  if (
+    m.nativeSession.status === "verified" &&
+    (!m.nativeSession.sessionId || !m.nativeSession.sha256 || m.content.session.sha256 !== m.nativeSession.sha256)
+  )
+    throw new TypeError("verified native session requires retained bytes");
+  return freeze(m);
+}
+export function parseExecutionRetentionManifest(text: string): ExecutionRetentionManifest {
+  if (Buffer.byteLength(text) > 64 * 1024) throw new TypeError("retention manifest exceeds bounds");
+  // Manifest numbers are integers; use the existing exact-token parser before lossy Number conversion.
+  let value: unknown;
+  try {
+    value = parseRetentionJson(text);
+  } catch {
+    throw new TypeError("invalid retention JSON");
+  }
+  return buildExecutionRetentionManifest(value);
+}

package/src/governance/retention-json.ts

@@ -0,0 +1,24 @@
+/** Strict JSON framing for retention metadata/native records. Floats in native usage remain supported. */
+export function parseRetentionJson(text: string, maxBytes = 64 * 1024): unknown {
+  if (Buffer.byteLength(text) > maxBytes) throw new TypeError("retention JSON exceeds bounds");
+  const value: unknown = JSON.parse(text);
+  const stack: Array<{ object: boolean; key: boolean; seen: Set<string> }> = [];
+  for (const match of text.matchAll(/"(?:[^"\\]|\\[\s\S])*"|[{}\[\],:]|[^\s{}\[\],:]+/g)) {
+    const token = match[0],
+      top = stack.at(-1);
+    if (token === "{" || token === "[") {
+      if (stack.length >= 64) throw new TypeError("retention JSON exceeds depth");
+      stack.push({ object: token === "{", key: token === "{", seen: new Set() });
+    } else if (token === "}" || token === "]") stack.pop();
+    else if (token === ",") {
+      if (top?.object) top.key = true;
+    } else if (token.startsWith('"') && top?.object && top.key) {
+      const key = JSON.parse(token) as string;
+      if (top.seen.has(key)) throw new TypeError("duplicate retention JSON member");
+      top.seen.add(key);
+      top.key = false;
+    } else if (/^-?\d/.test(token) && !Number.isFinite(Number(token)))
+      throw new TypeError("non-finite retention JSON number");
+  }
+  return value;
+}