pome-sh 0.6.6

package/dist/src/recorder/artifacts.d.ts

@@ -0,0 +1,37 @@
+import type { Scenario } from "../scenario/scenarioSchema.js";
+import type { RecorderEvent } from "../twin/github/types.js";
+export declare function toTwinHttpEvent(event: RecorderEvent): RecorderEvent & {
+    kind: "TwinHttpEvent";
+    event_id: string;
+    parent_id: null;
+};
+export type RunArtifacts = {
+    runId: string;
+    runDir: string;
+};
+export type RunArtifactCoreInput = {
+    artifactsDir: string;
+    runId: string;
+    scenario: Scenario;
+    startedAt: string;
+    completedAt: string;
+    stdout: string;
+    stderr: string;
+    exitCode: number | null;
+    events: RecorderEvent[];
+    stateInitial: unknown;
+    stateFinal: unknown;
+};
+export declare function writeRunArtifactsCore(input: RunArtifactCoreInput): Promise<RunArtifacts>;
+export declare function readLatestRun(artifactsDir: string): Promise<{
+    run_id: string;
+    scenario: string;
+    run_dir: string;
+} | undefined>;
+export type RunMetaSummary = {
+    run_id?: string;
+    scenario?: string;
+    title?: string;
+    twins: string[];
+};
+export declare function readMetaSummary(runDir: string): Promise<RunMetaSummary>;

package/dist/src/recorder/artifacts.js

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: Apache-2.0
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { redactEvent, redactSecrets } from "./redaction.js";
+// FDRS-399 / FDRS-398 — wrap a legacy RecorderEvent (or pass through any row
+// that already carries a `kind` discriminator) into the unified events.jsonl
+// shape. Reuses `request_id` as `event_id` so within-run lookups stay stable
+// across the legacy and unified views.
+//
+// Exported so `runScenarioHosted` can apply the same wrap before uploading
+// events.jsonl to cloud storage. Without it, cloud's FDRS-398 schema gate
+// rejects the upload as "legacy single-shape RecorderEvent".
+export function toTwinHttpEvent(event) {
+    const maybeKind = event.kind;
+    if (typeof maybeKind === "string") {
+        return event;
+    }
+    return {
+        ...event,
+        kind: "TwinHttpEvent",
+        event_id: event.request_id,
+        parent_id: null,
+    };
+}
+// FDRS-657 — the OSS CLI is CAPTURE-ONLY: it writes raw trace + state
+// artifacts and NEVER a score.json. Local artifacts are trace/audit only; a
+// verdict comes only from the cloud and is printed ephemerally to the
+// terminal (hosted `pome run`, `pome eval`) — never persisted next to the
+// trace. There is no `writeScoreJson`/`readScore` anymore.
+export async function writeRunArtifactsCore(input) {
+    const runDir = join(input.artifactsDir, input.scenario.slug, input.runId);
+    await mkdir(runDir, { recursive: true });
+    await writeJson(join(runDir, "meta.json"), {
+        run_id: input.runId,
+        scenario: input.scenario.slug,
+        title: input.scenario.title,
+        started_at: input.startedAt,
+        completed_at: input.completedAt,
+        exit_code: input.exitCode,
+        twins: input.scenario.config.twins
+    });
+    // tool_calls.jsonl keeps the legacy recorder shape (no `kind`) — it's the
+    // pre-FDRS-398 view that some downstream consumers still read.
+    const toolCallsJsonl = input.events.map((event) => JSON.stringify(redactEvent(event))).join("\n") + "\n";
+    await writeFile(join(runDir, "tool_calls.jsonl"), toolCallsJsonl);
+    // events.jsonl is the unified discriminated-union view (FDRS-398). The
+    // in-process twin recorders still emit legacy RecorderEvent rows; wrap each
+    // one into a TwinHttpEvent before serializing so `pome inspect` (FDRS-403)
+    // and the dashboard ingest (FDRS-415) can parse them. APPEND (not
+    // truncate) so rows already written by the capture-server child
+    // (FDRS-399) during the run survive.
+    const eventsJsonl = input.events.map((event) => JSON.stringify(redactEvent(toTwinHttpEvent(event)))).join("\n") + "\n";
+    await appendFile(join(runDir, "events.jsonl"), eventsJsonl);
+    await writeJson(join(runDir, "state_initial.json"), input.stateInitial);
+    await writeJson(join(runDir, "state_final.json"), input.stateFinal);
+    await writeJson(join(runDir, "state-before.json"), input.stateInitial);
+    await writeJson(join(runDir, "state-after.json"), input.stateFinal);
+    await writeJson(join(runDir, "state-diff.json"), summarizeStateDiff(input.stateInitial, input.stateFinal));
+    await writeFile(join(runDir, "stdout.txt"), redactText(input.stdout));
+    await writeFile(join(runDir, "stderr.log"), redactText(input.stderr));
+    await writeJson(join(input.artifactsDir, "latest.json"), {
+        run_id: input.runId,
+        scenario: input.scenario.slug,
+        run_dir: runDir
+    });
+    return { runId: input.runId, runDir };
+}
+export async function readLatestRun(artifactsDir) {
+    const latestPath = join(artifactsDir, "latest.json");
+    if (!existsSync(latestPath))
+        return undefined;
+    return JSON.parse(await readFile(latestPath, "utf8"));
+}
+export async function readMetaSummary(runDir) {
+    try {
+        const raw = await readFile(join(runDir, "meta.json"), "utf8");
+        const parsed = JSON.parse(raw);
+        const twins = Array.isArray(parsed.twins)
+            ? parsed.twins.filter((t) => typeof t === "string")
+            : [];
+        return { ...parsed, twins };
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return { twins: [] };
+        throw err;
+    }
+}
+async function writeJson(path, value) {
+    await writeFile(path, `${JSON.stringify(redactSecrets(value), null, 2)}\n`);
+}
+function redactText(value) {
+    return redactSecrets(value);
+}
+function summarizeStateDiff(before, after) {
+    const beforeSummary = summarizeState(before);
+    const afterSummary = summarizeState(after);
+    const keys = Array.from(new Set([...Object.keys(beforeSummary), ...Object.keys(afterSummary)])).sort();
+    return {
+        summary: keys.map((key) => ({
+            path: key,
+            before: beforeSummary[key] ?? 0,
+            after: afterSummary[key] ?? 0,
+            delta: (afterSummary[key] ?? 0) - (beforeSummary[key] ?? 0)
+        }))
+    };
+}
+function summarizeState(value, prefix = "") {
+    if (Array.isArray(value)) {
+        return prefix ? { [prefix]: value.length } : {};
+    }
+    if (!value || typeof value !== "object")
+        return {};
+    const out = {};
+    for (const [key, child] of Object.entries(value)) {
+        const path = prefix ? `${prefix}.${key}` : key;
+        Object.assign(out, summarizeState(child, path));
+    }
+    return out;
+}

package/dist/src/recorder/inspect.d.ts

@@ -0,0 +1,37 @@
+import { type Event } from "../types/shared.js";
+/**
+ * Result of reading + classifying a run's `events.jsonl`. The reader can take
+ * three paths:
+ *   - `kind: "events"` — every row parsed cleanly against the discriminated
+ *     union; the renderer walks `events` to print sections + trace health.
+ *   - `kind: "legacy"` — at least one row is missing the `kind` discriminator
+ *     (pre-FDRS-398 shape). `pome inspect` prints the legacy error and exits 2.
+ *   - `kind: "missing"` — `events.jsonl` does not exist (run never produced
+ *     trace blobs). Renderer prints a one-line note; no exit-2.
+ */
+export type ReadEventsResult = {
+    kind: "events";
+    events: Event[];
+} | {
+    kind: "legacy";
+} | {
+    kind: "missing";
+};
+export declare const LEGACY_EVENTS_MESSAGE = "this run was produced by an older CLI version (pre-M0); rerun against current CLI to view";
+export declare function readEventsJsonl(runDir: string): Promise<ReadEventsResult>;
+export type TraceHealthLayer = {
+    name: string;
+    count: number;
+    expectedAtLeast: number;
+    status: "ok" | "warning" | "none";
+    note: string | null;
+};
+export type TraceHealthInput = {
+    events: Event[];
+    scenarioUsesTwin?: boolean;
+    scenarioExpectsLlm?: boolean;
+    scenarioExpectsCas?: boolean;
+};
+export declare function computeTraceHealth(input: TraceHealthInput): TraceHealthLayer[];
+export declare function renderTraceHealth(layers: TraceHealthLayer[]): string[];
+export declare function renderEvents(events: Event[]): string[];

package/dist/src/recorder/inspect.js

@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { eventSchema, isLegacyEventRow, } from "../types/shared.js";
+export const LEGACY_EVENTS_MESSAGE = "this run was produced by an older CLI version (pre-M0); rerun against current CLI to view";
+export async function readEventsJsonl(runDir) {
+    let raw;
+    try {
+        raw = await readFile(join(runDir, "events.jsonl"), "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT") {
+            return { kind: "missing" };
+        }
+        throw err;
+    }
+    const rows = raw
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((line) => JSON.parse(line));
+    if (rows.some(isLegacyEventRow)) {
+        return { kind: "legacy" };
+    }
+    const events = rows.map((row) => eventSchema.parse(row));
+    return { kind: "events", events };
+}
+export function computeTraceHealth(input) {
+    const counts = countByKind(input.events);
+    // proxy layer — LLM calls captured by the HTTP_PROXY CONNECT tunnel.
+    // Default heuristic: presence implies it was expected (so a populated layer
+    // never warns). When the scenario manifest declares LLM usage we expect at
+    // least one row even with zero observed; absence then surfaces as a warning
+    // ("capture-server may not have started").
+    const proxyExpected = input.scenarioExpectsLlm
+        ? 1
+        : counts.LlmCallEvent > 0
+            ? 1
+            : 0;
+    const proxy = {
+        name: "proxy",
+        count: counts.LlmCallEvent,
+        expectedAtLeast: proxyExpected,
+        status: layerStatus(counts.LlmCallEvent, proxyExpected),
+        note: proxyExpected > 0 && counts.LlmCallEvent === 0
+            ? "warning: capture-server may not have started"
+            : null,
+    };
+    // twin layer — TwinHttpEvent rows. A scenario always declares ≥1 twin in
+    // `config.twins`, so when we know the scenario we expect ≥1 row.
+    const twinExpectedBool = input.scenarioUsesTwin ?? counts.TwinHttpEvent > 0;
+    const twinExpected = twinExpectedBool ? 1 : 0;
+    const twin = {
+        name: "twin",
+        count: counts.TwinHttpEvent,
+        expectedAtLeast: twinExpected,
+        status: layerStatus(counts.TwinHttpEvent, twinExpected),
+        note: twinExpected > 0 && counts.TwinHttpEvent === 0
+            ? "warning: twin runtime emitted no HTTP events"
+            : null,
+    };
+    // CAS adapter layer — only active when the agent is wired through
+    // `@pome-sh/claude-agent-sdk`. Heuristic: any of ToolUse/ToolResult/
+    // SubagentSpawn/Hook indicates adapter is live.
+    const casCount = counts.ToolUseEvent +
+        counts.ToolResultEvent +
+        counts.SubagentSpawnEvent +
+        counts.HookEvent;
+    const casExpected = input.scenarioExpectsCas ? 1 : casCount > 0 ? 1 : 0;
+    const cas = {
+        name: "CAS adapter",
+        count: casCount,
+        expectedAtLeast: casExpected,
+        status: layerStatus(casCount, casExpected),
+        note: casExpected > 0 && casCount === 0
+            ? "warning: scenario flagged adapter but no adapter events recorded"
+            : null,
+    };
+    return [proxy, twin, cas];
+}
+function layerStatus(count, expected) {
+    if (expected === 0)
+        return count > 0 ? "ok" : "none";
+    return count >= expected ? "ok" : "warning";
+}
+function emptyCounts() {
+    return {
+        TwinHttpEvent: 0,
+        LlmCallEvent: 0,
+        ToolUseEvent: 0,
+        ToolResultEvent: 0,
+        SubagentSpawnEvent: 0,
+        HookEvent: 0,
+    };
+}
+function countByKind(events) {
+    const counts = emptyCounts();
+    for (const event of events) {
+        counts[event.kind] += 1;
+    }
+    return counts;
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// Rendering
+//
+// Lines are emitted into a string array so tests can snapshot the full output
+// without intercepting console.log. The inspect command joins on "\n".
+// ─────────────────────────────────────────────────────────────────────────────
+export function renderTraceHealth(layers) {
+    const lines = ["Trace health:"];
+    for (const layer of layers) {
+        const expected = layer.expectedAtLeast > 0 ? `expected≥${layer.expectedAtLeast}` : "no expectation";
+        const tail = layer.note ? ` (${layer.note})` : "";
+        lines.push(`  ${layer.name}: ${layer.count}/${expected} [${layer.status}]${tail}`);
+    }
+    return lines;
+}
+export function renderEvents(events) {
+    if (events.length === 0) {
+        return ["Events: (none)"];
+    }
+    const lines = [`Events (${events.length}):`];
+    for (const event of events) {
+        lines.push(...renderEventLines(event));
+    }
+    return lines;
+}
+function renderEventLines(event) {
+    switch (event.kind) {
+        case "TwinHttpEvent":
+            return renderTwinHttp(event);
+        case "LlmCallEvent":
+            return renderLlmCall(event);
+        case "ToolUseEvent":
+            return renderToolUse(event);
+        case "ToolResultEvent":
+            return renderToolResult(event);
+        case "SubagentSpawnEvent":
+            return renderSubagentSpawn(event);
+        case "HookEvent":
+            return renderHook(event);
+    }
+}
+function renderTwinHttp(event) {
+    const status = event.error ? `ERR ${event.status}` : `${event.status}`;
+    const dedupe = event.idempotency_dedupe ? " [dedupe-replay]" : "";
+    const mutation = event.state_mutation ? " [mutation]" : "";
+    return [
+        `- TwinHttpEvent  ${event.method} ${event.path} → ${status} (${event.latency_ms}ms)${mutation}${dedupe}`,
+        `    twin=${event.twin}  fidelity=${event.fidelity}${event.error ? `  error=${event.error}` : ""}`,
+    ];
+}
+function renderLlmCall(event) {
+    const model = event.model ? ` model=${event.model}` : "";
+    const status = event.status !== null ? ` status=${event.status}` : "";
+    const tokens = event.prompt_tokens !== null || event.completion_tokens !== null
+        ? ` tokens=${event.prompt_tokens ?? "?"}/${event.completion_tokens ?? "?"}`
+        : "";
+    return [
+        `- LlmCallEvent   ${event.host}:${event.port}${status} (${event.latency_ms}ms)`,
+        `    bytes=${event.bytes_in}/${event.bytes_out}${model}${tokens}`,
+    ];
+}
+function renderToolUse(event) {
+    return [`- ToolUseEvent   ${event.tool_name} (id=${event.tool_use_id})`];
+}
+function renderToolResult(event) {
+    const status = event.is_error ? "error" : "ok";
+    return [`- ToolResultEvent ${status} (use_id=${event.tool_use_id})`];
+}
+function renderSubagentSpawn(event) {
+    return [`- SubagentSpawnEvent parent_tool_use_id=${event.parent_tool_use_id}`];
+}
+function renderHook(event) {
+    const tool = event.tool_name ? ` tool=${event.tool_name}` : "";
+    return [`- HookEvent      ${event.hook_name}${tool}`];
+}

package/dist/src/recorder/recorder.d.ts

@@ -0,0 +1,3 @@
+import type { Recorder } from "../twin/github/types.js";
+export declare function createRecorder(): Recorder;
+export declare function createRequestId(): string;

package/dist/src/recorder/recorder.js

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+import { randomUUID } from "node:crypto";
+export function createRecorder() {
+    const items = [];
+    return {
+        record(event) {
+            items.push(event);
+        },
+        events() {
+            return [...items];
+        }
+    };
+}
+export function createRequestId() {
+    return `req_${randomUUID()}`;
+}

package/dist/src/recorder/redaction.d.ts

@@ -0,0 +1,2 @@
+export declare function redactSecrets(value: unknown): unknown;
+export declare function redactEvent<T>(event: T): T;

package/dist/src/recorder/redaction.js

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Centralized redactor applied at every events.jsonl / trace write site
+// (FDRS-401, FDRS-402, FDRS-588, FDRS-608). This file is a BYTE-IDENTICAL
+// mirror kept in five places because these modules must not depend on each
+// other:
+//   - cli/src/recorder/redaction.ts
+//   - packages/adapter-claude-sdk/src/redaction.ts
+//   - packages/twin-github/src/redaction.ts
+//   - packages/twin-slack/src/redaction.ts
+//   - packages/twin-stripe/src/redaction.ts
+// When this list grows, update EVERY mirror and keep them byte-identical.
+//
+// Two layers, both unconditional:
+//   1. Key-based: any field whose key matches the hard-redact list is replaced
+//      with "[REDACTED]". `authorization`, `x-api-key`, and `cookie` are
+//      required by the ticket; the other entries are pre-existing safeguards
+//      against common secret-bearing field names that surfaced before FDRS-401.
+//   2. Regex-based: scrub well-known credential shapes (API key prefixes,
+//      JWTs, PEM blocks) anywhere inside string values, so secrets leaking
+//      through OTLP attributes, ToolUseEvent args, or twin request bodies
+//      are caught even when the field name is benign.
+const REDACTED = "[REDACTED]";
+const HARD_REDACT_KEYS = new Set([
+    "authorization",
+    "x-api-key",
+    "cookie",
+    "api_key",
+    "client_secret",
+    "webhook_secret",
+    "password",
+    "secret",
+    "token",
+    "access_token",
+    "refresh_token",
+    "session_token",
+    "agent_token",
+    "anthropic_api_key",
+]);
+// `sk-...` covers OpenAI + Anthropic (`sk-ant-...`) + variants like `sk-proj-`.
+// The leading boundary avoids mangling benign kebab-case words ending in "sk".
+// `(s|r)k_(test|live)_...` covers Stripe secret/restricted keys (FDRS-588) —
+// a shorter body threshold than the `pk`/`rk` rule below because seed keys like
+// `sk_test_pome_a` are only a few chars past the prefix.
+// `xapp-...` is a Slack app-level token; `AIza...` is a Google API key
+// (FDRS-608). `g` flag matters: a single string may contain multiple secrets.
+const SCRUB_PATTERNS = [
+    /redaction_fixture_secret_[A-Za-z0-9_-]{8,}/g,
+    /\bsk-[A-Za-z0-9_-]{20,}/g,
+    /\b[rs]k_(?:test|live)_[A-Za-z0-9_]{4,}/g,
+    /ghp_[A-Za-z0-9]{36}/g,
+    /github_pat_[A-Za-z0-9_]{20,}/g,
+    /xox[aboprs]-[A-Za-z0-9-]{20,}/g,
+    /xapp-[A-Za-z0-9-]{10,}/g,
+    /(?:pme|pk|rk)_[A-Za-z0-9_-]{20,}/g,
+    /AIza[0-9A-Za-z_-]{20,}/g,
+    /AKIA[0-9A-Z]{16}/g,
+    /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g,
+    /-----BEGIN [A-Z ]+-----[\s\S]*?-----END [A-Z ]+-----/g,
+    /-----BEGIN [A-Z ]+-----/g,
+];
+function scrubString(value) {
+    let out = value;
+    for (const pattern of SCRUB_PATTERNS) {
+        out = out.replace(pattern, REDACTED);
+    }
+    return out;
+}
+export function redactSecrets(value) {
+    if (typeof value === "string")
+        return scrubString(value);
+    if (Array.isArray(value))
+        return value.map(redactSecrets);
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([key, nested]) => [
+            key,
+            HARD_REDACT_KEYS.has(key.toLowerCase()) ? REDACTED : redactSecrets(nested),
+        ]));
+    }
+    return value;
+}
+// Public name used by events.jsonl writers. Same impl as redactSecrets — the
+// alias exists so write sites read as "redact this event before persisting"
+// rather than the generic "redact secrets."
+export function redactEvent(event) {
+    return redactSecrets(event);
+}

package/dist/src/recorder/verdictArtifact.d.ts

@@ -0,0 +1,71 @@
+import type { CriterionResult } from "../types/shared.js";
+export declare const VERDICT_ARTIFACT_VERSION = 1;
+export declare const VERDICT_FILENAME = "verdict.json";
+export interface VerdictArtifact {
+    version: number;
+    /** Provenance: the only writer is the /finalize response path. */
+    source: "cloud-finalize";
+    task_name: string;
+    /** The scenario path as the run invocation saw it (may move later —
+     *  readers must tolerate a dangling path). */
+    scenario_path: string;
+    /** Shared trial-group id (`grp_` + nanoid21); null on single runs. */
+    group_id: string | null;
+    session_id: string;
+    cloud_run_id: string;
+    cloud_dashboard_url: string;
+    judge_model: string | null;
+    /** Cloud-authoritative satisfaction score, 0-100. */
+    score: number;
+    pass_threshold: number;
+    passed: boolean;
+    criteria_results: CriterionResult[];
+    duration_ms: number;
+    finalized_at: string;
+}
+export interface TrialVerdict {
+    runDir: string;
+    verdict: VerdictArtifact;
+}
+export declare function writeVerdictArtifact(runDir: string, verdict: VerdictArtifact): Promise<void>;
+export declare function readVerdictArtifact(runDir: string): Promise<TrialVerdict | null>;
+/** Scan an artifacts root for finalized trials — exactly the two-level
+ *  layout `<root>/<task-slug>/<session-id>/verdict.json`. Anything
+ *  unreadable is skipped. */
+export declare function scanVerdictArtifacts(artifactsRoot: string): Promise<TrialVerdict[]>;
+export interface RunSet {
+    /** null = a single run that never had a group. */
+    groupId: string | null;
+    taskName: string;
+    /** The scenario path recorded at run time (first trial's). */
+    scenarioPath: string;
+    /** Trials sorted by finalized_at ascending. */
+    trials: TrialVerdict[];
+    latestFinalizedAt: string;
+    anyFailed: boolean;
+}
+/** Group trials into run sets: trials sharing a group_id form one set; a
+ *  null group_id is its own single-run set. */
+export declare function groupRunSets(trials: TrialVerdict[]): RunSet[];
+/** The newest run set with at least one failed (completed) trial. */
+export declare function latestFailedRunSet(sets: RunSet[]): RunSet | null;
+export interface RunSetDiscovery {
+    kind: "trial-dir" | "root";
+    /** The set to build a fix prompt from (per `kind` semantics: a trial dir
+     *  targets ITS set regardless of outcome; a root targets the latest
+     *  FAILED set). Null when nothing matches. */
+    set: RunSet | null;
+    /** Total finalized run sets seen — lets the caller distinguish "no runs
+     *  at all" from "runs exist but none failed". */
+    totalSets: number;
+}
+/**
+ * Resolve what `pome fix-prompt [target]` should read.
+ * - `target` = a trial run dir (has verdict.json): that trial's whole set —
+ *   the user pointed at it, outcome doesn't matter.
+ * - `target` = an artifacts root: the latest failed set under it.
+ */
+export declare function discoverRunSet(target: string): Promise<RunSetDiscovery>;
+/** Load a trial's captured events.jsonl (raw trace) for prompt assembly.
+ *  Missing/corrupt lines are skipped — the prompt degrades, never throws. */
+export declare function loadTrialEvents(runDir: string): Promise<unknown[]>;