vigiles 5.0.0 → 5.1.0

package/dist/adapters/codex/eval.js

@@ -0,0 +1,227 @@
+"use strict";
+/**
+ * Codex EVAL-tier transport — the runner + trace parser that
+ * `measureTriggerRate`/`runEval` dispatch to via the `ModelOutputParser` seam.
+ *
+ * SCHEMA: CONFIRMED against real `codex exec --json` (codex-cli 0.139.0, ChatGPT
+ * auth). The stream is the thread/item model:
+ *
+ *   {"type":"thread.started","thread_id":"…"}
+ *   {"type":"turn.started"}
+ *   {"type":"item.started","item":{"id":"item_0","type":"command_execution",…}}   // mid-flight
+ *   {"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"…","aggregated_output":"…","exit_code":0}}
+ *   {"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"…"}}
+ *   {"type":"turn.completed","usage":{"input_tokens":…,"cached_input_tokens":…,"output_tokens":…}}
+ *
+ * So: assistant text = `item.completed` with `item.type:"agent_message"` →
+ * `item.text`; a tool call = `item.type:"command_execution"` → `item.command`;
+ * usage rides `turn.completed`. We count `item.completed` ONLY (an `item.started`
+ * carries the same `id` mid-flight — counting both double-counts).
+ *
+ * THE SKILL FINDING: Codex has NO discrete "skill selected" event (its CLI has no
+ * Skill-tool concept). When a skill triggers, the model READS the skill's
+ * `SKILL.md` via a `command_execution` (`sed/cat … skills/<name>/SKILL.md`) and
+ * usually says so in an `agent_message`. So "did skill X fire" on Codex is not a
+ * clean trace event like Claude's `Skill` tool_use — it's detected by the
+ * SKILL.md read (`codexSkillFired`). Best-effort by nature (a cached skill might
+ * not be re-read); pair with a behavioral/judged check for certainty.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.codexEvalDriver = void 0;
+exports.parseCodexEvalRun = parseCodexEvalRun;
+exports.codexRunError = codexRunError;
+exports.codexSkillFired = codexSkillFired;
+exports.installCodexSkills = installCodexSkills;
+exports.codexEvalAgentRunner = codexEvalAgentRunner;
+exports.codexEvalRunner = codexEvalRunner;
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const str = (v) => (typeof v === "string" ? v : "");
+const num = (v) => (typeof v === "number" ? v : 0);
+/** Parse the JSONL stream, skipping blank / non-JSON / malformed lines. */
+function parseLines(stdout) {
+    const out = [];
+    for (const line of stdout.split(/\r?\n/)) {
+        const s = line.trim();
+        if (!s.startsWith("{"))
+            continue;
+        try {
+            out.push(JSON.parse(s));
+        }
+        catch {
+            /* tolerate a partial / non-event line */
+        }
+    }
+    return out;
+}
+/** Is this completed item a tool/command call (vs an agent_message / error)? */
+function isToolItem(itemType) {
+    return (itemType === "command_execution" || /function_call|tool/.test(itemType));
+}
+function buildCall(item, itemType) {
+    return {
+        // command_execution → the shell command; function-style → its name.
+        name: str(item.command) || str(item.name) || itemType,
+        input: item,
+        resultText: str(item.aggregated_output),
+        isError: num(item.exit_code) !== 0 && item.exit_code != null,
+    };
+}
+/** Map a `turn.completed` usage block to the common EvalUsage. */
+function usageFrom(u) {
+    return {
+        costUsd: 0, // codex on the ChatGPT sub reports no per-run USD
+        durationMs: 0,
+        inputTokens: num(u.input_tokens),
+        outputTokens: num(u.output_tokens),
+        cacheCreationTokens: 0,
+        cacheReadTokens: num(u.cached_input_tokens),
+    };
+}
+const ZERO_USAGE = {
+    costUsd: 0,
+    durationMs: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheCreationTokens: 0,
+    cacheReadTokens: 0,
+};
+/** Parse `codex exec --json` stdout into the common trace fields (confirmed schema). */
+function parseCodexEvalRun(out) {
+    const texts = [];
+    const toolCalls = [];
+    let usage = ZERO_USAGE;
+    for (const e of parseLines(out.stdout)) {
+        if (e.type === "turn.completed" && e.usage)
+            usage = usageFrom(e.usage);
+        // Count COMPLETED items only — item.started carries the same id mid-flight.
+        if (e.type !== "item.completed" || !e.item)
+            continue;
+        const itemType = str(e.item.type);
+        if (itemType === "agent_message") {
+            const t = str(e.item.text);
+            if (t)
+                texts.push(t);
+        }
+        else if (isToolItem(itemType)) {
+            toolCalls.push(buildCall(e.item, itemType));
+        }
+    }
+    return {
+        // Fallback to trimmed stdout if no agent_message was seen (keeps output non-empty).
+        output: texts.join("\n") || out.stdout.trim(),
+        turns: texts.length,
+        toolCalls,
+        hooks: [],
+        subagents: [],
+        usage,
+    };
+}
+/**
+ * The error message if the run errored or was rate-limited (an `error` /
+ * `turn.failed` event), else null. CRITICAL for the eval tier: an errored turn
+ * must NOT be scored as a clean "skill didn't fire" miss — dogfooding hit a Codex
+ * usage limit ("You've hit your usage limit…") whose `error` event left an empty
+ * trace that `codexSkillFired` read as recall 0. A caller should skip/retry an
+ * errored run, not count it. (The Claude path has `isRateLimited` + backoff; this
+ * is the Codex equivalent detector.)
+ */
+function codexRunError(out) {
+    for (const e of parseLines(out.stdout)) {
+        if (e.type === "error")
+            return str(e.message) || "codex error";
+        if (e.type === "turn.failed")
+            return str(e.error?.message) || "turn failed";
+    }
+    return null;
+}
+/**
+ * Did Codex activate skill `name` on this run? Detected by the SKILL.md read —
+ * Codex has no discrete skill-selection event, so when a skill triggers the model
+ * reads its `…/<name>/SKILL.md` via a `command_execution`. Best-effort (a cached
+ * skill might not be re-read); for the trigger-rate `fired` predicate over Codex.
+ */
+function codexSkillFired(run, name) {
+    const needle = `${name}/SKILL.md`;
+    return run.toolCalls.some((c) => str(c.name).includes(needle));
+}
+/**
+ * Materialize a (Claude-shaped) plugin dir's skills into `<cwd>/.codex/skills/` —
+ * where Codex actually discovers them (validated live: codex reads
+ * `<cwd>/.codex/skills/<name>/SKILL.md`). This is the Codex analog of Claude's
+ * `--plugin-dir`: `measureTriggerRate` hands the runner a `pluginDir` (the
+ * stubbed/packaged skills), and the Codex runner installs them here before the
+ * turn. Pure fs — unit-testable without a binary.
+ */
+function installCodexSkills(pluginDir, cwd) {
+    const skillsRoot = (0, node_path_1.join)(pluginDir, "skills");
+    if (!(0, node_fs_1.existsSync)(skillsRoot))
+        return 0;
+    let n = 0;
+    for (const name of (0, node_fs_1.readdirSync)(skillsRoot)) {
+        const src = (0, node_path_1.join)(skillsRoot, name, "SKILL.md");
+        if (!(0, node_fs_1.existsSync)(src))
+            continue;
+        const dest = (0, node_path_1.join)(cwd, ".codex", "skills", name, "SKILL.md");
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dest), { recursive: true });
+        (0, node_fs_1.writeFileSync)(dest, (0, node_fs_1.readFileSync)(src, "utf-8"));
+        n += 1;
+    }
+    return n;
+}
+/* v8 ignore start -- real codex subprocess; validated against the binary, not the unit gate */
+/**
+ * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
+ * (Codex's discovery path, vs Claude's `--plugin-dir`), then drive a real
+ * `codex exec --json` turn. The seam `measureTriggerRate(spec, { evalDriver:
+ * codexEvalDriver })` dispatches through.
+ */
+function codexEvalAgentRunner(args) {
+    if (args.pluginDir)
+        installCodexSkills(args.pluginDir, args.cwd);
+    return Promise.resolve(codexEvalRunner({
+        task: args.task,
+        cwd: args.cwd,
+        timeoutMs: args.timeoutMs,
+    }));
+}
+/**
+ * The Codex eval driver — pass to `measureTriggerRate(spec, { evalDriver:
+ * codexEvalDriver })` to run a trigger-rate eval natively on `codex exec`. Pair
+ * the spec's `fired` with `codexSkillFired` (Codex has no Skill-tool event).
+ */
+exports.codexEvalDriver = {
+    runner: codexEvalAgentRunner,
+    parse: parseCodexEvalRun,
+    runError: codexRunError,
+};
+/**
+ * Spawn real `codex exec --json` for the eval tier (real model, the user's codex
+ * auth — NOT the mock). CONFIRMED flags (codex 0.139.0): `--json` for the event
+ * stream, `--skip-git-repo-check` for a bare cwd, the approvals/sandbox bypass so
+ * the turn runs unattended, `-C <cwd>` for the working dir, prompt as the trailing
+ * positional, and stdin = /dev/null (`stdio: ["ignore",…]`) — codex otherwise
+ * blocks on "Reading additional input from stdin…". Needs ChatGPT/API auth +
+ * network egress to the model backend.
+ */
+function codexEvalRunner(args) {
+    const r = (0, node_child_process_1.spawnSync)("codex", [
+        "exec",
+        "--json",
+        "--skip-git-repo-check",
+        "--dangerously-bypass-approvals-and-sandbox",
+        "-C",
+        args.cwd,
+        args.task,
+    ], {
+        cwd: args.cwd,
+        encoding: "utf-8",
+        timeout: args.timeoutMs,
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: 64 * 1024 * 1024,
+    });
+    return { code: r.status ?? 1, stdout: r.stdout ?? "" };
+}
+/* v8 ignore stop */
+//# sourceMappingURL=eval.js.map