vigiles 5.0.1 → 5.2.0

package/dist/adapters/codex/eval.js

@@ -0,0 +1,227 @@
+"use strict";
+/**
+ * Codex EVAL-tier transport — the runner + trace parser that
+ * `measureTriggerRate`/`runEval` dispatch to via the `ModelOutputParser` seam.
+ *
+ * SCHEMA: CONFIRMED against real `codex exec --json` (codex-cli 0.139.0, ChatGPT
+ * auth). The stream is the thread/item model:
+ *
+ *   {"type":"thread.started","thread_id":"…"}
+ *   {"type":"turn.started"}
+ *   {"type":"item.started","item":{"id":"item_0","type":"command_execution",…}}   // mid-flight
+ *   {"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"…","aggregated_output":"…","exit_code":0}}
+ *   {"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"…"}}
+ *   {"type":"turn.completed","usage":{"input_tokens":…,"cached_input_tokens":…,"output_tokens":…}}
+ *
+ * So: assistant text = `item.completed` with `item.type:"agent_message"` →
+ * `item.text`; a tool call = `item.type:"command_execution"` → `item.command`;
+ * usage rides `turn.completed`. We count `item.completed` ONLY (an `item.started`
+ * carries the same `id` mid-flight — counting both double-counts).
+ *
+ * THE SKILL FINDING: Codex has NO discrete "skill selected" event (its CLI has no
+ * Skill-tool concept). When a skill triggers, the model READS the skill's
+ * `SKILL.md` via a `command_execution` (`sed/cat … skills/<name>/SKILL.md`) and
+ * usually says so in an `agent_message`. So "did skill X fire" on Codex is not a
+ * clean trace event like Claude's `Skill` tool_use — it's detected by the
+ * SKILL.md read (`codexSkillFired`). Best-effort by nature (a cached skill might
+ * not be re-read); pair with a behavioral/judged check for certainty.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.codexEvalDriver = void 0;
+exports.parseCodexEvalRun = parseCodexEvalRun;
+exports.codexRunError = codexRunError;
+exports.codexSkillFired = codexSkillFired;
+exports.installCodexSkills = installCodexSkills;
+exports.codexEvalAgentRunner = codexEvalAgentRunner;
+exports.codexEvalRunner = codexEvalRunner;
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const str = (v) => (typeof v === "string" ? v : "");
+const num = (v) => (typeof v === "number" ? v : 0);
+/** Parse the JSONL stream, skipping blank / non-JSON / malformed lines. */
+function parseLines(stdout) {
+    const out = [];
+    for (const line of stdout.split(/\r?\n/)) {
+        const s = line.trim();
+        if (!s.startsWith("{"))
+            continue;
+        try {
+            out.push(JSON.parse(s));
+        }
+        catch {
+            /* tolerate a partial / non-event line */
+        }
+    }
+    return out;
+}
+/** Is this completed item a tool/command call (vs an agent_message / error)? */
+function isToolItem(itemType) {
+    return (itemType === "command_execution" || /function_call|tool/.test(itemType));
+}
+function buildCall(item, itemType) {
+    return {
+        // command_execution → the shell command; function-style → its name.
+        name: str(item.command) || str(item.name) || itemType,
+        input: item,
+        resultText: str(item.aggregated_output),
+        isError: num(item.exit_code) !== 0 && item.exit_code != null,
+    };
+}
+/** Map a `turn.completed` usage block to the common EvalUsage. */
+function usageFrom(u) {
+    return {
+        costUsd: 0, // codex on the ChatGPT sub reports no per-run USD
+        durationMs: 0,
+        inputTokens: num(u.input_tokens),
+        outputTokens: num(u.output_tokens),
+        cacheCreationTokens: 0,
+        cacheReadTokens: num(u.cached_input_tokens),
+    };
+}
+const ZERO_USAGE = {
+    costUsd: 0,
+    durationMs: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheCreationTokens: 0,
+    cacheReadTokens: 0,
+};
+/** Parse `codex exec --json` stdout into the common trace fields (confirmed schema). */
+function parseCodexEvalRun(out) {
+    const texts = [];
+    const toolCalls = [];
+    let usage = ZERO_USAGE;
+    for (const e of parseLines(out.stdout)) {
+        if (e.type === "turn.completed" && e.usage)
+            usage = usageFrom(e.usage);
+        // Count COMPLETED items only — item.started carries the same id mid-flight.
+        if (e.type !== "item.completed" || !e.item)
+            continue;
+        const itemType = str(e.item.type);
+        if (itemType === "agent_message") {
+            const t = str(e.item.text);
+            if (t)
+                texts.push(t);
+        }
+        else if (isToolItem(itemType)) {
+            toolCalls.push(buildCall(e.item, itemType));
+        }
+    }
+    return {
+        // Fallback to trimmed stdout if no agent_message was seen (keeps output non-empty).
+        output: texts.join("\n") || out.stdout.trim(),
+        turns: texts.length,
+        toolCalls,
+        hooks: [],
+        subagents: [],
+        usage,
+    };
+}
+/**
+ * The error message if the run errored or was rate-limited (an `error` /
+ * `turn.failed` event), else null. CRITICAL for the eval tier: an errored turn
+ * must NOT be scored as a clean "skill didn't fire" miss — dogfooding hit a Codex
+ * usage limit ("You've hit your usage limit…") whose `error` event left an empty
+ * trace that `codexSkillFired` read as recall 0. A caller should skip/retry an
+ * errored run, not count it. (The Claude path has `isRateLimited` + backoff; this
+ * is the Codex equivalent detector.)
+ */
+function codexRunError(out) {
+    for (const e of parseLines(out.stdout)) {
+        if (e.type === "error")
+            return str(e.message) || "codex error";
+        if (e.type === "turn.failed")
+            return str(e.error?.message) || "turn failed";
+    }
+    return null;
+}
+/**
+ * Did Codex activate skill `name` on this run? Detected by the SKILL.md read —
+ * Codex has no discrete skill-selection event, so when a skill triggers the model
+ * reads its `…/<name>/SKILL.md` via a `command_execution`. Best-effort (a cached
+ * skill might not be re-read); for the trigger-rate `fired` predicate over Codex.
+ */
+function codexSkillFired(run, name) {
+    const needle = `${name}/SKILL.md`;
+    return run.toolCalls.some((c) => str(c.name).includes(needle));
+}
+/**
+ * Materialize a (Claude-shaped) plugin dir's skills into `<cwd>/.codex/skills/` —
+ * where Codex actually discovers them (validated live: codex reads
+ * `<cwd>/.codex/skills/<name>/SKILL.md`). This is the Codex analog of Claude's
+ * `--plugin-dir`: `measureTriggerRate` hands the runner a `pluginDir` (the
+ * stubbed/packaged skills), and the Codex runner installs them here before the
+ * turn. Pure fs — unit-testable without a binary.
+ */
+function installCodexSkills(pluginDir, cwd) {
+    const skillsRoot = (0, node_path_1.join)(pluginDir, "skills");
+    if (!(0, node_fs_1.existsSync)(skillsRoot))
+        return 0;
+    let n = 0;
+    for (const name of (0, node_fs_1.readdirSync)(skillsRoot)) {
+        const src = (0, node_path_1.join)(skillsRoot, name, "SKILL.md");
+        if (!(0, node_fs_1.existsSync)(src))
+            continue;
+        const dest = (0, node_path_1.join)(cwd, ".codex", "skills", name, "SKILL.md");
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dest), { recursive: true });
+        (0, node_fs_1.writeFileSync)(dest, (0, node_fs_1.readFileSync)(src, "utf-8"));
+        n += 1;
+    }
+    return n;
+}
+/* v8 ignore start -- real codex subprocess; validated against the binary, not the unit gate */
+/**
+ * The Codex eval-tier `AgentRunner`: install the run's skills into `.codex/skills`
+ * (Codex's discovery path, vs Claude's `--plugin-dir`), then drive a real
+ * `codex exec --json` turn. The seam `measureTriggerRate(spec, { evalDriver:
+ * codexEvalDriver })` dispatches through.
+ */
+function codexEvalAgentRunner(args) {
+    if (args.pluginDir)
+        installCodexSkills(args.pluginDir, args.cwd);
+    return Promise.resolve(codexEvalRunner({
+        task: args.task,
+        cwd: args.cwd,
+        timeoutMs: args.timeoutMs,
+    }));
+}
+/**
+ * The Codex eval driver — pass to `measureTriggerRate(spec, { evalDriver:
+ * codexEvalDriver })` to run a trigger-rate eval natively on `codex exec`. Pair
+ * the spec's `fired` with `codexSkillFired` (Codex has no Skill-tool event).
+ */
+exports.codexEvalDriver = {
+    runner: codexEvalAgentRunner,
+    parse: parseCodexEvalRun,
+    runError: codexRunError,
+};
+/**
+ * Spawn real `codex exec --json` for the eval tier (real model, the user's codex
+ * auth — NOT the mock). CONFIRMED flags (codex 0.139.0): `--json` for the event
+ * stream, `--skip-git-repo-check` for a bare cwd, the approvals/sandbox bypass so
+ * the turn runs unattended, `-C <cwd>` for the working dir, prompt as the trailing
+ * positional, and stdin = /dev/null (`stdio: ["ignore",…]`) — codex otherwise
+ * blocks on "Reading additional input from stdin…". Needs ChatGPT/API auth +
+ * network egress to the model backend.
+ */
+function codexEvalRunner(args) {
+    const r = (0, node_child_process_1.spawnSync)("codex", [
+        "exec",
+        "--json",
+        "--skip-git-repo-check",
+        "--dangerously-bypass-approvals-and-sandbox",
+        "-C",
+        args.cwd,
+        args.task,
+    ], {
+        cwd: args.cwd,
+        encoding: "utf-8",
+        timeout: args.timeoutMs,
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: 64 * 1024 * 1024,
+    });
+    return { code: r.status ?? 1, stdout: r.stdout ?? "" };
+}
+/* v8 ignore stop */
+//# sourceMappingURL=eval.js.map

package/dist/adapters/codex/layout.js

@@ -9,6 +9,9 @@ exports.codexLayout = {
     settingsFormat: "toml",
     instructionFile: "AGENTS.md",
     surfaceDirs: ["skills", "prompts"],
+    skillDir: "skills",
+    agentDir: "", // Codex `[agents]` is a TOML concurrency table, not a subagent dir
+    commandDir: "prompts",
     materializeRoot: ".codex",
     pluginRootToken: "${PLUGIN_ROOT}",
     mcpConfigFile: ".mcp.json",

package/dist/adapters/opencode/adapter.js

@@ -29,6 +29,7 @@ exports.opencodeAdapter = {
         referenceVerification: true,
         harnessTesting: true,
         shellHooks: false,
+        subagents: true,
     },
     dialect: dialect_js_1.opencodeDialect,
     layout: layout_js_1.opencodeLayout,

package/dist/adapters/opencode/layout.js

@@ -16,6 +16,9 @@ exports.opencodeLayout = {
     // `.opencode/` segment. (Contrast Claude Code: root-level `skills/` surfaces
     // relocated under `.claude`.)
     surfaceDirs: [".opencode/agent", ".opencode/command"],
+    skillDir: ".opencode/skill",
+    agentDir: ".opencode/agent",
+    commandDir: ".opencode/command",
     materializeRoot: "",
     pluginRootToken: "${OPENCODE_PLUGIN_ROOT}",
     mcpConfigFile: "opencode.json",

package/dist/check.d.ts

@@ -104,6 +104,14 @@ export declare function turns(opts: {
 }): Check<Trace>;
 /** The agent wrote (or left) a file at this path in the work dir. */
 export declare function wrote(path: string): Check<Trace>;
+/**
+ * The agent did NOT leave a file at this path — the **side-effect boundary**
+ * negative: a skill that declares it writes only `out.txt` should leave nothing
+ * at `secrets.env`. The symmetric sibling of `wrote()`; pairs with
+ * `notTool(...)` to assert a unit stayed inside its declared write surface
+ * deterministically (no model judge).
+ */
+export declare function didNotWrite(path: string): Check<Trace>;
 /** The named subagent (`Task` `subagent_type`) ran and passed every nested check. */
 export declare function subagent(name: string, checks: readonly Check<Trace>[]): Check<Trace>;
 /** The hook blocked the event (exit 2 / deny / block). */

package/dist/check.js

@@ -11,6 +11,7 @@ exports.hookFired = hookFired;
 exports.received = received;
 exports.turns = turns;
 exports.wrote = wrote;
+exports.didNotWrite = didNotWrite;
 exports.subagent = subagent;
 exports.blocked = blocked;
 exports.allowed = allowed;
@@ -236,16 +237,34 @@ function wrote(path) {
         toJSON: () => ({ kind: "wrote", path }),
     };
 }
+/**
+ * The agent did NOT leave a file at this path — the **side-effect boundary**
+ * negative: a skill that declares it writes only `out.txt` should leave nothing
+ * at `secrets.env`. The symmetric sibling of `wrote()`; pairs with
+ * `notTool(...)` to assert a unit stayed inside its declared write surface
+ * deterministically (no model judge).
+ */
+function didNotWrite(path) {
+    return {
+        kind: "didNotWrite",
+        eval: (t) => t.file(path) === null
+            ? ok(`file "${path}" was not created`)
+            : no(`expected the agent NOT to create "${path}", but it exists`),
+        toJSON: () => ({ kind: "didNotWrite", path }),
+    };
+}
 // ---------------------------------------------------------------------------
 // Subagent — a `Task` run as a nested trace. Run checks over what the SUBAGENT
 // did, not just that `Task` fired. Composes the whole vocabulary recursively.
 // ---------------------------------------------------------------------------
-/** Wrap a subagent's tool calls as a minimal `Trace` so checks run over it. */
+/** Wrap a subagent's tool calls + returned text as a minimal `Trace` so checks
+ * (incl. `output()` over the sub's RETURN — where a result() vigiles:ok/err block
+ * lands) run over it. */
 function subTrace(sub) {
     return {
         toolCalls: sub.toolCalls,
         hooks: [],
-        output: "",
+        output: sub.output,
         modelRequests: [],
         turns: 0,
         subagents: [],
@@ -258,7 +277,12 @@ function subagent(name, checks) {
         kind: "subagent",
         eval: (t) => {
             const subs = t.subagents ?? [];
-            const sub = subs.find((s) => s.name === name);
+            // A `--plugin-dir` agent's `subagent_type` is namespaced `plugin:agent`
+            // (e.g. "reviewer-spec:code-reviewer"), but callers pass the bare agent name
+            // — so match the full id OR its last `:`-segment. Non-namespaced (harness
+            // mock) names match exactly as before.
+            const bare = (n) => n.includes(":") ? n.slice(n.lastIndexOf(":") + 1) : n;
+            const sub = subs.find((s) => s.name === name || bare(s.name) === name);
             if (!sub) {
                 return no(`expected subagent "${name}" to run; subagents that ran: ${subs.length > 0 ? `[${subs.map((s) => s.name).join(", ")}]` : "none"}`);
             }