vigiles 4.0.1 → 4.0.2

package/dist/eval.js

@@ -1,6 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runEval = runEval;
+exports.measureWith = measureWith;
+exports.measure = measure;
+exports.measureArmsWith = measureArmsWith;
+exports.measureArms = measureArms;
+exports.compareCheck = compareCheck;
+exports.formatCheckReport = formatCheckReport;
+exports.assertRates = assertRates;
+exports.checkReportToJUnit = checkReportToJUnit;
 exports.parseUsage = parseUsage;
 exports.aggregate = aggregate;
 exports.aggregateStats = aggregateStats;
@@ -9,6 +17,12 @@ exports.isRateLimited = isRateLimited;
 exports.runPool = runPool;
 exports.runEvalWith = runEvalWith;
 exports.formatEvalReport = formatEvalReport;
+exports.packageSkillsDir = packageSkillsDir;
+exports.stubSkillBody = stubSkillBody;
+exports.stubbedPluginDir = stubbedPluginDir;
+exports.promptDistance = promptDistance;
+exports.checkPromptDiversity = checkPromptDiversity;
+exports.assertPromptDiversity = assertPromptDiversity;
 exports.measureTriggerRateWith = measureTriggerRateWith;
 exports.measureTriggerRate = measureTriggerRate;
 exports.formatTriggerRateReport = formatTriggerRateReport;
@@ -41,8 +55,10 @@ const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
 const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
 const runtime_js_1 = require("./adapters/claude-code/runtime.js");
+const proofs_js_1 = require("./core/proofs.js");
 const harness_test_js_1 = require("./harness-test.js");
 const eval_cache_js_1 = require("./eval-cache.js");
+const stats_js_1 = require("./stats.js");
 function writeFiles(cwd, files) {
     for (const [p, content] of Object.entries(files)) {
         const full = (0, node_path_1.resolve)(cwd, p);
@@ -98,7 +114,207 @@ function spawnAgent(a) {
 async function runEval(spec) {
     return runEvalWith(spec, spawnAgent);
 }
+/**
+ * Score a check vocabulary across trials — the scored counterpart to
+ * `assertChecks` (strict). Each check yields a `rate ± se` and `pass^k` over `n`
+ * runs. Reuses the tested `runEvalWith` aggregation (one arm), so the loop,
+ * cache, concurrency, and stats come for free. Exported with an injectable
+ * `runner` so the orchestration is unit-testable without a model.
+ */
+async function measureWith(spec, runner) {
+    if (spec.stubSkillBodies && !spec.pluginDir)
+        throw new Error("measure: `stubSkillBodies` requires `pluginDir`.");
+    const stubbed = spec.stubSkillBodies
+        ? stubbedPluginDir(spec.pluginDir)
+        : undefined;
+    const pluginDir = stubbed ?? spec.pluginDir;
+    try {
+        const keyed = spec.checks.map((c, i) => [`c${String(i)}`, c]);
+        const report = await runEvalWith({
+            fixture: spec.fixture,
+            arms: {
+                run: {
+                    settings: spec.settings,
+                    plugin: spec.plugin,
+                    pluginDir,
+                },
+            },
+            task: spec.task,
+            trials: spec.trials ?? 5,
+            model: spec.model ?? "sonnet",
+            allowedTools: spec.allowedTools,
+            timeoutMs: spec.timeoutMs,
+            spacingSec: spec.spacingSec,
+            measure: (ctx) => Object.fromEntries(keyed.map(([k, c]) => [k, c.eval(ctx).pass])),
+        }, runner);
+        const arm = report.arms.run;
+        return {
+            n: arm?.runs ?? 0,
+            perCheck: keyed.map(([k, c]) => {
+                const s = arm?.stats[k];
+                return {
+                    check: c.toJSON(),
+                    rate: s?.mean ?? 0,
+                    se: s?.se ?? 0,
+                    passK: s?.passK ?? 0,
+                    n: s?.n ?? 0,
+                };
+            }),
+        };
+    }
+    finally {
+        if (stubbed)
+            (0, node_fs_1.rmSync)(stubbed, { recursive: true, force: true });
+    }
+}
+/* v8 ignore start -- real claude subprocess; thin wrapper over measureWith */
+/** Score a check vocabulary across trials against the real `claude` CLI. */
+async function measure(spec) {
+    return measureWith(spec, spawnAgent);
+}
+/** Score checks across arms (injectable runner). Reuses `runEvalWith`. */
+async function measureArmsWith(spec, runner) {
+    const keyed = spec.checks.map((c, i) => [`c${String(i)}`, c]);
+    const { arms: runArms, temps } = spec.stubSkillBodies
+        ? stubArmPluginDirs(spec.arms)
+        : { arms: spec.arms, temps: [] };
+    try {
+        const report = await runEvalWith({
+            fixture: spec.fixture,
+            arms: runArms,
+            task: spec.task,
+            trials: spec.trials ?? 5,
+            model: spec.model ?? "sonnet",
+            allowedTools: spec.allowedTools,
+            timeoutMs: spec.timeoutMs,
+            spacingSec: spec.spacingSec,
+            measure: (ctx) => Object.fromEntries(keyed.map(([k, c]) => [k, c.eval(ctx).pass])),
+        }, runner);
+        const arms = {};
+        for (const [armName, arm] of Object.entries(report.arms)) {
+            arms[armName] = {
+                n: arm.runs,
+                perCheck: keyed.map(([k, c]) => {
+                    const s = arm.stats[k];
+                    return {
+                        check: c.toJSON(),
+                        rate: s?.mean ?? 0,
+                        se: s?.se ?? 0,
+                        passK: s?.passK ?? 0,
+                        n: s?.n ?? 0,
+                    };
+                }),
+            };
+        }
+        return { arms };
+    }
+    finally {
+        for (const t of temps)
+            (0, node_fs_1.rmSync)(t, { recursive: true, force: true });
+    }
+}
+/**
+ * Repackage every arm that sets a `pluginDir` with its skill bodies stubbed
+ * (frontmatter kept), for an A/B firing comparison. Returns the rewritten arms
+ * plus the throwaway dirs the caller must remove. Arms without a `pluginDir` pass
+ * through unchanged. See {@link stubbedPluginDir}.
+ */
+function stubArmPluginDirs(arms) {
+    const out = {};
+    const temps = [];
+    for (const [name, arm] of Object.entries(arms)) {
+        if (arm.pluginDir) {
+            const stubbed = stubbedPluginDir(arm.pluginDir);
+            temps.push(stubbed);
+            out[name] = { ...arm, pluginDir: stubbed };
+        }
+        else {
+            out[name] = arm;
+        }
+    }
+    return { arms: out, temps };
+}
+/* v8 ignore start -- real claude subprocess; thin wrapper over measureArmsWith */
+/** Score checks across arms against the real `claude` CLI. */
+async function measureArms(spec) {
+    return measureArmsWith(spec, spawnAgent);
+}
 /* v8 ignore stop */
+/**
+ * Welch significance on one check's rate between two arms (`arm` vs `baseline`),
+ * by index in `perCheck`. So "the gated arm resolves the skill significantly more
+ * than vanilla" is a p-value, not a vibe. Reuses `welchTTest` from stats.ts.
+ */
+function compareCheck(report, baseline, arm, checkIndex) {
+    const b = report.arms[baseline]?.perCheck[checkIndex];
+    const a = report.arms[arm]?.perCheck[checkIndex];
+    if (!a || !b) {
+        throw new Error(`compareCheck: unknown arm or check index (baseline="${baseline}", arm="${arm}", i=${String(checkIndex)})`);
+    }
+    return (0, stats_js_1.welchTTest)({ mean: a.rate, se: a.se, n: a.n }, { mean: b.rate, se: b.se, n: b.n });
+}
+/** A readable label for a check from its serialized form, e.g. `tool(Bash)`. */
+function checkLabel(json) {
+    const arg = json.name ?? json.id ?? json.event ?? json.path ?? json.matcher;
+    if (typeof arg === "string" ||
+        typeof arg === "number" ||
+        typeof arg === "boolean") {
+        return `${json.kind}(${String(arg)})`;
+    }
+    return json.kind;
+}
+/** Format a {@link CheckReport}: one line per check with its rate ± se and pass^k. */
+function formatCheckReport(report) {
+    const lines = [`measured ${String(report.n)} run(s):`];
+    for (const c of report.perCheck) {
+        lines.push(`  ${(c.rate * 100).toFixed(0)}% ± ${(c.se * 100).toFixed(0)}%  ${checkLabel(c.check)}` +
+            `  (pass^k ${String(c.passK)})`);
+    }
+    return lines.join("\n");
+}
+/**
+ * The scored gate (Phase 4): throw if any check's measured rate is below `min` —
+ * the `measure` counterpart to `assertChecks` (strict). Reads the rate, not a
+ * single run, so it never trips on one noisy trial.
+ */
+function assertRates(report, opts) {
+    const below = report.perCheck.filter((c) => c.rate < opts.min);
+    if (below.length > 0) {
+        throw new Error(`${String(below.length)} check(s) below the ${(opts.min * 100).toFixed(0)}% min rate:\n` +
+            below
+                .map((c) => `  ✗ ${checkLabel(c.check)}: ${(c.rate * 100).toFixed(0)}% ± ${(c.se * 100).toFixed(0)}%`)
+                .join("\n"));
+    }
+}
+function escapeXml(s) {
+    return s
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+/**
+ * Serialize a {@link CheckReport} to JUnit XML (Phase 4) — each check a
+ * `<testcase>`, failing when its rate is below `min`. Because a check is *data*,
+ * this falls out for free: CI test reporters, regression baselines, and a
+ * promptfoo bridge all consume the same shape.
+ */
+function checkReportToJUnit(report, opts = {}) {
+    const min = opts.min ?? 0;
+    const failures = report.perCheck.filter((c) => c.rate < min).length;
+    const cases = report.perCheck
+        .map((c) => {
+        const name = escapeXml(checkLabel(c.check));
+        const body = c.rate < min
+            ? `\n    <failure message="rate ${(c.rate * 100).toFixed(0)}% below min ${(min * 100).toFixed(0)}% (n=${String(c.n)})"/>\n  `
+            : "";
+        return `  <testcase classname="vigiles.checks" name="${name}">${body}</testcase>`;
+    })
+        .join("\n");
+    return (`<?xml version="1.0" encoding="UTF-8"?>\n` +
+        `<testsuite name="${escapeXml(opts.name ?? "vigiles measure")}" tests="${String(report.perCheck.length)}" failures="${String(failures)}">\n` +
+        `${cases}\n</testsuite>\n`);
+}
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 /** Pull cost / latency / tokens out of a parsed `result` event (0 when absent). */
 function usageFrom(result) {
@@ -127,6 +343,7 @@ function makeContext(cwd, out) {
         toolCalls: (0, harness_test_js_1.parseToolCalls)(out.stdout),
         hooks: (0, harness_test_js_1.parseHooks)(out.stdout),
         output,
+        subagents: (0, harness_test_js_1.parseSubagents)(out.stdout),
         usage: usageFrom(result),
         // The eval tier drives the real API (no mock between claude and the model),
         // so the requests can't be captured here — modelRequests is harness-tier only.
@@ -144,7 +361,7 @@ function makeContext(cwd, out) {
                 }).trim();
             }
             catch (e) {
-                // Return captured stdout even on a non-zero exit (e.g. `audit` exits 2
+                // Return captured stdout even on a non-zero exit (e.g. `lint` exits 2
                 // but still prints its findings), rather than swallowing it.
                 const out = e.stdout;
                 return typeof out === "string" ? out.trim() : "";
@@ -402,6 +619,178 @@ function formatEvalReport(report) {
     }
     return lines.join("\n");
 }
+/**
+ * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
+ * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
+ * can be trigger-tested without hand-rolling a `plugin.json`. Writes a minimal
+ * `.claude-plugin/plugin.json` and copies each `<name>/` (recursively, so
+ * `references/` etc. come along) under `skills/<name>/`. Returns the temp plugin
+ * dir; the caller removes it (`measureTriggerRate` does). Throws if the directory
+ * is missing or holds no `<name>/SKILL.md`.
+ */
+function packageSkillsDir(skillsDir, opts = {}) {
+    const abs = (0, node_path_1.resolve)(skillsDir);
+    if (!(0, node_fs_1.existsSync)(abs))
+        throw new Error(`skillsDir not found: ${skillsDir} (resolved ${abs})`);
+    const root = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-skills-"));
+    (0, node_fs_1.mkdirSync)((0, node_path_1.join)(root, ".claude-plugin"), { recursive: true });
+    (0, node_fs_1.writeFileSync)((0, node_path_1.join)(root, ".claude-plugin", "plugin.json"), JSON.stringify({ name: opts.name ?? "vigiles-loose-skills", version: "0.0.0" }, null, 2));
+    const skillsOut = (0, node_path_1.join)(root, "skills");
+    (0, node_fs_1.mkdirSync)(skillsOut, { recursive: true });
+    let copied = 0;
+    for (const entry of (0, node_fs_1.readdirSync)(abs, { withFileTypes: true })) {
+        if (!entry.isDirectory())
+            continue;
+        const srcSkill = (0, node_path_1.join)(abs, entry.name, "SKILL.md");
+        if (!(0, node_fs_1.existsSync)(srcSkill))
+            continue;
+        const destDir = (0, node_path_1.join)(skillsOut, entry.name);
+        if (opts.stub) {
+            // Frontmatter-only: keep the trigger surface (name + description), drop the
+            // body so a selected skill stops instead of running its procedure.
+            (0, node_fs_1.mkdirSync)(destDir, { recursive: true });
+            (0, node_fs_1.writeFileSync)((0, node_path_1.join)(destDir, "SKILL.md"), stubSkillBody((0, node_fs_1.readFileSync)(srcSkill, "utf-8")));
+        }
+        else {
+            (0, node_fs_1.cpSync)((0, node_path_1.join)(abs, entry.name), destDir, { recursive: true });
+        }
+        copied++;
+    }
+    if (copied === 0) {
+        (0, node_fs_1.rmSync)(root, { recursive: true, force: true });
+        throw new Error(`No <name>/SKILL.md skills found under ${skillsDir}`);
+    }
+    return root;
+}
+/**
+ * Rewrite a SKILL.md to keep its YAML frontmatter (the trigger surface — name +
+ * description) but replace the body with a no-op stub. Trigger-rate is a property
+ * of the frontmatter ONLY: the model picks a skill from its name + description
+ * before the body is ever loaded, so the body is causally downstream of selection
+ * and irrelevant to whether the skill fires. Stubbing it lets a trigger run stop
+ * AT selection instead of executing an expensive multi-step procedure — cheaper,
+ * faster, and side-effect-free, without changing what's measured. Pure.
+ */
+function stubSkillBody(skillMd) {
+    const m = /^(---\n[\s\S]*?\n---\n)/.exec(skillMd);
+    const frontmatter = m ? m[1] : "";
+    return `${frontmatter}\nThis skill was selected (trigger-test stub). Acknowledge and stop — do not perform any actions.\n`;
+}
+/** The skills directory inside a plugin (`skills/` or `.claude/skills/`). */
+function skillsDirOf(pluginDir) {
+    const direct = (0, node_path_1.join)(pluginDir, "skills");
+    if ((0, node_fs_1.existsSync)(direct))
+        return direct;
+    return (0, node_path_1.join)(pluginDir, ".claude", "skills");
+}
+/** A plugin's declared name (from `.claude-plugin/plugin.json`), for the skill
+ * id the `fired` predicate matches (`<name>:<skill>`). */
+function pluginName(pluginDir) {
+    const manifest = (0, node_path_1.join)(pluginDir, ".claude-plugin", "plugin.json");
+    try {
+        return JSON.parse((0, node_fs_1.readFileSync)(manifest, "utf-8"))
+            .name;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Build a throwaway plugin dir mirroring `pluginDir`'s skills with their BODIES
+ * stripped (frontmatter kept) — the trigger surface a description/firing check
+ * needs, without paying to run each skill's procedure. Keeps the original plugin
+ * NAME so `<name>:<skill>` ids still match. The caller removes the returned dir.
+ * See {@link stubSkillBody} for why the body is irrelevant to selection.
+ */
+function stubbedPluginDir(pluginDir) {
+    return packageSkillsDir(skillsDirOf(pluginDir), {
+        stub: true,
+        name: pluginName(pluginDir),
+    });
+}
+// ---------------------------------------------------------------------------
+// Prompt-set diversity (deterministic, pre-eval) — a trigger rate is only
+// meaningful over ENOUGH and DIFFERENT prompts. Catch a too-small or
+// near-duplicate set before spending a single model token. We measure
+// "different" with Normalized Compression Distance (gzip) — the SAME engine
+// `findSimilarRules` uses for near-duplicate rule detection — not edit
+// distance: NCD scores shared structure/redundancy (a templated prompt with one
+// word swapped compresses together), which is exactly the lazy-copy-paste set
+// we want to reject, and it's the project's house algorithm for "are these two
+// texts basically the same".
+// ---------------------------------------------------------------------------
+/** Normalize for comparison: lowercase, trim, collapse whitespace. */
+function normalizePrompt(s) {
+    return s.toLowerCase().trim().replace(/\s+/g, " ");
+}
+/**
+ * Distance between two prompts in ~0..1 (0 = identical, higher = more
+ * different) via Normalized Compression Distance over the normalized text.
+ * Reuses {@link ncd} from the proof engine.
+ */
+function promptDistance(a, b) {
+    return (0, proofs_js_1.ncd)(normalizePrompt(a), normalizePrompt(b));
+}
+/**
+ * Deterministically check a prompt set is big and varied enough to measure a
+ * trigger rate: at least `minPrompts` entries, and no two closer than
+ * `minDistance` in NCD. Pure — no model. `label` names the set in messages.
+ */
+function checkPromptDiversity(prompts, opts = {}) {
+    const minPrompts = opts.minPrompts ?? 10;
+    const minDistance = opts.minDistance ?? 0.3;
+    const label = opts.label ?? "prompts";
+    const issues = [];
+    if (prompts.length < minPrompts) {
+        issues.push({
+            kind: "too-few",
+            message: `${label}: ${String(prompts.length)} prompt(s), need at least ${String(minPrompts)} to measure a rate (set minPrompts to override).`,
+        });
+    }
+    for (let i = 0; i < prompts.length; i++) {
+        for (let j = i + 1; j < prompts.length; j++) {
+            const dist = promptDistance(prompts[i], prompts[j]);
+            if (dist < minDistance) {
+                issues.push({
+                    kind: "too-similar",
+                    message: `${label}: prompts #${String(i + 1)} and #${String(j + 1)} are near-duplicates (NCD ${dist.toFixed(2)} < ${String(minDistance)}) — vary the phrasing:\n    - ${prompts[i]}\n    - ${prompts[j]}`,
+                });
+            }
+        }
+    }
+    return issues;
+}
+/** Throw if a prompt set isn't big/varied enough. See {@link checkPromptDiversity}. */
+function assertPromptDiversity(prompts, opts = {}) {
+    const issues = checkPromptDiversity(prompts, opts);
+    if (issues.length > 0) {
+        throw new Error(`Prompt set is not eval-ready:\n  ${issues.map((i) => i.message).join("\n  ")}`);
+    }
+}
+/**
+ * Resolve the effective `--plugin-dir` for a trigger run: a caller's `pluginDir`
+ * as-is, or a throwaway package built from a loose `skillsDir`. Exactly one must
+ * be set. `packaged` is present only when vigiles built it, so the caller knows
+ * to remove it afterward.
+ */
+function resolveTriggerPluginDir(spec) {
+    if (spec.pluginDir && spec.skillsDir)
+        throw new Error("measureTriggerRate: set `pluginDir` OR `skillsDir`, not both.");
+    const stub = spec.stubSkillBodies ?? false;
+    if (spec.skillsDir) {
+        const packaged = packageSkillsDir(spec.skillsDir, { stub });
+        return { pluginDir: packaged, packaged };
+    }
+    if (spec.pluginDir) {
+        if (!stub)
+            return { pluginDir: spec.pluginDir };
+        // Stub a real plugin: build a minimal plugin from its skills/ with bodies
+        // stripped — keep the original plugin NAME so `<name>:<skill>` still matches.
+        const packaged = stubbedPluginDir(spec.pluginDir);
+        return { pluginDir: packaged, packaged };
+    }
+    throw new Error("measureTriggerRate: provide `pluginDir` or `skillsDir`.");
+}
 /** Run one prompt set × trials through `runner`, aggregating fired counts. */
 async function runTriggerSet(prompts, cfg, runner) {
     const perPrompt = [];
@@ -449,31 +838,52 @@ async function runTriggerSet(prompts, cfg, runner) {
  * `measureTriggerRate` is this with the real agent runner.
  */
 async function measureTriggerRateWith(spec, runner) {
+    // Deterministic gate FIRST — reject a too-small / near-duplicate prompt set
+    // before spending a token (and before packaging a skillsDir).
+    const diversity = {
+        minPrompts: spec.minPrompts,
+        minDistance: spec.minDistance,
+    };
+    assertPromptDiversity(spec.prompts, { ...diversity, label: "prompts" });
+    if (spec.irrelevantPrompts && spec.irrelevantPrompts.length > 0) {
+        assertPromptDiversity(spec.irrelevantPrompts, {
+            ...diversity,
+            label: "irrelevantPrompts",
+        });
+    }
+    const { pluginDir, packaged } = resolveTriggerPluginDir(spec);
     const cfg = {
         trials: spec.trials ?? 1,
         model: spec.model ?? "haiku",
         tools: spec.allowedTools ?? ["Read", "Edit", "Write", "Bash", "Skill"],
         timeoutMs: spec.timeoutMs ?? 240000,
         spacing: (spec.spacingSec ?? 4) * 1000,
-        pluginDir: spec.pluginDir,
+        pluginDir,
         fired: spec.fired,
     };
-    const relevant = await runTriggerSet(spec.prompts, cfg, runner);
-    const base = {
-        rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
-        n: relevant.n,
-        perPrompt: relevant.perPrompt,
-    };
-    if ((spec.irrelevantPrompts?.length ?? 0) === 0)
-        return base;
-    const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
-    const fires = relevant.fired + irrelevant.fired;
-    return {
-        ...base,
-        falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
-        precision: fires > 0 ? relevant.fired / fires : undefined,
-        perIrrelevant: irrelevant.perPrompt,
-    };
+    try {
+        const relevant = await runTriggerSet(spec.prompts, cfg, runner);
+        const base = {
+            rate: relevant.n > 0 ? relevant.fired / relevant.n : 0,
+            n: relevant.n,
+            perPrompt: relevant.perPrompt,
+        };
+        if ((spec.irrelevantPrompts?.length ?? 0) === 0)
+            return base;
+        const irrelevant = await runTriggerSet(spec.irrelevantPrompts ?? [], cfg, runner);
+        const fires = relevant.fired + irrelevant.fired;
+        return {
+            ...base,
+            falsePositiveRate: irrelevant.n > 0 ? irrelevant.fired / irrelevant.n : 0,
+            precision: fires > 0 ? relevant.fired / fires : undefined,
+            perIrrelevant: irrelevant.perPrompt,
+        };
+    }
+    finally {
+        // Remove the throwaway plugin dir we built from a loose `skillsDir`.
+        if (packaged)
+            (0, node_fs_1.rmSync)(packaged, { recursive: true, force: true });
+    }
 }
 /* v8 ignore start -- real claude subprocess; thin wrapper over measureTriggerRateWith */
 /**

package/dist/harness-assert.d.ts

@@ -16,6 +16,7 @@
 import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
 import type { EvalReport, TriggerRateReport } from "./eval.js";
 import type { HookRunResult, EgressAttempt } from "./run-hook.js";
+import { type Check } from "./check.js";
 import type { OutputContract } from "./core/spec.js";
 import { type ParsedAgentResult } from "./adapters/claude-code/agent-result.js";
 import { type BaselineFile, type DiffOptions } from "./eval-baseline.js";
@@ -282,5 +283,7 @@ export declare const vigilesMatchers: {
     toHaveCreated(received: HarnessTestResult, path: string): MatcherOutput;
     toBlock(received: HookRunResult): MatcherOutput;
     toBeatBaseline(received: EvalReport, baseline: string, arm: string, metric: string, by?: number): MatcherOutput;
+    toPass<T>(received: T, check: Check<T>): MatcherOutput;
+    toPassAll<T>(received: T, checks: readonly Check<T>[]): MatcherOutput;
 };
 //# sourceMappingURL=harness-assert.d.ts.map

package/dist/harness-assert.js

@@ -58,6 +58,7 @@ exports.assertTriggerRate = assertTriggerRate;
  *     vitest and jest, so the same object supports both.
  */
 const harness_test_js_1 = require("./harness-test.js");
+const check_js_1 = require("./check.js");
 const agent_result_js_1 = require("./adapters/claude-code/agent-result.js");
 const stats_js_1 = require("./stats.js");
 const eval_baseline_js_1 = require("./eval-baseline.js");
@@ -576,5 +577,20 @@ exports.vigilesMatchers = {
             message: () => `expected ${arm} ${pass ? "not " : ""}to beat ${baseline} on ${metric} by > ${String(by)} (got ${delta.toFixed(3)})`,
         };
     },
+    // The check-vocabulary veneer (Phase 1): ONE matcher works for EVERY check,
+    // carrying the check's own failure message. `expect(r).toPass(tool("Bash"))`.
+    toPass(received, check) {
+        const r = check.eval(received);
+        return { pass: r.pass, message: () => r.message };
+    },
+    toPassAll(received, checks) {
+        const failed = (0, check_js_1.evalChecks)(received, checks).filter((x) => !x.pass);
+        return {
+            pass: failed.length === 0,
+            message: () => failed.length === 0
+                ? "all checks passed"
+                : failed.map((f) => `  ✗ ${f.message}`).join("\n"),
+        };
+    },
 };
 //# sourceMappingURL=harness-assert.js.map

package/dist/harness-test.d.ts

@@ -93,9 +93,24 @@ export interface Trace {
     readonly modelRequests: readonly ModelRequest[];
     /** Number of model turns. */
     readonly turns: number;
+    /**
+     * Sub-agent (`Task`) runs as nested traces, keyed by `subagent_type`. A
+     * subagent runs its own session; CC tags its events with `parent_tool_use_id`
+     * (= the `Task` tool call) so its tool calls are recovered into a sub-trace
+     * here, lettng a test assert what the subagent DID (not just that `Task` fired).
+     * Empty unless the stream was captured / the harness emits subagent events.
+     */
+    readonly subagents?: readonly SubagentTrace[];
     /** Final contents of a file under the working dir, or null if absent. */
     file(path: string): string | null;
 }
+/** A sub-agent (`Task`) run as a nested trace: its name + the tools it used. */
+export interface SubagentTrace {
+    /** The `subagent_type` from the `Task` tool input. */
+    readonly name: string;
+    /** The tools the subagent invoked (events tagged with the Task's id). */
+    readonly toolCalls: readonly ToolCall[];
+}
 export interface HarnessTestResult extends Trace {
     readonly exitCode: number;
     readonly stdout: string;
@@ -115,6 +130,19 @@ export interface HarnessTestResult extends Trace {
  * actions, not a brittle stdout substring.
  */
 export declare function parseToolCalls(streamJson: string): ToolCall[];
+/**
+ * Recover sub-agent runs as nested traces. A subagent-dispatch tool call (the
+ * `Agent` tool on the live CLI — older docs say `Task` — carrying an
+ * `input.subagent_type`) spawns a subagent whose own events the CLI tags with a
+ * top-level `parent_tool_use_id` = the dispatch tool-use id. We group those
+ * tagged tool calls under their dispatch, keyed by `subagent_type`. **Schema
+ * verified against real claude output** (`parent_tool_use_id` sibling of
+ * `message`, `subagent_type` in the dispatch input; tool named `Agent`) — the
+ * same `message.content` line shape `parseToolCalls` consumes, and we match the
+ * input field NOT the tool name so a future rename can't break it. Pure; empty
+ * for a harness that doesn't emit `parent_tool_use_id` (e.g. Codex).
+ */
+export declare function parseSubagents(streamJson: string): SubagentTrace[];
 /**
  * The terminal `result` event — present in BOTH `--output-format` shapes (a
  * `{type:"result", …}` line in stream-json, the single object in `json`), or
@@ -163,4 +191,22 @@ export interface RunHarnessTestOptions {
  * only — requesting confinement for another harness throws.
  */
 export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise<HarnessTestResult>;
+/**
+ * `runHarness` — the harness-scope entry of the revamped API (Phase 2 of
+ * `research/testing-api-design.md`). The harness has two execution scopes, `hook`
+ * (`runHook`) and `harness` (the whole assembled agent); today's `integration` /
+ * `e2e` / `eval` are all the **harness** scope under realness flags. This entry is
+ * the **deterministic** harness run (`model: "mock"`, the default) — the
+ * workhorse you gate every commit, with no key. A **real-model** harness run is
+ * non-deterministic by definition, so you don't *assert* a single one — you
+ * `measure()` it across trials (the eval scope). `egress` is a capability of this
+ * scope (the e2e tier), not a separate tier.
+ *
+ * Behaviour is identical to `runHarnessTest` (which it wraps); the new name +
+ * `model` flag make the scope/realness explicit and steer real-model runs to the
+ * right tool.
+ */
+export declare function runHarness(spec: HarnessTestSpec, opts?: RunHarnessTestOptions & {
+    model?: "mock" | "real";
+}): Promise<HarnessTestResult>;
 //# sourceMappingURL=harness-test.d.ts.map