vigiles 4.0.0 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/.claude-plugin/plugin.json +1 -1
  2. package/README.md +29 -18
  3. package/action.yml +1 -1
  4. package/dist/adapter-conformance.js +1 -1
  5. package/dist/adapters/claude-code/adapter.js +1 -1
  6. package/dist/check.d.ts +132 -0
  7. package/dist/check.js +318 -0
  8. package/dist/claude-code.d.ts +1 -1
  9. package/dist/claude-code.js +1 -1
  10. package/dist/cli.js +133 -55
  11. package/dist/core/compile.d.ts +1 -1
  12. package/dist/core/compile.js +1 -1
  13. package/dist/core/compose.d.ts +1 -1
  14. package/dist/core/compose.js +1 -1
  15. package/dist/core/generate-schema.d.ts +1 -1
  16. package/dist/core/generate-schema.js +4 -4
  17. package/dist/core/linters.js +2 -2
  18. package/dist/core/orphans.js +57 -14
  19. package/dist/core/proofs.js +1 -1
  20. package/dist/core/refs.d.ts +1 -1
  21. package/dist/core/refs.js +2 -2
  22. package/dist/core/sidecar.d.ts +1 -1
  23. package/dist/core/sidecar.js +1 -1
  24. package/dist/core/spec.d.ts +1 -1
  25. package/dist/core/spec.js +1 -1
  26. package/dist/core/types.d.ts +1 -1
  27. package/dist/core/validate.js +2 -2
  28. package/dist/e2e.d.ts +10 -13
  29. package/dist/e2e.js +10 -17
  30. package/dist/{adapters/claude-code/eval-cache.d.ts → eval-cache.d.ts} +1 -1
  31. package/dist/{adapters/claude-code/eval-cache.js → eval-cache.js} +1 -1
  32. package/dist/{adapters/claude-code/eval.d.ts → eval.d.ts} +217 -2
  33. package/dist/{adapters/claude-code/eval.js → eval.js} +430 -20
  34. package/dist/harness-assert.d.ts +11 -8
  35. package/dist/harness-assert.js +21 -5
  36. package/dist/{adapters/claude-code/harness-test.d.ts → harness-test.d.ts} +50 -4
  37. package/dist/{adapters/claude-code/harness-test.js → harness-test.js} +106 -4
  38. package/dist/integration.d.ts +11 -3
  39. package/dist/integration.js +12 -3
  40. package/dist/jest.d.ts +3 -1
  41. package/dist/jest.js +3 -2
  42. package/dist/{adapters/claude-code/mock-model.d.ts → mock-model.d.ts} +2 -2
  43. package/dist/{adapters/claude-code/run-hook.d.ts → run-hook.d.ts} +23 -1
  44. package/dist/{adapters/claude-code/run-hook.js → run-hook.js} +29 -1
  45. package/dist/{adapters/claude-code/sandbox.js → sandbox.js} +1 -1
  46. package/dist/scan.d.ts +1 -1
  47. package/dist/scan.js +1 -1
  48. package/dist/setup-plan.d.ts +5 -1
  49. package/dist/setup-plan.js +11 -1
  50. package/dist/test-coverage.js +8 -1
  51. package/dist/testing.d.ts +10 -3
  52. package/dist/testing.js +15 -3
  53. package/dist/unit.d.ts +4 -2
  54. package/dist/unit.js +8 -2
  55. package/dist/vitest.d.mts +3 -1
  56. package/hooks/refs-nudge.sh +1 -1
  57. package/hooks/session-start.sh +3 -3
  58. package/package.json +8 -7
  59. package/skills/edit-spec/SKILL.md +21 -10
  60. package/skills/linter-docs/SKILL.md +23 -0
  61. package/skills/migrate-to-spec/SKILL.md +1 -1
  62. package/skills/strengthen/SKILL.md +1 -2
  63. package/skills/generate-rule/SKILL.md +0 -64
  64. /package/dist/{adapters/claude-code/egress-entry.d.ts → egress-entry.d.ts} +0 -0
  65. /package/dist/{adapters/claude-code/egress-entry.js → egress-entry.js} +0 -0
  66. /package/dist/{adapters/claude-code/egress-proxy.d.ts → egress-proxy.d.ts} +0 -0
  67. /package/dist/{adapters/claude-code/egress-proxy.js → egress-proxy.js} +0 -0
  68. /package/dist/{adapters/claude-code/egress.d.ts → egress.d.ts} +0 -0
  69. /package/dist/{adapters/claude-code/egress.js → egress.js} +0 -0
  70. /package/dist/{adapters/claude-code/eval-baseline.d.ts → eval-baseline.d.ts} +0 -0
  71. /package/dist/{adapters/claude-code/eval-baseline.js → eval-baseline.js} +0 -0
  72. /package/dist/{adapters/claude-code/judge.d.ts → judge.d.ts} +0 -0
  73. /package/dist/{adapters/claude-code/judge.js → judge.js} +0 -0
  74. /package/dist/{adapters/claude-code/mock-entry.d.ts → mock-entry.d.ts} +0 -0
  75. /package/dist/{adapters/claude-code/mock-entry.js → mock-entry.js} +0 -0
  76. /package/dist/{adapters/claude-code/mock-model.js → mock-model.js} +0 -0
  77. /package/dist/{adapters/claude-code/sandbox.d.ts → sandbox.d.ts} +0 -0
  78. /package/dist/{adapters/claude-code/stats.d.ts → stats.d.ts} +0 -0
  79. /package/dist/{adapters/claude-code/stats.js → stats.js} +0 -0
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "vigiles",
3
- "version": "3.0.0",
3
+ "version": "4.0.1",
4
4
  "description": "Verify CLAUDE.md/AGENTS.md references, compile typed specs, and test the agent harness",
5
5
  "author": {
6
6
  "name": "zernie"
package/README.md CHANGED
@@ -100,13 +100,15 @@ hijacking unrelated ones (precision). **[Full guide →](docs/harness-testing.md
100
100
 
101
101
  ## Quick start
102
102
 
103
- > **Paste into Claude Code or Codex:**
104
- >
105
- > > Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
106
- > > references and show me what's stale, then write and run a harness test for one
107
- > > of my hooks or skills. Use good defaults (both pillars, non-interactive), but
108
- > > **ask me first** whether to gate it in CI, whether to add a real-model eval,
109
- > > and whether to enforce strictly (`--strict`).
103
+ **Paste into Claude Code or Codex:**
104
+
105
+ ```text
106
+ Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
107
+ references and show me what's stale, then write and run a harness test for one
108
+ of my hooks or skills. Use good defaults (both pillars, non-interactive), but
109
+ ask me first whether to gate it in CI, whether to add a real-model eval, and
110
+ whether to enforce strictly (--strict).
111
+ ```
110
112
 
111
113
  Or do it yourself:
112
114
 
@@ -118,21 +120,30 @@ It's interactive in a terminal and non-interactive for agents/CI (or with
118
120
  `--yes`), so "set up vigiles" from a Claude Code / Codex prompt Just Works — and
119
121
  it installs a model-invocable **`test-harness` skill**, so afterward you can just
120
122
  tell your agent _"test my skills"_ and it picks the tier and writes the test.
121
- Scope `init` with `--lint` / `--test` (one pillar or both). Or write harness
122
- tests yourself in JS **or** TS (`*.harness.{mjs,ts}`) and run `npx vigiles test`.
123
- `init` also adds `vigiles` to your `devDependencies` and installs the Claude Code
124
- plugin (skills + hooks) via the marketplace — globally, never vendored into your
125
- repo. It wires CI as a `zernie/vigiles@v1` workflow a composite over the same CLI:
126
-
127
- ```yaml
128
- - uses: actions/checkout@v4
129
- - uses: zernie/vigiles@v1 # lints by default; posts a sticky PR comment + a `valid` output
130
- ```
123
+
124
+ <details>
125
+ <summary>What <code>init</code> sets up</summary>
126
+
127
+ - **Both pillars** by default; scope with `--lint` / `--test` (one or both).
128
+ - Adds `vigiles` to your `devDependencies`.
129
+ - Installs the Claude Code plugin (skills + hooks) via the marketplace —
130
+ globally, never vendored into your repo.
131
+ - Wires CI as a `zernie/vigiles@v1` workflow (a composite over the same CLI):
132
+
133
+ ```yaml
134
+ - uses: actions/checkout@v4
135
+ - uses: zernie/vigiles@v1 # lints by default; posts a sticky PR comment + a `valid` output
136
+ ```
137
+
138
+ Prefer to write tests yourself? They can be JS **or** TS
139
+ (`*.harness.{mjs,ts}`) — run them with `npx vigiles test`.
140
+
141
+ </details>
131
142
 
132
143
  ## More
133
144
 
134
145
  - **[CLI & GitHub Action →](docs/cli.md)** — every command, the Action (inputs / output / versioning), the Claude Code plugin, and the five `lint` rules.
135
- - **[Skills →](docs/skills.md)** — consumer skills (`strengthen`, `migrate-to-spec`, `test-harness`, `edit-spec`, …) installed as a Claude Code plugin: `/plugin marketplace add zernie/vigiles` then `/plugin install vigiles@vigiles` (or let `vigiles init` do it).
146
+ - **[Skills →](docs/skills.md)** — consumer skills installed as a Claude Code plugin: `/plugin marketplace add zernie/vigiles` then `/plugin install vigiles@vigiles` (or let `vigiles init` do it). The model-invocable ones (`test-harness`, `strengthen`, `edit-spec`) fire on their own — ask _"test my skills"_, _"strengthen my rules"_, or _"add a rule to CLAUDE.md"_ and the agent reaches for them; `migrate-to-spec` and `linter-docs` are user-invoked.
136
147
  - **[Docs index →](docs/README.md)** · **[Research →](research/README.md)** · **[Related tools →](docs/related-tools.md)** (ast-grep, Dependency Cruiser, Ruler, rulesync).
137
148
  - Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
138
149
 
package/action.yml CHANGED
@@ -11,7 +11,7 @@ inputs:
11
11
  required: false
12
12
  default: "lint"
13
13
  paths:
14
- description: "Comma- or space-separated paths to operate on (.md files for audit, .spec.ts files for compile). Auto-discovers when empty."
14
+ description: "Comma- or space-separated paths to operate on (.md files for lint, .spec.ts files for compile). Auto-discovers when empty."
15
15
  required: false
16
16
  default: ""
17
17
  version:
@@ -118,7 +118,7 @@ function assertHarnessTestable(adapter) {
118
118
  if (!adapter.capabilities.harnessTesting ||
119
119
  !adapter.runtime ||
120
120
  !adapter.modelMock) {
121
- throw new Error(`Adapter "${adapter.name}" does not support harness testing (pillar 2): it is reference-verification-only (no mockable runtime). Use it for compile/scan/audit, not runHarnessTest/runEval.`);
121
+ throw new Error(`Adapter "${adapter.name}" does not support harness testing (pillar 2): it is reference-verification-only (no mockable runtime). Use it for compile/scan/lint, not runHarnessTest/runEval.`);
122
122
  }
123
123
  return { runtime: adapter.runtime, modelMock: adapter.modelMock };
124
124
  }
@@ -14,7 +14,7 @@ const layout_js_1 = require("./layout.js");
14
14
  const runtime_js_1 = require("./runtime.js");
15
15
  const hook_protocol_js_1 = require("./hook-protocol.js");
16
16
  const model_mock_js_1 = require("./model-mock.js");
17
- const harness_test_js_1 = require("./harness-test.js");
17
+ const harness_test_js_1 = require("../../harness-test.js");
18
18
  exports.claudeCodeAdapter = {
19
19
  name: "claude-code",
20
20
  // The reference harness: every tier. Mockable transport (Anthropic SSE) and
@@ -0,0 +1,132 @@
1
+ /**
2
+ * `vigiles/check` — the declarative check vocabulary (Phase 0 of the testing-API
3
+ * revamp; see `research/testing-api-design.md`).
4
+ *
5
+ * A **check** is *data*, not a throwing assertion: a small object that knows how
6
+ * to `eval` itself against a result and how to `toJSON`. One vocabulary, evaluated
7
+ * two ways downstream — strict (`expect`, throws on first fail) for the
8
+ * deterministic tiers, and scored (`measure`, 0–1 across trials) for evals — so
9
+ * `tool("Bash")` reads as pass/fail on one run AND as a rate across many. Because
10
+ * a check serializes, JSON/JUnit/baseline output and a promptfoo bridge fall out
11
+ * for free.
12
+ *
13
+ * Pure + model-free: every check is fully unit-testable without a `claude`
14
+ * subprocess. Checks over a {@link Trace} (the agent-run shape, from
15
+ * `runHarness`/`runEval`) and checks over a {@link HookRunResult} (the hook
16
+ * decision shape, from `runHook`) are distinguished at the type level, so
17
+ * `expect(result, checks)` only accepts checks that match the result.
18
+ */
19
+ import type { Trace } from "./harness-test.js";
20
+ import type { HookRunResult } from "./run-hook.js";
21
+ /** The outcome of evaluating one check against one result. */
22
+ export interface CheckResult {
23
+ /** Did the check hold? */
24
+ readonly pass: boolean;
25
+ /** 0..1 — `pass ? 1 : 0` for a boolean check; a fraction for graded ones. */
26
+ readonly score: number;
27
+ /** A human message: confirming on pass, actionable on failure. */
28
+ readonly message: string;
29
+ }
30
+ /** A check's serialized form (for JSON/JUnit output and the promptfoo bridge). */
31
+ export interface CheckJSON {
32
+ readonly kind: string;
33
+ readonly [field: string]: unknown;
34
+ }
35
+ /** A declarative check over a result of type `T` ({@link Trace} or {@link HookRunResult}). */
36
+ export interface Check<T> {
37
+ /** Discriminator, e.g. `"tool"`, `"skill"`, `"blocked"`. */
38
+ readonly kind: string;
39
+ /** Evaluate this check against a result. Pure. */
40
+ eval(target: T): CheckResult;
41
+ /** Serialize to a plain object — `expect`/`measure` use this for reports. */
42
+ toJSON(): CheckJSON;
43
+ }
44
+ /** Evaluate every check against a target. Pure — the shared core of `expect`
45
+ * (strict) and `measure` (scored). */
46
+ export declare function evalChecks<T>(target: T, checks: readonly Check<T>[]): CheckResult[];
47
+ /**
48
+ * Strict evaluator (Phase 1): throw if any check fails, collecting **all**
49
+ * failures into one actionable message (Validation-applicative, not
50
+ * short-circuit). The deterministic-tier verdict — use it in `node:test`, or any
51
+ * runner, over a `runHook` / `runHarness` result. `measure` (Phase 3) is the
52
+ * scored counterpart over trials.
53
+ */
54
+ export declare function assertChecks<T>(target: T, checks: readonly Check<T>[]): void;
55
+ /** The agent invoked a tool by this name (regardless of result). */
56
+ export declare function tool(name: string): Check<Trace>;
57
+ /** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
58
+ export declare function skill(id: string): Check<Trace>;
59
+ /** The agent's final output contains a substring / matches a RegExp. */
60
+ export declare function output(matcher: string | RegExp): Check<Trace>;
61
+ /** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
62
+ export declare function hookFired(event: string): Check<Trace>;
63
+ /**
64
+ * The model RECEIVED text matching `matcher` in some request — i.e. it actually
65
+ * reached the model. Covers **slash-command expansion** (a `commands/` file
66
+ * expands into the user prompt) and **injected context** (a SessionStart hook's
67
+ * text). Reads `modelRequests`, which the harness/mock tier captures; the eval
68
+ * tier drives the real API and captures none, so use this on `runHarness`.
69
+ */
70
+ export declare function received(matcher: string | RegExp): Check<Trace>;
71
+ /**
72
+ * The agent took a number of model turns in range — a **multi-turn** observable
73
+ * (`{ min: 2 }` asserts a back-and-forth happened, not a one-shot answer; `{ max }`
74
+ * caps runaway loops). The deterministic harness scripts the model turns; this
75
+ * checks how many the agent actually took.
76
+ */
77
+ export declare function turns(opts: {
78
+ min?: number;
79
+ max?: number;
80
+ }): Check<Trace>;
81
+ /** The agent wrote (or left) a file at this path in the work dir. */
82
+ export declare function wrote(path: string): Check<Trace>;
83
+ /** The named subagent (`Task` `subagent_type`) ran and passed every nested check. */
84
+ export declare function subagent(name: string, checks: readonly Check<Trace>[]): Check<Trace>;
85
+ /** The hook blocked the event (exit 2 / deny / block). */
86
+ export declare function blocked(): Check<HookRunResult>;
87
+ /** The hook allowed the event through. */
88
+ export declare function allowed(): Check<HookRunResult>;
89
+ /** The agent used an MCP tool `<server>/<tool>` (CC names it `mcp__server__tool`). */
90
+ export declare function mcp(server: string, toolName: string): Check<Trace>;
91
+ /** A model-graded judge: grades `output` against a `rubric` → score in [0,1]. */
92
+ export type JudgeFn = (opts: {
93
+ output: string;
94
+ rubric: string;
95
+ threshold?: number;
96
+ }) => {
97
+ score: number;
98
+ pass: boolean;
99
+ reason?: string;
100
+ };
101
+ /**
102
+ * A model-graded check: the agent's `output` scores ≥ `min` against `rubric`,
103
+ * judged by a model. Unlike the deterministic checks this one calls a model
104
+ * (cost), so it's for the scored `measure` tier; the `judge` fn is injectable
105
+ * (default: the real `judge()`), so the logic is unit-testable with a fake.
106
+ */
107
+ export declare function judged(rubric: string, opts?: {
108
+ min?: number;
109
+ judge?: JudgeFn;
110
+ }): Check<Trace>;
111
+ interface UsageTrace {
112
+ readonly usage: {
113
+ readonly costUsd: number;
114
+ readonly durationMs: number;
115
+ readonly inputTokens: number;
116
+ readonly outputTokens: number;
117
+ };
118
+ }
119
+ /** The run cost at most `maxUsd`. */
120
+ export declare function cost(opts: {
121
+ maxUsd: number;
122
+ }): Check<UsageTrace>;
123
+ /** The run took at most `maxMs` of wall-clock time. */
124
+ export declare function latency(opts: {
125
+ maxMs: number;
126
+ }): Check<UsageTrace>;
127
+ /** The run used at most `max` total (input + output) tokens. */
128
+ export declare function tokens(opts: {
129
+ max: number;
130
+ }): Check<UsageTrace>;
131
+ export {};
132
+ //# sourceMappingURL=check.d.ts.map
package/dist/check.js ADDED
@@ -0,0 +1,318 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.evalChecks = evalChecks;
4
+ exports.assertChecks = assertChecks;
5
+ exports.tool = tool;
6
+ exports.skill = skill;
7
+ exports.output = output;
8
+ exports.hookFired = hookFired;
9
+ exports.received = received;
10
+ exports.turns = turns;
11
+ exports.wrote = wrote;
12
+ exports.subagent = subagent;
13
+ exports.blocked = blocked;
14
+ exports.allowed = allowed;
15
+ exports.mcp = mcp;
16
+ exports.judged = judged;
17
+ exports.cost = cost;
18
+ exports.latency = latency;
19
+ exports.tokens = tokens;
20
+ const judge_js_1 = require("./judge.js");
21
+ /** Evaluate every check against a target. Pure — the shared core of `expect`
22
+ * (strict) and `measure` (scored). */
23
+ function evalChecks(target, checks) {
24
+ return checks.map((c) => c.eval(target));
25
+ }
26
+ /**
27
+ * Strict evaluator (Phase 1): throw if any check fails, collecting **all**
28
+ * failures into one actionable message (Validation-applicative, not
29
+ * short-circuit). The deterministic-tier verdict — use it in `node:test`, or any
30
+ * runner, over a `runHook` / `runHarness` result. `measure` (Phase 3) is the
31
+ * scored counterpart over trials.
32
+ */
33
+ function assertChecks(target, checks) {
34
+ const failures = evalChecks(target, checks).filter((r) => !r.pass);
35
+ if (failures.length > 0) {
36
+ throw new Error(`${String(failures.length)} of ${String(checks.length)} check(s) failed:\n` +
37
+ failures.map((f) => ` ✗ ${f.message}`).join("\n"));
38
+ }
39
+ }
40
+ // ---------------------------------------------------------------------------
41
+ // Internals
42
+ // ---------------------------------------------------------------------------
43
+ const ok = (message) => ({
44
+ pass: true,
45
+ score: 1,
46
+ message,
47
+ });
48
+ const no = (message) => ({
49
+ pass: false,
50
+ score: 0,
51
+ message,
52
+ });
53
+ function distinctToolNames(calls) {
54
+ const names = [...new Set(calls.map((c) => c.name))];
55
+ return names.length > 0 ? `[${names.join(", ")}]` : "no tools";
56
+ }
57
+ function truncate(s, n = 120) {
58
+ const flat = s.replace(/\s+/g, " ").trim();
59
+ return flat.length > n ? `${flat.slice(0, n)}…` : flat;
60
+ }
61
+ // ---------------------------------------------------------------------------
62
+ // Trace checks (an agent run — runHarness / runEval)
63
+ // ---------------------------------------------------------------------------
64
+ /** The agent invoked a tool by this name (regardless of result). */
65
+ function tool(name) {
66
+ return {
67
+ kind: "tool",
68
+ eval: (t) => t.toolCalls.some((c) => c.name === name)
69
+ ? ok(`agent used tool "${name}"`)
70
+ : no(`expected the agent to use tool "${name}", but it used ${distinctToolNames(t.toolCalls)}`),
71
+ toJSON: () => ({ kind: "tool", name }),
72
+ };
73
+ }
74
+ /** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
75
+ function skill(id) {
76
+ return {
77
+ kind: "skill",
78
+ eval: (t) => {
79
+ const call = t.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === id);
80
+ if (call && !call.isError)
81
+ return ok(`skill "${id}" resolved`);
82
+ const skills = t.toolCalls
83
+ .filter((c) => c.name === "Skill")
84
+ .map((c) => c.input?.skill ?? "?");
85
+ return no(`expected skill "${id}" to resolve; it did not (skills invoked: ${skills.length > 0 ? `[${skills.join(", ")}]` : "none"})`);
86
+ },
87
+ toJSON: () => ({ kind: "skill", id }),
88
+ };
89
+ }
90
+ /** The agent's final output contains a substring / matches a RegExp. */
91
+ function output(matcher) {
92
+ const isRe = matcher instanceof RegExp;
93
+ return {
94
+ kind: "output",
95
+ eval: (t) => {
96
+ const pass = isRe ? matcher.test(t.output) : t.output.includes(matcher);
97
+ return pass
98
+ ? ok(`output matched ${String(matcher)}`)
99
+ : no(`expected output to ${isRe ? "match" : "contain"} ${String(matcher)}; got "${truncate(t.output) || "(empty)"}"`);
100
+ },
101
+ toJSON: () => ({ kind: "output", matcher: String(matcher), regex: isRe }),
102
+ };
103
+ }
104
+ /** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
105
+ function hookFired(event) {
106
+ return {
107
+ kind: "hookFired",
108
+ eval: (t) => {
109
+ const fired = t.hooks.filter((h) => h.event === event);
110
+ return fired.length > 0
111
+ ? ok(`hook fired for ${event}`)
112
+ : no(`expected a hook to fire for ${event}; hooks that fired: ${t.hooks.length > 0 ? `[${[...new Set(t.hooks.map((h) => h.event))].join(", ")}]` : "none"}`);
113
+ },
114
+ toJSON: () => ({ kind: "hookFired", event }),
115
+ };
116
+ }
117
+ /**
118
+ * The model RECEIVED text matching `matcher` in some request — i.e. it actually
119
+ * reached the model. Covers **slash-command expansion** (a `commands/` file
120
+ * expands into the user prompt) and **injected context** (a SessionStart hook's
121
+ * text). Reads `modelRequests`, which the harness/mock tier captures; the eval
122
+ * tier drives the real API and captures none, so use this on `runHarness`.
123
+ */
124
+ function received(matcher) {
125
+ const isRe = matcher instanceof RegExp;
126
+ return {
127
+ kind: "received",
128
+ eval: (t) => {
129
+ const text = t.modelRequests
130
+ .map((r) => `${r.system} ${r.messages.map((m) => m.text).join(" ")}`)
131
+ .join(" ");
132
+ const pass = isRe ? matcher.test(text) : text.includes(matcher);
133
+ return pass
134
+ ? ok(`the model received ${String(matcher)}`)
135
+ : no(`expected the model to receive ${String(matcher)} (a slash-command expansion or injected context); ${t.modelRequests.length === 0 ? "no requests captured (eval tier captures none — use runHarness)" : `got "${truncate(text)}"`}`);
136
+ },
137
+ toJSON: () => ({ kind: "received", matcher: String(matcher), regex: isRe }),
138
+ };
139
+ }
140
+ /**
141
+ * The agent took a number of model turns in range — a **multi-turn** observable
142
+ * (`{ min: 2 }` asserts a back-and-forth happened, not a one-shot answer; `{ max }`
143
+ * caps runaway loops). The deterministic harness scripts the model turns; this
144
+ * checks how many the agent actually took.
145
+ */
146
+ function turns(opts) {
147
+ return {
148
+ kind: "turns",
149
+ eval: (t) => {
150
+ const n = t.turns;
151
+ const pass = (opts.min === undefined || n >= opts.min) &&
152
+ (opts.max === undefined || n <= opts.max);
153
+ const bound = [
154
+ opts.min !== undefined ? `≥ ${String(opts.min)}` : null,
155
+ opts.max !== undefined ? `≤ ${String(opts.max)}` : null,
156
+ ]
157
+ .filter((x) => x !== null)
158
+ .join(" and ");
159
+ return pass
160
+ ? ok(`${String(n)} turn(s) (${bound || "any"})`)
161
+ : no(`expected ${bound || "any"} turn(s), got ${String(n)}`);
162
+ },
163
+ toJSON: () => ({ kind: "turns", min: opts.min, max: opts.max }),
164
+ };
165
+ }
166
+ /** The agent wrote (or left) a file at this path in the work dir. */
167
+ function wrote(path) {
168
+ return {
169
+ kind: "wrote",
170
+ eval: (t) => t.file(path) !== null
171
+ ? ok(`file "${path}" exists`)
172
+ : no(`expected the agent to create "${path}", but it does not exist`),
173
+ toJSON: () => ({ kind: "wrote", path }),
174
+ };
175
+ }
176
+ // ---------------------------------------------------------------------------
177
+ // Subagent — a `Task` run as a nested trace. Run checks over what the SUBAGENT
178
+ // did, not just that `Task` fired. Composes the whole vocabulary recursively.
179
+ // ---------------------------------------------------------------------------
180
+ /** Wrap a subagent's tool calls as a minimal `Trace` so checks run over it. */
181
+ function subTrace(sub) {
182
+ return {
183
+ toolCalls: sub.toolCalls,
184
+ hooks: [],
185
+ output: "",
186
+ modelRequests: [],
187
+ turns: 0,
188
+ subagents: [],
189
+ file: () => null,
190
+ };
191
+ }
192
+ /** The named subagent (`Task` `subagent_type`) ran and passed every nested check. */
193
+ function subagent(name, checks) {
194
+ return {
195
+ kind: "subagent",
196
+ eval: (t) => {
197
+ const subs = t.subagents ?? [];
198
+ const sub = subs.find((s) => s.name === name);
199
+ if (!sub) {
200
+ return no(`expected subagent "${name}" to run; subagents that ran: ${subs.length > 0 ? `[${subs.map((s) => s.name).join(", ")}]` : "none"}`);
201
+ }
202
+ const failures = checks
203
+ .map((c) => c.eval(subTrace(sub)))
204
+ .filter((r) => !r.pass);
205
+ return failures.length === 0
206
+ ? ok(`subagent "${name}" passed ${String(checks.length)} check(s)`)
207
+ : no(`subagent "${name}": ${failures.map((f) => f.message).join("; ")}`);
208
+ },
209
+ toJSON: () => ({
210
+ kind: "subagent",
211
+ name,
212
+ checks: checks.map((c) => c.toJSON()),
213
+ }),
214
+ };
215
+ }
216
+ // ---------------------------------------------------------------------------
217
+ // Hook-decision checks (a single hook — runHook)
218
+ // ---------------------------------------------------------------------------
219
+ /** The hook blocked the event (exit 2 / deny / block). */
220
+ function blocked() {
221
+ return {
222
+ kind: "blocked",
223
+ eval: (r) => r.blocked
224
+ ? ok("hook blocked the event")
225
+ : no(`expected the hook to block, but it allowed (exit ${String(r.exitCode)})`),
226
+ toJSON: () => ({ kind: "blocked" }),
227
+ };
228
+ }
229
+ /** The hook allowed the event through. */
230
+ function allowed() {
231
+ return {
232
+ kind: "allowed",
233
+ eval: (r) => r.blocked
234
+ ? no(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)})`)
235
+ : ok("hook allowed the event"),
236
+ toJSON: () => ({ kind: "allowed" }),
237
+ };
238
+ }
239
+ // ---------------------------------------------------------------------------
240
+ // MCP — a tool call is just a name in the Trace; MCP tools are `mcp__srv__tool`
241
+ // ---------------------------------------------------------------------------
242
+ /** The agent used an MCP tool `<server>/<tool>` (CC names it `mcp__server__tool`). */
243
+ function mcp(server, toolName) {
244
+ const full = `mcp__${server}__${toolName}`;
245
+ return {
246
+ kind: "mcp",
247
+ eval: (t) => t.toolCalls.some((c) => c.name === full)
248
+ ? ok(`used MCP tool ${server}/${toolName}`)
249
+ : no(`expected MCP tool ${server}/${toolName} (${full}), but the agent used ${distinctToolNames(t.toolCalls)}`),
250
+ toJSON: () => ({ kind: "mcp", server, tool: toolName }),
251
+ };
252
+ }
253
+ /**
254
+ * A model-graded check: the agent's `output` scores ≥ `min` against `rubric`,
255
+ * judged by a model. Unlike the deterministic checks this one calls a model
256
+ * (cost), so it's for the scored `measure` tier; the `judge` fn is injectable
257
+ * (default: the real `judge()`), so the logic is unit-testable with a fake.
258
+ */
259
+ function judged(rubric, opts = {}) {
260
+ const min = opts.min ?? 0.5;
261
+ const judgeFn = opts.judge ?? ((o) => (0, judge_js_1.judge)(o));
262
+ return {
263
+ kind: "judged",
264
+ eval: (t) => {
265
+ const r = judgeFn({ output: t.output, rubric, threshold: min });
266
+ const pass = r.score >= min;
267
+ const tail = r.reason ? ` — ${r.reason}` : "";
268
+ return {
269
+ pass,
270
+ score: r.score,
271
+ message: pass
272
+ ? `judge ${r.score.toFixed(2)} ≥ ${String(min)}${tail}`
273
+ : `judge ${r.score.toFixed(2)} < ${String(min)} for "${rubric}"${tail}`,
274
+ };
275
+ },
276
+ toJSON: () => ({ kind: "judged", rubric, min }),
277
+ };
278
+ }
279
+ /** The run cost at most `maxUsd`. */
280
+ function cost(opts) {
281
+ return {
282
+ kind: "cost",
283
+ eval: (t) => {
284
+ const v = t.usage.costUsd;
285
+ return v <= opts.maxUsd
286
+ ? ok(`cost $${v.toFixed(4)} ≤ $${String(opts.maxUsd)}`)
287
+ : no(`expected cost ≤ $${String(opts.maxUsd)}, got $${v.toFixed(4)}`);
288
+ },
289
+ toJSON: () => ({ kind: "cost", maxUsd: opts.maxUsd }),
290
+ };
291
+ }
292
+ /** The run took at most `maxMs` of wall-clock time. */
293
+ function latency(opts) {
294
+ return {
295
+ kind: "latency",
296
+ eval: (t) => {
297
+ const v = t.usage.durationMs;
298
+ return v <= opts.maxMs
299
+ ? ok(`latency ${String(v)}ms ≤ ${String(opts.maxMs)}ms`)
300
+ : no(`expected latency ≤ ${String(opts.maxMs)}ms, got ${String(v)}ms`);
301
+ },
302
+ toJSON: () => ({ kind: "latency", maxMs: opts.maxMs }),
303
+ };
304
+ }
305
+ /** The run used at most `max` total (input + output) tokens. */
306
+ function tokens(opts) {
307
+ return {
308
+ kind: "tokens",
309
+ eval: (t) => {
310
+ const v = t.usage.inputTokens + t.usage.outputTokens;
311
+ return v <= opts.max
312
+ ? ok(`${String(v)} tokens ≤ ${String(opts.max)}`)
313
+ : no(`expected ≤ ${String(opts.max)} tokens, got ${String(v)}`);
314
+ },
315
+ toJSON: () => ({ kind: "tokens", max: opts.max }),
316
+ };
317
+ }
318
+ //# sourceMappingURL=check.js.map
@@ -6,7 +6,7 @@
6
6
  * is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
7
7
  */
8
8
  export * from "./adapters/claude-code/plugin-loader.js";
9
- export * from "./adapters/claude-code/mock-model.js";
9
+ export * from "./mock-model.js";
10
10
  export * from "./adapters/claude-code/dialect.js";
11
11
  export * from "./adapters/claude-code/layout.js";
12
12
  export * from "./adapters/claude-code/runtime.js";
@@ -22,7 +22,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
22
22
  * is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
23
23
  */
24
24
  __exportStar(require("./adapters/claude-code/plugin-loader.js"), exports);
25
- __exportStar(require("./adapters/claude-code/mock-model.js"), exports);
25
+ __exportStar(require("./mock-model.js"), exports);
26
26
  __exportStar(require("./adapters/claude-code/dialect.js"), exports);
27
27
  __exportStar(require("./adapters/claude-code/layout.js"), exports);
28
28
  __exportStar(require("./adapters/claude-code/runtime.js"), exports);