vigiles 4.0.0 → 4.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +29 -18
- package/action.yml +1 -1
- package/dist/adapter-conformance.js +1 -1
- package/dist/adapters/claude-code/adapter.js +1 -1
- package/dist/check.d.ts +132 -0
- package/dist/check.js +318 -0
- package/dist/claude-code.d.ts +1 -1
- package/dist/claude-code.js +1 -1
- package/dist/cli.js +133 -55
- package/dist/core/compile.d.ts +1 -1
- package/dist/core/compile.js +1 -1
- package/dist/core/compose.d.ts +1 -1
- package/dist/core/compose.js +1 -1
- package/dist/core/generate-schema.d.ts +1 -1
- package/dist/core/generate-schema.js +4 -4
- package/dist/core/linters.js +2 -2
- package/dist/core/orphans.js +57 -14
- package/dist/core/proofs.js +1 -1
- package/dist/core/refs.d.ts +1 -1
- package/dist/core/refs.js +2 -2
- package/dist/core/sidecar.d.ts +1 -1
- package/dist/core/sidecar.js +1 -1
- package/dist/core/spec.d.ts +1 -1
- package/dist/core/spec.js +1 -1
- package/dist/core/types.d.ts +1 -1
- package/dist/core/validate.js +2 -2
- package/dist/e2e.d.ts +10 -13
- package/dist/e2e.js +10 -17
- package/dist/{adapters/claude-code/eval-cache.d.ts → eval-cache.d.ts} +1 -1
- package/dist/{adapters/claude-code/eval-cache.js → eval-cache.js} +1 -1
- package/dist/{adapters/claude-code/eval.d.ts → eval.d.ts} +217 -2
- package/dist/{adapters/claude-code/eval.js → eval.js} +430 -20
- package/dist/harness-assert.d.ts +11 -8
- package/dist/harness-assert.js +21 -5
- package/dist/{adapters/claude-code/harness-test.d.ts → harness-test.d.ts} +50 -4
- package/dist/{adapters/claude-code/harness-test.js → harness-test.js} +106 -4
- package/dist/integration.d.ts +11 -3
- package/dist/integration.js +12 -3
- package/dist/jest.d.ts +3 -1
- package/dist/jest.js +3 -2
- package/dist/{adapters/claude-code/mock-model.d.ts → mock-model.d.ts} +2 -2
- package/dist/{adapters/claude-code/run-hook.d.ts → run-hook.d.ts} +23 -1
- package/dist/{adapters/claude-code/run-hook.js → run-hook.js} +29 -1
- package/dist/{adapters/claude-code/sandbox.js → sandbox.js} +1 -1
- package/dist/scan.d.ts +1 -1
- package/dist/scan.js +1 -1
- package/dist/setup-plan.d.ts +5 -1
- package/dist/setup-plan.js +11 -1
- package/dist/test-coverage.js +8 -1
- package/dist/testing.d.ts +10 -3
- package/dist/testing.js +15 -3
- package/dist/unit.d.ts +4 -2
- package/dist/unit.js +8 -2
- package/dist/vitest.d.mts +3 -1
- package/hooks/refs-nudge.sh +1 -1
- package/hooks/session-start.sh +3 -3
- package/package.json +8 -7
- package/skills/edit-spec/SKILL.md +21 -10
- package/skills/linter-docs/SKILL.md +23 -0
- package/skills/migrate-to-spec/SKILL.md +1 -1
- package/skills/strengthen/SKILL.md +1 -2
- package/skills/generate-rule/SKILL.md +0 -64
- /package/dist/{adapters/claude-code/egress-entry.d.ts → egress-entry.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress-entry.js → egress-entry.js} +0 -0
- /package/dist/{adapters/claude-code/egress-proxy.d.ts → egress-proxy.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress-proxy.js → egress-proxy.js} +0 -0
- /package/dist/{adapters/claude-code/egress.d.ts → egress.d.ts} +0 -0
- /package/dist/{adapters/claude-code/egress.js → egress.js} +0 -0
- /package/dist/{adapters/claude-code/eval-baseline.d.ts → eval-baseline.d.ts} +0 -0
- /package/dist/{adapters/claude-code/eval-baseline.js → eval-baseline.js} +0 -0
- /package/dist/{adapters/claude-code/judge.d.ts → judge.d.ts} +0 -0
- /package/dist/{adapters/claude-code/judge.js → judge.js} +0 -0
- /package/dist/{adapters/claude-code/mock-entry.d.ts → mock-entry.d.ts} +0 -0
- /package/dist/{adapters/claude-code/mock-entry.js → mock-entry.js} +0 -0
- /package/dist/{adapters/claude-code/mock-model.js → mock-model.js} +0 -0
- /package/dist/{adapters/claude-code/sandbox.d.ts → sandbox.d.ts} +0 -0
- /package/dist/{adapters/claude-code/stats.d.ts → stats.d.ts} +0 -0
- /package/dist/{adapters/claude-code/stats.js → stats.js} +0 -0
package/README.md
CHANGED
|
@@ -100,13 +100,15 @@ hijacking unrelated ones (precision). **[Full guide →](docs/harness-testing.md
|
|
|
100
100
|
|
|
101
101
|
## Quick start
|
|
102
102
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
**Paste into Claude Code or Codex:**
|
|
104
|
+
|
|
105
|
+
```text
|
|
106
|
+
Install vigiles in this repo and run it. Verify my CLAUDE.md / AGENTS.md
|
|
107
|
+
references and show me what's stale, then write and run a harness test for one
|
|
108
|
+
of my hooks or skills. Use good defaults (both pillars, non-interactive), but
|
|
109
|
+
ask me first whether to gate it in CI, whether to add a real-model eval, and
|
|
110
|
+
whether to enforce strictly (--strict).
|
|
111
|
+
```
|
|
110
112
|
|
|
111
113
|
Or do it yourself:
|
|
112
114
|
|
|
@@ -118,21 +120,30 @@ It's interactive in a terminal and non-interactive for agents/CI (or with
|
|
|
118
120
|
`--yes`), so "set up vigiles" from a Claude Code / Codex prompt Just Works — and
|
|
119
121
|
it installs a model-invocable **`test-harness` skill**, so afterward you can just
|
|
120
122
|
tell your agent _"test my skills"_ and it picks the tier and writes the test.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
-
|
|
130
|
-
|
|
123
|
+
|
|
124
|
+
<details>
|
|
125
|
+
<summary>What <code>init</code> sets up</summary>
|
|
126
|
+
|
|
127
|
+
- **Both pillars** by default; scope with `--lint` / `--test` (one or both).
|
|
128
|
+
- Adds `vigiles` to your `devDependencies`.
|
|
129
|
+
- Installs the Claude Code plugin (skills + hooks) via the marketplace —
|
|
130
|
+
globally, never vendored into your repo.
|
|
131
|
+
- Wires CI as a `zernie/vigiles@v1` workflow (a composite over the same CLI):
|
|
132
|
+
|
|
133
|
+
```yaml
|
|
134
|
+
- uses: actions/checkout@v4
|
|
135
|
+
- uses: zernie/vigiles@v1 # lints by default; posts a sticky PR comment + a `valid` output
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Prefer to write tests yourself? They can be JS **or** TS
|
|
139
|
+
(`*.harness.{mjs,ts}`) — run them with `npx vigiles test`.
|
|
140
|
+
|
|
141
|
+
</details>
|
|
131
142
|
|
|
132
143
|
## More
|
|
133
144
|
|
|
134
145
|
- **[CLI & GitHub Action →](docs/cli.md)** — every command, the Action (inputs / output / versioning), the Claude Code plugin, and the five `lint` rules.
|
|
135
|
-
- **[Skills →](docs/skills.md)** — consumer skills
|
|
146
|
+
- **[Skills →](docs/skills.md)** — consumer skills installed as a Claude Code plugin: `/plugin marketplace add zernie/vigiles` then `/plugin install vigiles@vigiles` (or let `vigiles init` do it). The model-invocable ones (`test-harness`, `strengthen`, `edit-spec`) fire on their own — ask _"test my skills"_, _"strengthen my rules"_, or _"add a rule to CLAUDE.md"_ and the agent reaches for them; `migrate-to-spec` and `linter-docs` are user-invoked.
|
|
136
147
|
- **[Docs index →](docs/README.md)** · **[Research →](research/README.md)** · **[Related tools →](docs/related-tools.md)** (ast-grep, Dependency Cruiser, Ruler, rulesync).
|
|
137
148
|
- Companion to [Feedback Loop Is All You Need](https://zernie.com/blog/feedback-loop-is-all-you-need).
|
|
138
149
|
|
package/action.yml
CHANGED
|
@@ -11,7 +11,7 @@ inputs:
|
|
|
11
11
|
required: false
|
|
12
12
|
default: "lint"
|
|
13
13
|
paths:
|
|
14
|
-
description: "Comma- or space-separated paths to operate on (.md files for
|
|
14
|
+
description: "Comma- or space-separated paths to operate on (.md files for lint, .spec.ts files for compile). Auto-discovers when empty."
|
|
15
15
|
required: false
|
|
16
16
|
default: ""
|
|
17
17
|
version:
|
|
@@ -118,7 +118,7 @@ function assertHarnessTestable(adapter) {
|
|
|
118
118
|
if (!adapter.capabilities.harnessTesting ||
|
|
119
119
|
!adapter.runtime ||
|
|
120
120
|
!adapter.modelMock) {
|
|
121
|
-
throw new Error(`Adapter "${adapter.name}" does not support harness testing (pillar 2): it is reference-verification-only (no mockable runtime). Use it for compile/scan/
|
|
121
|
+
throw new Error(`Adapter "${adapter.name}" does not support harness testing (pillar 2): it is reference-verification-only (no mockable runtime). Use it for compile/scan/lint, not runHarnessTest/runEval.`);
|
|
122
122
|
}
|
|
123
123
|
return { runtime: adapter.runtime, modelMock: adapter.modelMock };
|
|
124
124
|
}
|
|
@@ -14,7 +14,7 @@ const layout_js_1 = require("./layout.js");
|
|
|
14
14
|
const runtime_js_1 = require("./runtime.js");
|
|
15
15
|
const hook_protocol_js_1 = require("./hook-protocol.js");
|
|
16
16
|
const model_mock_js_1 = require("./model-mock.js");
|
|
17
|
-
const harness_test_js_1 = require("
|
|
17
|
+
const harness_test_js_1 = require("../../harness-test.js");
|
|
18
18
|
exports.claudeCodeAdapter = {
|
|
19
19
|
name: "claude-code",
|
|
20
20
|
// The reference harness: every tier. Mockable transport (Anthropic SSE) and
|
package/dist/check.d.ts
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vigiles/check` — the declarative check vocabulary (Phase 0 of the testing-API
|
|
3
|
+
* revamp; see `research/testing-api-design.md`).
|
|
4
|
+
*
|
|
5
|
+
* A **check** is *data*, not a throwing assertion: a small object that knows how
|
|
6
|
+
* to `eval` itself against a result and how to `toJSON`. One vocabulary, evaluated
|
|
7
|
+
* two ways downstream — strict (`expect`, throws on first fail) for the
|
|
8
|
+
* deterministic tiers, and scored (`measure`, 0–1 across trials) for evals — so
|
|
9
|
+
* `tool("Bash")` reads as pass/fail on one run AND as a rate across many. Because
|
|
10
|
+
* a check serializes, JSON/JUnit/baseline output and a promptfoo bridge fall out
|
|
11
|
+
* for free.
|
|
12
|
+
*
|
|
13
|
+
* Pure + model-free: every check is fully unit-testable without a `claude`
|
|
14
|
+
* subprocess. Checks over a {@link Trace} (the agent-run shape, from
|
|
15
|
+
* `runHarness`/`runEval`) and checks over a {@link HookRunResult} (the hook
|
|
16
|
+
* decision shape, from `runHook`) are distinguished at the type level, so
|
|
17
|
+
* `expect(result, checks)` only accepts checks that match the result.
|
|
18
|
+
*/
|
|
19
|
+
import type { Trace } from "./harness-test.js";
|
|
20
|
+
import type { HookRunResult } from "./run-hook.js";
|
|
21
|
+
/** The outcome of evaluating one check against one result. */
|
|
22
|
+
export interface CheckResult {
|
|
23
|
+
/** Did the check hold? */
|
|
24
|
+
readonly pass: boolean;
|
|
25
|
+
/** 0..1 — `pass ? 1 : 0` for a boolean check; a fraction for graded ones. */
|
|
26
|
+
readonly score: number;
|
|
27
|
+
/** A human message: confirming on pass, actionable on failure. */
|
|
28
|
+
readonly message: string;
|
|
29
|
+
}
|
|
30
|
+
/** A check's serialized form (for JSON/JUnit output and the promptfoo bridge). */
|
|
31
|
+
export interface CheckJSON {
|
|
32
|
+
readonly kind: string;
|
|
33
|
+
readonly [field: string]: unknown;
|
|
34
|
+
}
|
|
35
|
+
/** A declarative check over a result of type `T` ({@link Trace} or {@link HookRunResult}). */
|
|
36
|
+
export interface Check<T> {
|
|
37
|
+
/** Discriminator, e.g. `"tool"`, `"skill"`, `"blocked"`. */
|
|
38
|
+
readonly kind: string;
|
|
39
|
+
/** Evaluate this check against a result. Pure. */
|
|
40
|
+
eval(target: T): CheckResult;
|
|
41
|
+
/** Serialize to a plain object — `expect`/`measure` use this for reports. */
|
|
42
|
+
toJSON(): CheckJSON;
|
|
43
|
+
}
|
|
44
|
+
/** Evaluate every check against a target. Pure — the shared core of `expect`
|
|
45
|
+
* (strict) and `measure` (scored). */
|
|
46
|
+
export declare function evalChecks<T>(target: T, checks: readonly Check<T>[]): CheckResult[];
|
|
47
|
+
/**
|
|
48
|
+
* Strict evaluator (Phase 1): throw if any check fails, collecting **all**
|
|
49
|
+
* failures into one actionable message (Validation-applicative, not
|
|
50
|
+
* short-circuit). The deterministic-tier verdict — use it in `node:test`, or any
|
|
51
|
+
* runner, over a `runHook` / `runHarness` result. `measure` (Phase 3) is the
|
|
52
|
+
* scored counterpart over trials.
|
|
53
|
+
*/
|
|
54
|
+
export declare function assertChecks<T>(target: T, checks: readonly Check<T>[]): void;
|
|
55
|
+
/** The agent invoked a tool by this name (regardless of result). */
|
|
56
|
+
export declare function tool(name: string): Check<Trace>;
|
|
57
|
+
/** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
|
|
58
|
+
export declare function skill(id: string): Check<Trace>;
|
|
59
|
+
/** The agent's final output contains a substring / matches a RegExp. */
|
|
60
|
+
export declare function output(matcher: string | RegExp): Check<Trace>;
|
|
61
|
+
/** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
|
|
62
|
+
export declare function hookFired(event: string): Check<Trace>;
|
|
63
|
+
/**
|
|
64
|
+
* The model RECEIVED text matching `matcher` in some request — i.e. it actually
|
|
65
|
+
* reached the model. Covers **slash-command expansion** (a `commands/` file
|
|
66
|
+
* expands into the user prompt) and **injected context** (a SessionStart hook's
|
|
67
|
+
* text). Reads `modelRequests`, which the harness/mock tier captures; the eval
|
|
68
|
+
* tier drives the real API and captures none, so use this on `runHarness`.
|
|
69
|
+
*/
|
|
70
|
+
export declare function received(matcher: string | RegExp): Check<Trace>;
|
|
71
|
+
/**
|
|
72
|
+
* The agent took a number of model turns in range — a **multi-turn** observable
|
|
73
|
+
* (`{ min: 2 }` asserts a back-and-forth happened, not a one-shot answer; `{ max }`
|
|
74
|
+
* caps runaway loops). The deterministic harness scripts the model turns; this
|
|
75
|
+
* checks how many the agent actually took.
|
|
76
|
+
*/
|
|
77
|
+
export declare function turns(opts: {
|
|
78
|
+
min?: number;
|
|
79
|
+
max?: number;
|
|
80
|
+
}): Check<Trace>;
|
|
81
|
+
/** The agent wrote (or left) a file at this path in the work dir. */
|
|
82
|
+
export declare function wrote(path: string): Check<Trace>;
|
|
83
|
+
/** The named subagent (`Task` `subagent_type`) ran and passed every nested check. */
|
|
84
|
+
export declare function subagent(name: string, checks: readonly Check<Trace>[]): Check<Trace>;
|
|
85
|
+
/** The hook blocked the event (exit 2 / deny / block). */
|
|
86
|
+
export declare function blocked(): Check<HookRunResult>;
|
|
87
|
+
/** The hook allowed the event through. */
|
|
88
|
+
export declare function allowed(): Check<HookRunResult>;
|
|
89
|
+
/** The agent used an MCP tool `<server>/<tool>` (CC names it `mcp__server__tool`). */
|
|
90
|
+
export declare function mcp(server: string, toolName: string): Check<Trace>;
|
|
91
|
+
/** A model-graded judge: grades `output` against a `rubric` → score in [0,1]. */
|
|
92
|
+
export type JudgeFn = (opts: {
|
|
93
|
+
output: string;
|
|
94
|
+
rubric: string;
|
|
95
|
+
threshold?: number;
|
|
96
|
+
}) => {
|
|
97
|
+
score: number;
|
|
98
|
+
pass: boolean;
|
|
99
|
+
reason?: string;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* A model-graded check: the agent's `output` scores ≥ `min` against `rubric`,
|
|
103
|
+
* judged by a model. Unlike the deterministic checks this one calls a model
|
|
104
|
+
* (cost), so it's for the scored `measure` tier; the `judge` fn is injectable
|
|
105
|
+
* (default: the real `judge()`), so the logic is unit-testable with a fake.
|
|
106
|
+
*/
|
|
107
|
+
export declare function judged(rubric: string, opts?: {
|
|
108
|
+
min?: number;
|
|
109
|
+
judge?: JudgeFn;
|
|
110
|
+
}): Check<Trace>;
|
|
111
|
+
interface UsageTrace {
|
|
112
|
+
readonly usage: {
|
|
113
|
+
readonly costUsd: number;
|
|
114
|
+
readonly durationMs: number;
|
|
115
|
+
readonly inputTokens: number;
|
|
116
|
+
readonly outputTokens: number;
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/** The run cost at most `maxUsd`. */
|
|
120
|
+
export declare function cost(opts: {
|
|
121
|
+
maxUsd: number;
|
|
122
|
+
}): Check<UsageTrace>;
|
|
123
|
+
/** The run took at most `maxMs` of wall-clock time. */
|
|
124
|
+
export declare function latency(opts: {
|
|
125
|
+
maxMs: number;
|
|
126
|
+
}): Check<UsageTrace>;
|
|
127
|
+
/** The run used at most `max` total (input + output) tokens. */
|
|
128
|
+
export declare function tokens(opts: {
|
|
129
|
+
max: number;
|
|
130
|
+
}): Check<UsageTrace>;
|
|
131
|
+
export {};
|
|
132
|
+
//# sourceMappingURL=check.d.ts.map
|
package/dist/check.js
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.evalChecks = evalChecks;
|
|
4
|
+
exports.assertChecks = assertChecks;
|
|
5
|
+
exports.tool = tool;
|
|
6
|
+
exports.skill = skill;
|
|
7
|
+
exports.output = output;
|
|
8
|
+
exports.hookFired = hookFired;
|
|
9
|
+
exports.received = received;
|
|
10
|
+
exports.turns = turns;
|
|
11
|
+
exports.wrote = wrote;
|
|
12
|
+
exports.subagent = subagent;
|
|
13
|
+
exports.blocked = blocked;
|
|
14
|
+
exports.allowed = allowed;
|
|
15
|
+
exports.mcp = mcp;
|
|
16
|
+
exports.judged = judged;
|
|
17
|
+
exports.cost = cost;
|
|
18
|
+
exports.latency = latency;
|
|
19
|
+
exports.tokens = tokens;
|
|
20
|
+
const judge_js_1 = require("./judge.js");
|
|
21
|
+
/** Evaluate every check against a target. Pure — the shared core of `expect`
|
|
22
|
+
* (strict) and `measure` (scored). */
|
|
23
|
+
function evalChecks(target, checks) {
|
|
24
|
+
return checks.map((c) => c.eval(target));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Strict evaluator (Phase 1): throw if any check fails, collecting **all**
|
|
28
|
+
* failures into one actionable message (Validation-applicative, not
|
|
29
|
+
* short-circuit). The deterministic-tier verdict — use it in `node:test`, or any
|
|
30
|
+
* runner, over a `runHook` / `runHarness` result. `measure` (Phase 3) is the
|
|
31
|
+
* scored counterpart over trials.
|
|
32
|
+
*/
|
|
33
|
+
function assertChecks(target, checks) {
|
|
34
|
+
const failures = evalChecks(target, checks).filter((r) => !r.pass);
|
|
35
|
+
if (failures.length > 0) {
|
|
36
|
+
throw new Error(`${String(failures.length)} of ${String(checks.length)} check(s) failed:\n` +
|
|
37
|
+
failures.map((f) => ` ✗ ${f.message}`).join("\n"));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
// Internals
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
const ok = (message) => ({
|
|
44
|
+
pass: true,
|
|
45
|
+
score: 1,
|
|
46
|
+
message,
|
|
47
|
+
});
|
|
48
|
+
const no = (message) => ({
|
|
49
|
+
pass: false,
|
|
50
|
+
score: 0,
|
|
51
|
+
message,
|
|
52
|
+
});
|
|
53
|
+
function distinctToolNames(calls) {
|
|
54
|
+
const names = [...new Set(calls.map((c) => c.name))];
|
|
55
|
+
return names.length > 0 ? `[${names.join(", ")}]` : "no tools";
|
|
56
|
+
}
|
|
57
|
+
function truncate(s, n = 120) {
|
|
58
|
+
const flat = s.replace(/\s+/g, " ").trim();
|
|
59
|
+
return flat.length > n ? `${flat.slice(0, n)}…` : flat;
|
|
60
|
+
}
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
// Trace checks (an agent run — runHarness / runEval)
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
/** The agent invoked a tool by this name (regardless of result). */
|
|
65
|
+
function tool(name) {
|
|
66
|
+
return {
|
|
67
|
+
kind: "tool",
|
|
68
|
+
eval: (t) => t.toolCalls.some((c) => c.name === name)
|
|
69
|
+
? ok(`agent used tool "${name}"`)
|
|
70
|
+
: no(`expected the agent to use tool "${name}", but it used ${distinctToolNames(t.toolCalls)}`),
|
|
71
|
+
toJSON: () => ({ kind: "tool", name }),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/** A skill resolved to this id (`<plugin>:<skill>`) without erroring. */
|
|
75
|
+
function skill(id) {
|
|
76
|
+
return {
|
|
77
|
+
kind: "skill",
|
|
78
|
+
eval: (t) => {
|
|
79
|
+
const call = t.toolCalls.find((c) => c.name === "Skill" && c.input?.skill === id);
|
|
80
|
+
if (call && !call.isError)
|
|
81
|
+
return ok(`skill "${id}" resolved`);
|
|
82
|
+
const skills = t.toolCalls
|
|
83
|
+
.filter((c) => c.name === "Skill")
|
|
84
|
+
.map((c) => c.input?.skill ?? "?");
|
|
85
|
+
return no(`expected skill "${id}" to resolve; it did not (skills invoked: ${skills.length > 0 ? `[${skills.join(", ")}]` : "none"})`);
|
|
86
|
+
},
|
|
87
|
+
toJSON: () => ({ kind: "skill", id }),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/** The agent's final output contains a substring / matches a RegExp. */
|
|
91
|
+
function output(matcher) {
|
|
92
|
+
const isRe = matcher instanceof RegExp;
|
|
93
|
+
return {
|
|
94
|
+
kind: "output",
|
|
95
|
+
eval: (t) => {
|
|
96
|
+
const pass = isRe ? matcher.test(t.output) : t.output.includes(matcher);
|
|
97
|
+
return pass
|
|
98
|
+
? ok(`output matched ${String(matcher)}`)
|
|
99
|
+
: no(`expected output to ${isRe ? "match" : "contain"} ${String(matcher)}; got "${truncate(t.output) || "(empty)"}"`);
|
|
100
|
+
},
|
|
101
|
+
toJSON: () => ({ kind: "output", matcher: String(matcher), regex: isRe }),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
|
|
105
|
+
function hookFired(event) {
|
|
106
|
+
return {
|
|
107
|
+
kind: "hookFired",
|
|
108
|
+
eval: (t) => {
|
|
109
|
+
const fired = t.hooks.filter((h) => h.event === event);
|
|
110
|
+
return fired.length > 0
|
|
111
|
+
? ok(`hook fired for ${event}`)
|
|
112
|
+
: no(`expected a hook to fire for ${event}; hooks that fired: ${t.hooks.length > 0 ? `[${[...new Set(t.hooks.map((h) => h.event))].join(", ")}]` : "none"}`);
|
|
113
|
+
},
|
|
114
|
+
toJSON: () => ({ kind: "hookFired", event }),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* The model RECEIVED text matching `matcher` in some request — i.e. it actually
|
|
119
|
+
* reached the model. Covers **slash-command expansion** (a `commands/` file
|
|
120
|
+
* expands into the user prompt) and **injected context** (a SessionStart hook's
|
|
121
|
+
* text). Reads `modelRequests`, which the harness/mock tier captures; the eval
|
|
122
|
+
* tier drives the real API and captures none, so use this on `runHarness`.
|
|
123
|
+
*/
|
|
124
|
+
function received(matcher) {
|
|
125
|
+
const isRe = matcher instanceof RegExp;
|
|
126
|
+
return {
|
|
127
|
+
kind: "received",
|
|
128
|
+
eval: (t) => {
|
|
129
|
+
const text = t.modelRequests
|
|
130
|
+
.map((r) => `${r.system} ${r.messages.map((m) => m.text).join(" ")}`)
|
|
131
|
+
.join(" ");
|
|
132
|
+
const pass = isRe ? matcher.test(text) : text.includes(matcher);
|
|
133
|
+
return pass
|
|
134
|
+
? ok(`the model received ${String(matcher)}`)
|
|
135
|
+
: no(`expected the model to receive ${String(matcher)} (a slash-command expansion or injected context); ${t.modelRequests.length === 0 ? "no requests captured (eval tier captures none — use runHarness)" : `got "${truncate(text)}"`}`);
|
|
136
|
+
},
|
|
137
|
+
toJSON: () => ({ kind: "received", matcher: String(matcher), regex: isRe }),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The agent took a number of model turns in range — a **multi-turn** observable
|
|
142
|
+
* (`{ min: 2 }` asserts a back-and-forth happened, not a one-shot answer; `{ max }`
|
|
143
|
+
* caps runaway loops). The deterministic harness scripts the model turns; this
|
|
144
|
+
* checks how many the agent actually took.
|
|
145
|
+
*/
|
|
146
|
+
function turns(opts) {
|
|
147
|
+
return {
|
|
148
|
+
kind: "turns",
|
|
149
|
+
eval: (t) => {
|
|
150
|
+
const n = t.turns;
|
|
151
|
+
const pass = (opts.min === undefined || n >= opts.min) &&
|
|
152
|
+
(opts.max === undefined || n <= opts.max);
|
|
153
|
+
const bound = [
|
|
154
|
+
opts.min !== undefined ? `≥ ${String(opts.min)}` : null,
|
|
155
|
+
opts.max !== undefined ? `≤ ${String(opts.max)}` : null,
|
|
156
|
+
]
|
|
157
|
+
.filter((x) => x !== null)
|
|
158
|
+
.join(" and ");
|
|
159
|
+
return pass
|
|
160
|
+
? ok(`${String(n)} turn(s) (${bound || "any"})`)
|
|
161
|
+
: no(`expected ${bound || "any"} turn(s), got ${String(n)}`);
|
|
162
|
+
},
|
|
163
|
+
toJSON: () => ({ kind: "turns", min: opts.min, max: opts.max }),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/** The agent wrote (or left) a file at this path in the work dir. */
|
|
167
|
+
function wrote(path) {
|
|
168
|
+
return {
|
|
169
|
+
kind: "wrote",
|
|
170
|
+
eval: (t) => t.file(path) !== null
|
|
171
|
+
? ok(`file "${path}" exists`)
|
|
172
|
+
: no(`expected the agent to create "${path}", but it does not exist`),
|
|
173
|
+
toJSON: () => ({ kind: "wrote", path }),
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
// ---------------------------------------------------------------------------
|
|
177
|
+
// Subagent — a `Task` run as a nested trace. Run checks over what the SUBAGENT
|
|
178
|
+
// did, not just that `Task` fired. Composes the whole vocabulary recursively.
|
|
179
|
+
// ---------------------------------------------------------------------------
|
|
180
|
+
/** Wrap a subagent's tool calls as a minimal `Trace` so checks run over it. */
|
|
181
|
+
function subTrace(sub) {
|
|
182
|
+
return {
|
|
183
|
+
toolCalls: sub.toolCalls,
|
|
184
|
+
hooks: [],
|
|
185
|
+
output: "",
|
|
186
|
+
modelRequests: [],
|
|
187
|
+
turns: 0,
|
|
188
|
+
subagents: [],
|
|
189
|
+
file: () => null,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
/** The named subagent (`Task` `subagent_type`) ran and passed every nested check. */
|
|
193
|
+
function subagent(name, checks) {
|
|
194
|
+
return {
|
|
195
|
+
kind: "subagent",
|
|
196
|
+
eval: (t) => {
|
|
197
|
+
const subs = t.subagents ?? [];
|
|
198
|
+
const sub = subs.find((s) => s.name === name);
|
|
199
|
+
if (!sub) {
|
|
200
|
+
return no(`expected subagent "${name}" to run; subagents that ran: ${subs.length > 0 ? `[${subs.map((s) => s.name).join(", ")}]` : "none"}`);
|
|
201
|
+
}
|
|
202
|
+
const failures = checks
|
|
203
|
+
.map((c) => c.eval(subTrace(sub)))
|
|
204
|
+
.filter((r) => !r.pass);
|
|
205
|
+
return failures.length === 0
|
|
206
|
+
? ok(`subagent "${name}" passed ${String(checks.length)} check(s)`)
|
|
207
|
+
: no(`subagent "${name}": ${failures.map((f) => f.message).join("; ")}`);
|
|
208
|
+
},
|
|
209
|
+
toJSON: () => ({
|
|
210
|
+
kind: "subagent",
|
|
211
|
+
name,
|
|
212
|
+
checks: checks.map((c) => c.toJSON()),
|
|
213
|
+
}),
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
// ---------------------------------------------------------------------------
|
|
217
|
+
// Hook-decision checks (a single hook — runHook)
|
|
218
|
+
// ---------------------------------------------------------------------------
|
|
219
|
+
/** The hook blocked the event (exit 2 / deny / block). */
|
|
220
|
+
function blocked() {
|
|
221
|
+
return {
|
|
222
|
+
kind: "blocked",
|
|
223
|
+
eval: (r) => r.blocked
|
|
224
|
+
? ok("hook blocked the event")
|
|
225
|
+
: no(`expected the hook to block, but it allowed (exit ${String(r.exitCode)})`),
|
|
226
|
+
toJSON: () => ({ kind: "blocked" }),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
/** The hook allowed the event through. */
|
|
230
|
+
function allowed() {
|
|
231
|
+
return {
|
|
232
|
+
kind: "allowed",
|
|
233
|
+
eval: (r) => r.blocked
|
|
234
|
+
? no(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)})`)
|
|
235
|
+
: ok("hook allowed the event"),
|
|
236
|
+
toJSON: () => ({ kind: "allowed" }),
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
// ---------------------------------------------------------------------------
|
|
240
|
+
// MCP — a tool call is just a name in the Trace; MCP tools are `mcp__srv__tool`
|
|
241
|
+
// ---------------------------------------------------------------------------
|
|
242
|
+
/** The agent used an MCP tool `<server>/<tool>` (CC names it `mcp__server__tool`). */
|
|
243
|
+
function mcp(server, toolName) {
|
|
244
|
+
const full = `mcp__${server}__${toolName}`;
|
|
245
|
+
return {
|
|
246
|
+
kind: "mcp",
|
|
247
|
+
eval: (t) => t.toolCalls.some((c) => c.name === full)
|
|
248
|
+
? ok(`used MCP tool ${server}/${toolName}`)
|
|
249
|
+
: no(`expected MCP tool ${server}/${toolName} (${full}), but the agent used ${distinctToolNames(t.toolCalls)}`),
|
|
250
|
+
toJSON: () => ({ kind: "mcp", server, tool: toolName }),
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* A model-graded check: the agent's `output` scores ≥ `min` against `rubric`,
|
|
255
|
+
* judged by a model. Unlike the deterministic checks this one calls a model
|
|
256
|
+
* (cost), so it's for the scored `measure` tier; the `judge` fn is injectable
|
|
257
|
+
* (default: the real `judge()`), so the logic is unit-testable with a fake.
|
|
258
|
+
*/
|
|
259
|
+
function judged(rubric, opts = {}) {
|
|
260
|
+
const min = opts.min ?? 0.5;
|
|
261
|
+
const judgeFn = opts.judge ?? ((o) => (0, judge_js_1.judge)(o));
|
|
262
|
+
return {
|
|
263
|
+
kind: "judged",
|
|
264
|
+
eval: (t) => {
|
|
265
|
+
const r = judgeFn({ output: t.output, rubric, threshold: min });
|
|
266
|
+
const pass = r.score >= min;
|
|
267
|
+
const tail = r.reason ? ` — ${r.reason}` : "";
|
|
268
|
+
return {
|
|
269
|
+
pass,
|
|
270
|
+
score: r.score,
|
|
271
|
+
message: pass
|
|
272
|
+
? `judge ${r.score.toFixed(2)} ≥ ${String(min)}${tail}`
|
|
273
|
+
: `judge ${r.score.toFixed(2)} < ${String(min)} for "${rubric}"${tail}`,
|
|
274
|
+
};
|
|
275
|
+
},
|
|
276
|
+
toJSON: () => ({ kind: "judged", rubric, min }),
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
/** The run cost at most `maxUsd`. */
|
|
280
|
+
function cost(opts) {
|
|
281
|
+
return {
|
|
282
|
+
kind: "cost",
|
|
283
|
+
eval: (t) => {
|
|
284
|
+
const v = t.usage.costUsd;
|
|
285
|
+
return v <= opts.maxUsd
|
|
286
|
+
? ok(`cost $${v.toFixed(4)} ≤ $${String(opts.maxUsd)}`)
|
|
287
|
+
: no(`expected cost ≤ $${String(opts.maxUsd)}, got $${v.toFixed(4)}`);
|
|
288
|
+
},
|
|
289
|
+
toJSON: () => ({ kind: "cost", maxUsd: opts.maxUsd }),
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
/** The run took at most `maxMs` of wall-clock time. */
|
|
293
|
+
function latency(opts) {
|
|
294
|
+
return {
|
|
295
|
+
kind: "latency",
|
|
296
|
+
eval: (t) => {
|
|
297
|
+
const v = t.usage.durationMs;
|
|
298
|
+
return v <= opts.maxMs
|
|
299
|
+
? ok(`latency ${String(v)}ms ≤ ${String(opts.maxMs)}ms`)
|
|
300
|
+
: no(`expected latency ≤ ${String(opts.maxMs)}ms, got ${String(v)}ms`);
|
|
301
|
+
},
|
|
302
|
+
toJSON: () => ({ kind: "latency", maxMs: opts.maxMs }),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
/** The run used at most `max` total (input + output) tokens. */
|
|
306
|
+
function tokens(opts) {
|
|
307
|
+
return {
|
|
308
|
+
kind: "tokens",
|
|
309
|
+
eval: (t) => {
|
|
310
|
+
const v = t.usage.inputTokens + t.usage.outputTokens;
|
|
311
|
+
return v <= opts.max
|
|
312
|
+
? ok(`${String(v)} tokens ≤ ${String(opts.max)}`)
|
|
313
|
+
: no(`expected ≤ ${String(opts.max)} tokens, got ${String(v)}`);
|
|
314
|
+
},
|
|
315
|
+
toJSON: () => ({ kind: "tokens", max: opts.max }),
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
//# sourceMappingURL=check.js.map
|
package/dist/claude-code.d.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
|
|
7
7
|
*/
|
|
8
8
|
export * from "./adapters/claude-code/plugin-loader.js";
|
|
9
|
-
export * from "./
|
|
9
|
+
export * from "./mock-model.js";
|
|
10
10
|
export * from "./adapters/claude-code/dialect.js";
|
|
11
11
|
export * from "./adapters/claude-code/layout.js";
|
|
12
12
|
export * from "./adapters/claude-code/runtime.js";
|
package/dist/claude-code.js
CHANGED
|
@@ -22,7 +22,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
22
22
|
* is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
|
|
23
23
|
*/
|
|
24
24
|
__exportStar(require("./adapters/claude-code/plugin-loader.js"), exports);
|
|
25
|
-
__exportStar(require("./
|
|
25
|
+
__exportStar(require("./mock-model.js"), exports);
|
|
26
26
|
__exportStar(require("./adapters/claude-code/dialect.js"), exports);
|
|
27
27
|
__exportStar(require("./adapters/claude-code/layout.js"), exports);
|
|
28
28
|
__exportStar(require("./adapters/claude-code/runtime.js"), exports);
|