vigiles 33.1.0 → 33.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/adapter-conformance.js +14 -10
- package/dist/adapter.d.ts +3 -1
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/output-style.d.ts +3 -0
- package/dist/adapters/claude-code/output-style.js +81 -0
- package/dist/adapters/claude-code/replies.d.ts +14 -0
- package/dist/adapters/claude-code/replies.js +78 -0
- package/dist/adapters/claude-code/runtime.js +70 -0
- package/dist/adapters/claude-code/skill-reachability.d.ts +42 -21
- package/dist/adapters/claude-code/skill-reachability.js +91 -28
- package/dist/audit-report.d.ts +5 -0
- package/dist/audit-report.js +29 -20
- package/dist/audit-report.template.html +25 -25
- package/dist/audit-score.js +2 -2
- package/dist/check.d.ts +20 -0
- package/dist/check.js +87 -2
- package/dist/cli-main.js +249 -30
- package/dist/core/adapter.d.ts +7 -0
- package/dist/core/config-schema.d.ts +9 -0
- package/dist/core/config-schema.js +1 -0
- package/dist/core/eval-driver.d.ts +10 -0
- package/dist/core/frontmatter-read.d.ts +31 -0
- package/dist/core/frontmatter-read.js +4 -2
- package/dist/core/harness-driver.d.ts +2 -0
- package/dist/core/install-reader.js +15 -0
- package/dist/core/layout.d.ts +17 -0
- package/dist/core/layout.js +17 -0
- package/dist/core/output-style.d.ts +84 -0
- package/dist/core/output-style.js +90 -0
- package/dist/core/rule-meta.d.ts +1 -1
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/run-env.d.ts +50 -0
- package/dist/core/run-env.js +51 -0
- package/dist/core/runtime.d.ts +44 -0
- package/dist/core/skill-link-target.d.ts +20 -0
- package/dist/core/skill-link-target.js +33 -0
- package/dist/core/types.d.ts +2 -0
- package/dist/coverage-evidence.d.ts +42 -0
- package/dist/coverage-evidence.js +39 -0
- package/dist/eval.d.ts +65 -42
- package/dist/eval.js +116 -120
- package/dist/harness-test.d.ts +65 -1
- package/dist/harness-test.js +114 -22
- package/dist/optimize.js +1 -5
- package/dist/output-style-arms.d.ts +27 -0
- package/dist/output-style-arms.js +51 -0
- package/dist/plugin-declaration.d.ts +6 -6
- package/dist/plugin-declaration.js +6 -6
- package/dist/scan-files.js +1 -1
- package/dist/scan.d.ts +6 -0
- package/dist/scan.js +14 -4
- package/dist/score-core.d.ts +3 -0
- package/dist/score-core.js +5 -1
- package/dist/setup-plan.d.ts +50 -10
- package/dist/setup-plan.js +103 -68
- package/dist/skill-links.d.ts +183 -0
- package/dist/skill-links.js +255 -0
- package/dist/test-coverage-files.d.ts +3 -1
- package/dist/test-coverage-files.js +20 -6
- package/dist/test-coverage.d.ts +18 -1
- package/dist/test-coverage.js +101 -15
- package/dist/test.d.ts +3 -1
- package/dist/test.js +8 -2
- package/dist/vigilesrc.schema.json +134 -0
- package/package.json +1 -1
package/dist/harness-test.d.ts
CHANGED
|
@@ -2,10 +2,15 @@ import type { HarnessAdapter } from "./core/adapter.js";
|
|
|
2
2
|
import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn } from "./core/harness-driver.js";
|
|
3
3
|
import type { Trace, SubagentTrace } from "./core/eval-driver.js";
|
|
4
4
|
export type { Trace, SubagentTrace } from "./core/eval-driver.js";
|
|
5
|
+
import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
|
|
6
|
+
/** Re-exported for the eval tier's parser, which reads the same stream. */
|
|
7
|
+
export { parseReplies, reachedModel };
|
|
8
|
+
import type { HarnessRuntime } from "./core/runtime.js";
|
|
5
9
|
import { type SandboxMode } from "./sandbox.js";
|
|
6
10
|
export { scriptModel } from "./mock-model.js";
|
|
7
11
|
export type { ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, } from "./core/harness-driver.js";
|
|
8
|
-
|
|
12
|
+
import { loadPlugin, resolveHarness } from "./adapters/claude-code/plugin-loader.js";
|
|
13
|
+
export { loadPlugin, resolveHarness };
|
|
9
14
|
export { decideSandbox, specTrusted, sandboxAvailable, type SandboxMode, } from "./sandbox.js";
|
|
10
15
|
export interface HarnessTestSpec {
|
|
11
16
|
/** Fixture files to write in a fresh temp working dir (path → contents). */
|
|
@@ -28,6 +33,21 @@ export interface HarnessTestSpec {
|
|
|
28
33
|
* `plugin` still layer on top. Resolved to an absolute path.
|
|
29
34
|
*/
|
|
30
35
|
readonly pluginDir?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Path to an OUTPUT STYLE file to switch on for this run. The style is
|
|
38
|
+
* written where the harness keeps styles, and selected by the name the
|
|
39
|
+
* harness reads from the file — never by a name you type, because a setting
|
|
40
|
+
* that misses the name loads no style and raises no error.
|
|
41
|
+
*
|
|
42
|
+
* The run is then checked: if no model request carried the style,
|
|
43
|
+
* `runHarnessTest` THROWS instead of returning a trace, so a style that never
|
|
44
|
+
* loaded cannot pass a test. Refused up front on a harness without output
|
|
45
|
+
* styles, or when the fixture's files or settings already decide.
|
|
46
|
+
*
|
|
47
|
+
* What it proves is delivery. A scripted model does not read the style, so
|
|
48
|
+
* nothing in this tier shows that a model FOLLOWS it.
|
|
49
|
+
*/
|
|
50
|
+
readonly outputStyle?: string;
|
|
31
51
|
/** The scripted model turns the agent will take. */
|
|
32
52
|
readonly model: readonly ModelTurn[];
|
|
33
53
|
/** The user prompt. Default: "go". */
|
|
@@ -107,6 +127,31 @@ export declare function parseHooks(stdout: string): HookFire[];
|
|
|
107
127
|
export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[];
|
|
108
128
|
/** Parse the `claude` stdout/stream into the unified trace fields. */
|
|
109
129
|
export declare function parseClaudeRun(stdout: string): ParsedRun;
|
|
130
|
+
/**
|
|
131
|
+
* Where a harness-tier run's agent gets HOME and the rest of its environment.
|
|
132
|
+
*
|
|
133
|
+
* - `inherit` — the caller's env (real HOME), minus the harness's declared
|
|
134
|
+
* session identity. The default of `runHarnessTest`: a test that reads the
|
|
135
|
+
* machine's config keeps doing so, but no run executes AS the caller's live
|
|
136
|
+
* session.
|
|
137
|
+
* - `throwaway` — a fresh HOME/TMPDIR at `dir`, the OS essentials, and nothing
|
|
138
|
+
* else from the caller: no harness auth (the scripted mock needs none, and a
|
|
139
|
+
* config-dir variable would point the run back at the real config), no
|
|
140
|
+
* identity, no secrets. The same scrubbed shape as the eval tier's
|
|
141
|
+
* `ephemeralEnv` (`scrubbedRunEnv`), without the auth.
|
|
142
|
+
*/
|
|
143
|
+
export type HarnessRunHome = {
|
|
144
|
+
readonly home: "inherit";
|
|
145
|
+
} | {
|
|
146
|
+
readonly home: "throwaway";
|
|
147
|
+
readonly dir: string;
|
|
148
|
+
};
|
|
149
|
+
/**
|
|
150
|
+
* The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
|
|
151
|
+
* `base`, with the runtime's mock wiring on top. Pure — the testable seam of
|
|
152
|
+
* the un-coverable spawn.
|
|
153
|
+
*/
|
|
154
|
+
export declare function harnessSpawnEnv(runtime: HarnessRuntime, wiredEnv: Readonly<Record<string, string>>, home: HarnessRunHome, base?: NodeJS.ProcessEnv): Readonly<Record<string, string>>;
|
|
110
155
|
/** Whether the agent CLI is available — harness tests need it. */
|
|
111
156
|
export declare function claudeAvailable(): boolean;
|
|
112
157
|
/**
|
|
@@ -138,6 +183,15 @@ export interface RunHarnessTestOptions {
|
|
|
138
183
|
* only — requesting confinement for another harness throws.
|
|
139
184
|
*/
|
|
140
185
|
export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise<HarnessTestResult>;
|
|
186
|
+
/**
|
|
187
|
+
* {@link runHarnessTest} with the run's HOME chosen by the caller — see
|
|
188
|
+
* {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
|
|
189
|
+
* a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
|
|
190
|
+
* run must not depend on, or act as, the machine it happens to run on. The
|
|
191
|
+
* confined (bubblewrap) path ignores `home`: it already runs with its own HOME
|
|
192
|
+
* and a cleared env.
|
|
193
|
+
*/
|
|
194
|
+
export declare function runHarnessTestIn(spec: HarnessTestSpec, opts: RunHarnessTestOptions, home: HarnessRunHome["home"]): Promise<HarnessTestResult>;
|
|
141
195
|
/**
|
|
142
196
|
* `runHarness` — the harness-scope entry of the revamped API (Phase 2 of
|
|
143
197
|
* `research/testing-api-design.md`). The harness has two execution scopes, `hook`
|
|
@@ -156,4 +210,14 @@ export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessT
|
|
|
156
210
|
export declare function runHarness(spec: HarnessTestSpec, opts?: RunHarnessTestOptions & {
|
|
157
211
|
model?: "mock" | "real";
|
|
158
212
|
}): Promise<HarnessTestResult>;
|
|
213
|
+
/**
|
|
214
|
+
* The files and settings a run with `outputStyle` writes — the style at its
|
|
215
|
+
* place, selected by the name the harness reads from it. The same fixture
|
|
216
|
+
* `runHarnessTest({ outputStyle })` runs, so an eval arm built from it gets
|
|
217
|
+
* exactly what that run proved reaches the model.
|
|
218
|
+
*/
|
|
219
|
+
export declare function outputStyleFixture(outputStyle: string, adapter?: HarnessAdapter): {
|
|
220
|
+
readonly files: Readonly<Record<string, string>>;
|
|
221
|
+
readonly settings: unknown;
|
|
222
|
+
};
|
|
159
223
|
//# sourceMappingURL=harness-test.d.ts.map
|
package/dist/harness-test.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.claudeCodeDriver = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
|
|
3
|
+
exports.claudeCodeDriver = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = exports.reachedModel = exports.parseReplies = void 0;
|
|
4
4
|
exports.parseToolCalls = parseToolCalls;
|
|
5
5
|
exports.parseSubagents = parseSubagents;
|
|
6
6
|
exports.parseResultEvent = parseResultEvent;
|
|
@@ -8,9 +8,12 @@ exports.parseOutput = parseOutput;
|
|
|
8
8
|
exports.parseHooks = parseHooks;
|
|
9
9
|
exports.buildClaudeArgs = buildClaudeArgs;
|
|
10
10
|
exports.parseClaudeRun = parseClaudeRun;
|
|
11
|
+
exports.harnessSpawnEnv = harnessSpawnEnv;
|
|
11
12
|
exports.claudeAvailable = claudeAvailable;
|
|
12
13
|
exports.runHarnessTest = runHarnessTest;
|
|
14
|
+
exports.runHarnessTestIn = runHarnessTestIn;
|
|
13
15
|
exports.runHarness = runHarness;
|
|
16
|
+
exports.outputStyleFixture = outputStyleFixture;
|
|
14
17
|
/**
|
|
15
18
|
* vigiles — deterministic Claude Code harness testing.
|
|
16
19
|
*
|
|
@@ -49,13 +52,18 @@ const check_count_js_1 = require("./check-count.js");
|
|
|
49
52
|
const coverage_probe_js_1 = require("./coverage-probe.js");
|
|
50
53
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
51
54
|
const mock_model_js_1 = require("./mock-model.js");
|
|
52
|
-
const
|
|
55
|
+
const replies_js_1 = require("./adapters/claude-code/replies.js");
|
|
56
|
+
Object.defineProperty(exports, "parseReplies", { enumerable: true, get: function () { return replies_js_1.parseReplies; } });
|
|
57
|
+
Object.defineProperty(exports, "reachedModel", { enumerable: true, get: function () { return replies_js_1.reachedModel; } });
|
|
58
|
+
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
59
|
+
const output_style_js_1 = require("./core/output-style.js");
|
|
60
|
+
const run_env_js_1 = require("./core/run-env.js");
|
|
53
61
|
const sandbox_js_1 = require("./sandbox.js");
|
|
54
62
|
var mock_model_js_2 = require("./mock-model.js");
|
|
55
63
|
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
56
|
-
|
|
57
|
-
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return
|
|
58
|
-
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return
|
|
64
|
+
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
65
|
+
Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_1.loadPlugin; } });
|
|
66
|
+
Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_1.resolveHarness; } });
|
|
59
67
|
var sandbox_js_2 = require("./sandbox.js");
|
|
60
68
|
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return sandbox_js_2.decideSandbox; } });
|
|
61
69
|
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return sandbox_js_2.specTrusted; } });
|
|
@@ -285,8 +293,9 @@ function buildClaudeArgs(spec, hasSettings) {
|
|
|
285
293
|
? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
|
|
286
294
|
: []),
|
|
287
295
|
...(hasSettings ? ["--settings", "settings.json"] : []),
|
|
288
|
-
|
|
289
|
-
|
|
296
|
+
// A permission allowlist: an empty one approves nothing, so the flag is
|
|
297
|
+
// left out (a bare `--allowedTools` exits before any model turn).
|
|
298
|
+
...(tools.length === 0 ? [] : ["--allowedTools", ...tools]),
|
|
290
299
|
];
|
|
291
300
|
}
|
|
292
301
|
/** Build the `claude` argv from the driver context — wraps `buildClaudeArgs`. */
|
|
@@ -305,8 +314,22 @@ function parseClaudeRun(stdout) {
|
|
|
305
314
|
toolCalls: parseToolCalls(stdout),
|
|
306
315
|
hooks: parseHooks(stdout),
|
|
307
316
|
output: parseOutput(stdout),
|
|
317
|
+
replies: (0, replies_js_1.parseReplies)(stdout),
|
|
308
318
|
};
|
|
309
319
|
}
|
|
320
|
+
/**
|
|
321
|
+
* The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
|
|
322
|
+
* `base`, with the runtime's mock wiring on top. Pure — the testable seam of
|
|
323
|
+
* the un-coverable spawn.
|
|
324
|
+
*/
|
|
325
|
+
function harnessSpawnEnv(runtime, wiredEnv, home, base = process.env) {
|
|
326
|
+
switch (home.home) {
|
|
327
|
+
case "inherit":
|
|
328
|
+
return { ...(0, run_env_js_1.withoutSessionIdentity)(base, runtime.runEnv), ...wiredEnv };
|
|
329
|
+
case "throwaway":
|
|
330
|
+
return { ...(0, run_env_js_1.scrubbedRunEnv)(base, { home: home.dir }), ...wiredEnv };
|
|
331
|
+
}
|
|
332
|
+
}
|
|
310
333
|
/* v8 ignore start -- spawns the real claude CLI + filesystem; exercised by the
|
|
311
334
|
claude-backed suite, excluded from the deterministic coverage gate (the parse
|
|
312
335
|
helpers above carry the testable logic). */
|
|
@@ -338,13 +361,24 @@ function writeFixture(cwd, files, settings) {
|
|
|
338
361
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), json);
|
|
339
362
|
}
|
|
340
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* The {@link HarnessRunHome} for a run in `cwd`. A throwaway HOME lives under
|
|
366
|
+
* the run's own temp dir, as the eval tier's does, so `cleanup()` removes it
|
|
367
|
+
* with everything else.
|
|
368
|
+
*/
|
|
369
|
+
function runHomeIn(cwd, home) {
|
|
370
|
+
return home === "throwaway"
|
|
371
|
+
? { home, dir: (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(cwd, "home-")) }
|
|
372
|
+
: { home };
|
|
373
|
+
}
|
|
341
374
|
/**
|
|
342
375
|
* Spawn the agent binary against the mock. The mock-wiring *args* are already in
|
|
343
376
|
* `args` (the driver placed `wireMock(url).args` at the correct argv position
|
|
344
|
-
* via `ctx.mockArgs`); here
|
|
345
|
-
* env. Driver-agnostic at the transport seam.
|
|
377
|
+
* via `ctx.mockArgs`); here the env is {@link harnessSpawnEnv} of the caller's,
|
|
378
|
+
* with `wireMock(url).env` on top. Driver-agnostic at the transport seam.
|
|
346
379
|
*/
|
|
347
|
-
function spawnAgent(runtime, args,
|
|
380
|
+
function spawnAgent(runtime, args, run) {
|
|
381
|
+
const { cwd, baseUrl, timeoutMs, home } = run;
|
|
348
382
|
const wired = runtime.wireMock(baseUrl);
|
|
349
383
|
return new Promise((resolvePromise) => {
|
|
350
384
|
// vigiles:free-tier — the deterministic harness tier. `runtime.wireMock`
|
|
@@ -354,9 +388,8 @@ function spawnAgent(runtime, args, cwd, baseUrl, timeoutMs) {
|
|
|
354
388
|
const child = (0, node_child_process_1.spawn)(runtime.agentBinary, [...args], {
|
|
355
389
|
cwd,
|
|
356
390
|
// Any key works — the mock ignores auth. wireMock supplies the overlay
|
|
357
|
-
// env (base-URL var + dummy key for CC; the dummy key for Codex)
|
|
358
|
-
|
|
359
|
-
env: { ...process.env, ...wired.env },
|
|
391
|
+
// env (base-URL var + dummy key for CC; the dummy key for Codex).
|
|
392
|
+
env: harnessSpawnEnv(runtime, wired.env, home),
|
|
360
393
|
stdio: ["ignore", "pipe", "pipe"],
|
|
361
394
|
});
|
|
362
395
|
let stdout = "";
|
|
@@ -399,6 +432,7 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
|
|
|
399
432
|
toolCalls: parsed.toolCalls,
|
|
400
433
|
hooks: parsed.hooks,
|
|
401
434
|
output: parsed.output,
|
|
435
|
+
...(parsed.replies === undefined ? {} : { replies: parsed.replies }),
|
|
402
436
|
modelRequests,
|
|
403
437
|
subagents: parseSubagents(out.stdout),
|
|
404
438
|
file: (p) => {
|
|
@@ -423,6 +457,17 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
|
|
|
423
457
|
* only — requesting confinement for another harness throws.
|
|
424
458
|
*/
|
|
425
459
|
async function runHarnessTest(spec, opts = {}) {
|
|
460
|
+
return runHarnessTestIn(spec, opts, "inherit");
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* {@link runHarnessTest} with the run's HOME chosen by the caller — see
|
|
464
|
+
* {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
|
|
465
|
+
* a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
|
|
466
|
+
* run must not depend on, or act as, the machine it happens to run on. The
|
|
467
|
+
* confined (bubblewrap) path ignores `home`: it already runs with its own HOME
|
|
468
|
+
* and a cleared env.
|
|
469
|
+
*/
|
|
470
|
+
async function runHarnessTestIn(spec, opts, home) {
|
|
426
471
|
// Tell the CLI runner this script exercised the harness, so a file that runs
|
|
427
472
|
// NOTHING can be told apart from one that ran and passed. See check-count.ts.
|
|
428
473
|
(0, check_count_js_1.recordCheck)();
|
|
@@ -443,12 +488,8 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
443
488
|
if (decision.action === "sandbox" && !isClaudeCode) {
|
|
444
489
|
throw new Error(`sandbox not supported for ${driver.runtime.name}: confined execution is Claude Code only. Pass sandbox: false to run ${driver.runtime.name} unconfined (you audited the code, or trust the outer container).`);
|
|
445
490
|
}
|
|
491
|
+
const { files, settings, check } = fixtureFor(spec, opts.adapter);
|
|
446
492
|
const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
|
|
447
|
-
const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
|
|
448
|
-
plugin: spec.plugin,
|
|
449
|
-
settings: spec.settings,
|
|
450
|
-
files: spec.files,
|
|
451
|
-
});
|
|
452
493
|
writeFixture(cwd, files, settings);
|
|
453
494
|
const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
|
|
454
495
|
const timeoutMs = spec.timeoutMs ?? 60000;
|
|
@@ -481,14 +522,19 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
481
522
|
const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(count, sideChannelCount);
|
|
482
523
|
if (unconsumed !== undefined)
|
|
483
524
|
console.error(unconsumed);
|
|
484
|
-
return makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests);
|
|
525
|
+
return check(makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests));
|
|
485
526
|
}
|
|
486
527
|
// Direct path: mock runs in this process; the agent reaches it over localhost.
|
|
487
528
|
// Start the mock first so its URL feeds the driver's mock-wiring args.
|
|
488
529
|
const mock = await driver.startMock(spec.model);
|
|
489
530
|
try {
|
|
490
531
|
const args = buildArgs(driver.runtime.wireMock(mock.url).args);
|
|
491
|
-
const out = await spawnAgent(driver.runtime, args,
|
|
532
|
+
const out = await spawnAgent(driver.runtime, args, {
|
|
533
|
+
cwd,
|
|
534
|
+
baseUrl: mock.url,
|
|
535
|
+
timeoutMs,
|
|
536
|
+
home: runHomeIn(cwd, home),
|
|
537
|
+
});
|
|
492
538
|
// A script that was never consumed is otherwise invisible — the run just
|
|
493
539
|
// looks empty. `scriptUnconsumedWarning` names the one shape that produces
|
|
494
540
|
// it (every request arriving without tool declarations, so nothing looked
|
|
@@ -496,9 +542,9 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
496
542
|
const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(mock.count, mock.sideChannelCount ?? 0);
|
|
497
543
|
if (unconsumed !== undefined)
|
|
498
544
|
console.error(unconsumed);
|
|
499
|
-
return makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
|
|
545
|
+
return check(makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
|
|
500
546
|
...mock.requests,
|
|
501
|
-
]);
|
|
547
|
+
]));
|
|
502
548
|
}
|
|
503
549
|
finally {
|
|
504
550
|
await mock.close();
|
|
@@ -527,6 +573,52 @@ async function runHarness(spec, opts = {}) {
|
|
|
527
573
|
}
|
|
528
574
|
return runHarnessTest(spec, opts);
|
|
529
575
|
}
|
|
576
|
+
/**
|
|
577
|
+
* The fixture to write, and the check its result must pass. Without
|
|
578
|
+
* `outputStyle` the check returns the result as it is.
|
|
579
|
+
*/
|
|
580
|
+
function fixtureFor(spec, adapter) {
|
|
581
|
+
// One layout for both phases: the plugin is loaded the way the selected
|
|
582
|
+
// harness lays it out, the same layout the style is then planned against.
|
|
583
|
+
const { layout } = adapter ?? adapter_registry_js_1.defaultAdapter;
|
|
584
|
+
const resolved = (0, plugin_loader_js_1.resolveHarness)({ plugin: spec.plugin, settings: spec.settings, files: spec.files }, layout);
|
|
585
|
+
if (spec.outputStyle === undefined)
|
|
586
|
+
return { ...resolved, check: (result) => result };
|
|
587
|
+
const rules = layout.outputStyles;
|
|
588
|
+
if (rules === undefined)
|
|
589
|
+
throw new Error("outputStyle: this harness has no output styles");
|
|
590
|
+
const plan = (0, output_style_js_1.planStyleRun)(layout, { path: spec.outputStyle, text: readStyleFile(spec.outputStyle) }, resolved);
|
|
591
|
+
if (plan.kind === "refused")
|
|
592
|
+
throw new Error(`outputStyle: ${plan.reason}`);
|
|
593
|
+
return {
|
|
594
|
+
files: plan.files,
|
|
595
|
+
settings: plan.settings,
|
|
596
|
+
check: (result) => {
|
|
597
|
+
if ((0, output_style_js_1.styleReached)(rules, plan.style, result.modelRequests))
|
|
598
|
+
return result;
|
|
599
|
+
throw new Error(`outputStyle: "${plan.style.name ?? ""}" never reached the model — ` +
|
|
600
|
+
`no request carried it, so the harness did not load it. ` +
|
|
601
|
+
`Work dir kept for inspection: ${result.cwd}`);
|
|
602
|
+
},
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* The files and settings a run with `outputStyle` writes — the style at its
|
|
607
|
+
* place, selected by the name the harness reads from it. The same fixture
|
|
608
|
+
* `runHarnessTest({ outputStyle })` runs, so an eval arm built from it gets
|
|
609
|
+
* exactly what that run proved reaches the model.
|
|
610
|
+
*/
|
|
611
|
+
function outputStyleFixture(outputStyle, adapter) {
|
|
612
|
+
const { files, settings } = fixtureFor({ outputStyle, model: [] }, adapter);
|
|
613
|
+
return { files, settings };
|
|
614
|
+
}
|
|
615
|
+
/** The style file's text, or a clear error naming the path. */
|
|
616
|
+
function readStyleFile(path) {
|
|
617
|
+
const abs = (0, node_path_1.resolve)(path);
|
|
618
|
+
if (!(0, node_fs_1.existsSync)(abs))
|
|
619
|
+
throw new Error(`outputStyle: no such file: ${path}`);
|
|
620
|
+
return (0, node_fs_1.readFileSync)(abs, "utf-8");
|
|
621
|
+
}
|
|
530
622
|
/** Pull the pillar-2 driver off an adapter, asserting it supports testing. */
|
|
531
623
|
async function requireDriver(adapter) {
|
|
532
624
|
(0, adapter_conformance_js_1.assertHarnessTestable)(adapter);
|
package/dist/optimize.js
CHANGED
|
@@ -36,10 +36,6 @@ const score_explainer_js_1 = require("./score-explainer.js");
|
|
|
36
36
|
function actionFor(e) {
|
|
37
37
|
return e.symptom === "wrong-skill-fires" ? "differentiate" : "fix";
|
|
38
38
|
}
|
|
39
|
-
function isEmptyMachine(r) {
|
|
40
|
-
const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
|
|
41
|
-
return surfaces === 0 && !r.mcp;
|
|
42
|
-
}
|
|
43
39
|
/**
|
|
44
40
|
* Turn a scan report into a prioritized optimization plan: the structural-health
|
|
45
41
|
* score + a typed recommendation per deterministic finding (`likely` dead-ends
|
|
@@ -60,7 +56,7 @@ function optimize(report) {
|
|
|
60
56
|
score,
|
|
61
57
|
grade: (0, score_core_js_1.gradeFor)(score),
|
|
62
58
|
recommendations,
|
|
63
|
-
empty: isEmptyMachine(report),
|
|
59
|
+
empty: (0, score_core_js_1.isEmptyMachine)(report),
|
|
64
60
|
};
|
|
65
61
|
}
|
|
66
62
|
const ACTION_LABEL = {
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { HarnessAdapter } from "./core/adapter.js";
|
|
2
|
+
import type { EvalArm } from "./eval.js";
|
|
3
|
+
/** The style on and the style off, ready for `paid_measureArms({ arms })`. */
|
|
4
|
+
export interface OutputStyleArms {
|
|
5
|
+
/** The style file at its place in the work dir, selected in the settings. */
|
|
6
|
+
readonly with: EvalArm;
|
|
7
|
+
/** Every style switched off, including one the user set for all projects. */
|
|
8
|
+
readonly without: EvalArm;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Build the with/without arms for the style at `path` (a file in the shape the
|
|
12
|
+
* harness loads, e.g. a Claude Code `.md` style), for the harness the paid tier
|
|
13
|
+
* drives. Spends nothing: the one run it makes uses the scripted model. Throws,
|
|
14
|
+
* before any eval, when the harness has no output styles, cannot tell the
|
|
15
|
+
* style's name, or the style does not reach the model.
|
|
16
|
+
*
|
|
17
|
+
* @experimental One real-model run behind it; how arms are built may change
|
|
18
|
+
* once a style can be authored as a compiled spec with its checks attached.
|
|
19
|
+
*/
|
|
20
|
+
export declare function experimental_outputStyleArms(path: string): Promise<OutputStyleArms>;
|
|
21
|
+
/**
|
|
22
|
+
* {@link experimental_outputStyleArms} on a given adapter. Not public: the paid
|
|
23
|
+
* tier drives only the default harness, so arms built for another one would be
|
|
24
|
+
* run by a harness they were not checked on.
|
|
25
|
+
*/
|
|
26
|
+
export declare function outputStyleArmsWith(path: string, adapter: HarnessAdapter): Promise<OutputStyleArms>;
|
|
27
|
+
//# sourceMappingURL=output-style-arms.d.ts.map
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.experimental_outputStyleArms = experimental_outputStyleArms;
|
|
4
|
+
exports.outputStyleArmsWith = outputStyleArmsWith;
|
|
5
|
+
/**
|
|
6
|
+
* The two arms of a paid output-style eval — the style on, the style off — built
|
|
7
|
+
* from the style FILE, after one free run has shown the style reaches the model.
|
|
8
|
+
*
|
|
9
|
+
* Three silent failures this closes. A hand-typed arm name that misses the
|
|
10
|
+
* style's name loads no style and raises no error, so the "with" arm is the
|
|
11
|
+
* "without" arm and the eval compares a run with itself. A bare "without" arm
|
|
12
|
+
* still loads a style the user switched on for every project, which is the same
|
|
13
|
+
* comparison from the other side. And the paid tier cannot check delivery at
|
|
14
|
+
* all: it drives the real API, so no request is captured. The free run here
|
|
15
|
+
* uses the scripted model, where requests ARE captured, with the same fixture
|
|
16
|
+
* the paid arm gets — so a style that would not load stops the eval before the
|
|
17
|
+
* first billed trial. That free run uses a throwaway HOME and none of the
|
|
18
|
+
* caller's environment beyond the OS essentials.
|
|
19
|
+
*/
|
|
20
|
+
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
21
|
+
const harness_test_js_1 = require("./harness-test.js");
|
|
22
|
+
/**
|
|
23
|
+
* Build the with/without arms for the style at `path` (a file in the shape the
|
|
24
|
+
* harness loads, e.g. a Claude Code `.md` style), for the harness the paid tier
|
|
25
|
+
* drives. Spends nothing: the one run it makes uses the scripted model. Throws,
|
|
26
|
+
* before any eval, when the harness has no output styles, cannot tell the
|
|
27
|
+
* style's name, or the style does not reach the model.
|
|
28
|
+
*
|
|
29
|
+
* @experimental One real-model run behind it; how arms are built may change
|
|
30
|
+
* once a style can be authored as a compiled spec with its checks attached.
|
|
31
|
+
*/
|
|
32
|
+
async function experimental_outputStyleArms(path) {
|
|
33
|
+
return outputStyleArmsWith(path, adapter_registry_js_1.defaultAdapter);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* {@link experimental_outputStyleArms} on a given adapter. Not public: the paid
|
|
37
|
+
* tier drives only the default harness, so arms built for another one would be
|
|
38
|
+
* run by a harness they were not checked on.
|
|
39
|
+
*/
|
|
40
|
+
async function outputStyleArmsWith(path, adapter) {
|
|
41
|
+
// A throwaway HOME: this run happens on every import of an eval file (even
|
|
42
|
+
// `vigiles eval --check`), so it must not read the machine's user config or
|
|
43
|
+
// run as the session that imported it. What it proves — the style reaches the
|
|
44
|
+
// model — is a fact about the fixture, not about this machine.
|
|
45
|
+
const preflight = await (0, harness_test_js_1.runHarnessTestIn)({ outputStyle: path, prompt: "hi", model: [{ text: "ok" }] }, { adapter }, "throwaway");
|
|
46
|
+
preflight.cleanup();
|
|
47
|
+
const { files, settings } = (0, harness_test_js_1.outputStyleFixture)(path, adapter);
|
|
48
|
+
const none = adapter.layout.outputStyles?.selectNone ?? {};
|
|
49
|
+
return { with: { files, settings }, without: { settings: none } };
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=output-style-arms.js.map
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
* boundary is deliberate: plugins "can execute arbitrary code on your machine
|
|
18
18
|
* with your user privileges", so a repo is not allowed to install one for you.
|
|
19
19
|
*
|
|
20
|
-
* The declaration buys exactly one thing
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
20
|
+
* The declaration buys exactly one thing: a collaborator who clones the repo is
|
|
21
|
+
* told the project wants this plugin, with the command that installs it.
|
|
22
|
+
* **Silent absence becomes a prompt.** It does not become a working install. What
|
|
23
|
+
* the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
|
|
24
|
+
* any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
|
|
25
|
+
* the links travel with the clone.
|
|
26
26
|
*
|
|
27
27
|
* Nothing is vendored: what lands in the repo is a *reference* (a marketplace
|
|
28
28
|
* source plus an enabled flag). The plugin content still lives in the global
|
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
* boundary is deliberate: plugins "can execute arbitrary code on your machine
|
|
19
19
|
* with your user privileges", so a repo is not allowed to install one for you.
|
|
20
20
|
*
|
|
21
|
-
* The declaration buys exactly one thing
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
21
|
+
* The declaration buys exactly one thing: a collaborator who clones the repo is
|
|
22
|
+
* told the project wants this plugin, with the command that installs it.
|
|
23
|
+
* **Silent absence becomes a prompt.** It does not become a working install. What
|
|
24
|
+
* the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
|
|
25
|
+
* any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
|
|
26
|
+
* the links travel with the clone.
|
|
27
27
|
*
|
|
28
28
|
* Nothing is vendored: what lands in the repo is a *reference* (a marketplace
|
|
29
29
|
* source plus an enabled flag). The plugin content still lives in the global
|
package/dist/scan-files.js
CHANGED
|
@@ -539,7 +539,7 @@ function scanFiles(files, layout = layout_js_1.claudeCodeLayout, dialect = diale
|
|
|
539
539
|
hooks,
|
|
540
540
|
inlineHooks: inline,
|
|
541
541
|
manualHookCount: manual,
|
|
542
|
-
|
|
542
|
+
...(0, coverage_evidence_js_1.inventoryCounts)(loaded.files, cls.isCommand, [lay], coverage),
|
|
543
543
|
// A declared server set counts even when the loader emitted no warning —
|
|
544
544
|
// otherwise a plugin whose servers come from the Agent Plugins `mcp.json`
|
|
545
545
|
// reports "MCP servers: no" while the report lists an MCP finding.
|
package/dist/scan.d.ts
CHANGED
|
@@ -211,6 +211,12 @@ export interface ScanReport {
|
|
|
211
211
|
*/
|
|
212
212
|
readonly manualHookCount: number;
|
|
213
213
|
readonly commands: number;
|
|
214
|
+
/**
|
|
215
|
+
* Output styles found, exempt ones aside, or `"not-supported"` when the
|
|
216
|
+
* harness has none — the inventory side of `untested-output-style`. Optional
|
|
217
|
+
* so a hand-built report predating the field still type-checks.
|
|
218
|
+
*/
|
|
219
|
+
readonly outputStyles?: number | "not-supported";
|
|
214
220
|
readonly mcp: boolean;
|
|
215
221
|
/**
|
|
216
222
|
* Intra-plugin file references (hook scripts, skill bodies) pointing at files
|
package/dist/scan.js
CHANGED
|
@@ -258,6 +258,7 @@ function scanPlugin(dir, layout, dialect, opts = {}) {
|
|
|
258
258
|
const coverage = (0, test_coverage_js_1.findUntestedSurfaces)({
|
|
259
259
|
basePath: dir,
|
|
260
260
|
layout: lay,
|
|
261
|
+
harnessLayouts: declared.map((h) => h.layout),
|
|
261
262
|
// The SAME `.vigilesrc.json#exclude`. Untested-surface discovery is a
|
|
262
263
|
// second walk over the same trees, so leaving it out would have excluded a
|
|
263
264
|
// skill from the GRADE while still naming it in "Untested surfaces: 1" — a
|
|
@@ -276,7 +277,7 @@ function scanPlugin(dir, layout, dialect, opts = {}) {
|
|
|
276
277
|
hooks,
|
|
277
278
|
inlineHooks: inline,
|
|
278
279
|
manualHookCount: manual,
|
|
279
|
-
|
|
280
|
+
...(0, coverage_evidence_js_1.inventoryCounts)(loaded.files, cls.isCommand, declared.map((h) => h.layout), coverage),
|
|
280
281
|
// A declared server set counts even when the loader emitted no warning —
|
|
281
282
|
// otherwise a plugin whose servers come from the Agent Plugins `mcp.json`
|
|
282
283
|
// reports "MCP servers: no" while the report lists an MCP finding.
|
|
@@ -655,6 +656,16 @@ function instructionWeightLines(w) {
|
|
|
655
656
|
...tail,
|
|
656
657
|
];
|
|
657
658
|
}
|
|
659
|
+
/** The commands and output-style lines of a scan's facts; a harness without styles says so. */
|
|
660
|
+
function inventoryFacts(r) {
|
|
661
|
+
const commands = r.commands > 0 ? [`Commands: ${String(r.commands)}`] : [];
|
|
662
|
+
if (r.outputStyles === "not-supported")
|
|
663
|
+
return [...commands, "Output styles: n/a (this harness has none)"];
|
|
664
|
+
const styles = r.outputStyles ?? 0;
|
|
665
|
+
return styles > 0
|
|
666
|
+
? [...commands, `Output styles: ${String(styles)}`]
|
|
667
|
+
: commands;
|
|
668
|
+
}
|
|
658
669
|
function formatScanReport(r) {
|
|
659
670
|
const out = [`Scan: ${r.dir}`, ""];
|
|
660
671
|
if (r.instructions) {
|
|
@@ -715,15 +726,14 @@ function formatScanReport(r) {
|
|
|
715
726
|
out.push(...section("Ineffective hook guards (false confidence)", r.hookBlockFindings.map((h) => ` ✗ [${h.event}] ${h.scriptPath ?? "(inline)"}: ${h.message}`)));
|
|
716
727
|
out.push(...section("Hook matchers that don't fire as written", r.hookMatcherFindings.map((m) => ` ✗ ${m.message}`)));
|
|
717
728
|
const facts = [];
|
|
718
|
-
|
|
719
|
-
facts.push(`Commands: ${String(r.commands)}`);
|
|
729
|
+
facts.push(...inventoryFacts(r));
|
|
720
730
|
facts.push(`MCP servers: ${r.mcp ? "yes" : "no"}`);
|
|
721
731
|
facts.push(`Untested surfaces: ${String(r.untested)}`);
|
|
722
732
|
// The two tiers, named separately — a surface with a deterministic harness and
|
|
723
733
|
// no eval is NOT the same position as one with neither. Shown only when the
|
|
724
734
|
// producer supplied the split (a hand-built report may not have).
|
|
725
735
|
if (r.untestedHarness !== undefined && r.unevaluated !== undefined) {
|
|
726
|
-
facts.push(` no harness: ${String(r.untestedHarness)} ·
|
|
736
|
+
facts.push(` no harness: ${String(r.untestedHarness)} · never measured with a real model: ${String(r.unevaluated)}`);
|
|
727
737
|
}
|
|
728
738
|
// …and HOW the rest passed. Without this the count is unfalsifiable from the
|
|
729
739
|
// outside: "28 covered" and "28 names that appear in some file" print the same.
|
package/dist/score-core.d.ts
CHANGED
|
@@ -180,6 +180,9 @@ export declare function reportDeductions(r: ScanReport): Deduction[];
|
|
|
180
180
|
* surface of any kind", because it reads as "this plugin is broken" — a
|
|
181
181
|
* different statement from "this plugin has no skills or subagents".
|
|
182
182
|
*
|
|
183
|
+
* An output style counts too: a repo can ship only a style, and "nothing to
|
|
184
|
+
* audit" for it would hide the untested-output-style finding.
|
|
185
|
+
*
|
|
183
186
|
* Hooks count in BOTH forms: `hooks[]` is the script-backed ones, `inlineHooks`
|
|
184
187
|
* the shell one-liners that carry no script file to path-check. The second is
|
|
185
188
|
* still a gate that runs.
|
package/dist/score-core.js
CHANGED
|
@@ -332,6 +332,9 @@ function reportDeductions(r) {
|
|
|
332
332
|
* surface of any kind", because it reads as "this plugin is broken" — a
|
|
333
333
|
* different statement from "this plugin has no skills or subagents".
|
|
334
334
|
*
|
|
335
|
+
* An output style counts too: a repo can ship only a style, and "nothing to
|
|
336
|
+
* audit" for it would hide the untested-output-style finding.
|
|
337
|
+
*
|
|
335
338
|
* Hooks count in BOTH forms: `hooks[]` is the script-backed ones, `inlineHooks`
|
|
336
339
|
* the shell one-liners that carry no script file to path-check. The second is
|
|
337
340
|
* still a gate that runs.
|
|
@@ -341,7 +344,8 @@ function isEmptyMachine(r) {
|
|
|
341
344
|
r.agents.length +
|
|
342
345
|
r.hooks.length +
|
|
343
346
|
r.inlineHooks +
|
|
344
|
-
r.commands
|
|
347
|
+
r.commands +
|
|
348
|
+
(typeof r.outputStyles === "number" ? r.outputStyles : 0);
|
|
345
349
|
return surfaces === 0 && !r.mcp;
|
|
346
350
|
}
|
|
347
351
|
/**
|