vigiles 33.1.0 → 33.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-conformance.js +14 -10
- package/dist/adapter.d.ts +2 -0
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/output-style.d.ts +3 -0
- package/dist/adapters/claude-code/output-style.js +81 -0
- package/dist/adapters/claude-code/replies.d.ts +14 -0
- package/dist/adapters/claude-code/replies.js +78 -0
- package/dist/audit-report.d.ts +5 -0
- package/dist/audit-report.js +29 -20
- package/dist/audit-report.template.html +25 -25
- package/dist/audit-score.js +2 -2
- package/dist/check.d.ts +20 -0
- package/dist/check.js +87 -2
- package/dist/cli-main.js +25 -13
- package/dist/core/config-schema.d.ts +9 -0
- package/dist/core/config-schema.js +1 -0
- package/dist/core/eval-driver.d.ts +10 -0
- package/dist/core/frontmatter-read.d.ts +31 -0
- package/dist/core/frontmatter-read.js +4 -2
- package/dist/core/harness-driver.d.ts +2 -0
- package/dist/core/layout.d.ts +7 -0
- package/dist/core/output-style.d.ts +84 -0
- package/dist/core/output-style.js +90 -0
- package/dist/core/rule-meta.d.ts +1 -1
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/types.d.ts +2 -0
- package/dist/coverage-evidence.d.ts +42 -0
- package/dist/coverage-evidence.js +39 -0
- package/dist/eval.d.ts +26 -3
- package/dist/eval.js +54 -5
- package/dist/harness-test.d.ts +30 -1
- package/dist/harness-test.js +65 -15
- package/dist/optimize.js +1 -5
- package/dist/output-style-arms.d.ts +27 -0
- package/dist/output-style-arms.js +46 -0
- package/dist/scan-files.js +1 -1
- package/dist/scan.d.ts +6 -0
- package/dist/scan.js +14 -4
- package/dist/score-core.d.ts +3 -0
- package/dist/score-core.js +5 -1
- package/dist/setup-plan.d.ts +1 -1
- package/dist/setup-plan.js +1 -0
- package/dist/test-coverage-files.d.ts +3 -1
- package/dist/test-coverage-files.js +20 -6
- package/dist/test-coverage.d.ts +18 -1
- package/dist/test-coverage.js +64 -14
- package/dist/test.d.ts +3 -1
- package/dist/test.js +8 -2
- package/dist/vigilesrc.schema.json +134 -0
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type PluginLayout } from "./core/layout.js";
|
|
2
2
|
import type { CoverageDecision, CoverageTier, Surface } from "./test-coverage.js";
|
|
3
3
|
/**
|
|
4
|
-
* The untested harness surfaces (skills / agents / hooks) in a file map — the
|
|
4
|
+
* The untested harness surfaces (skills / agents / hooks / output styles) in a file map — the
|
|
5
5
|
* browser-safe twin of `findUntestedSurfaces`, returning the union `untested` list
|
|
6
6
|
* plus the same per-tier split (`scanFiles` needs the counts). Same coverage rules
|
|
7
7
|
* as the disk detector; the tier split is by suffix, exactly as on disk.
|
|
@@ -17,5 +17,7 @@ repoName: string): {
|
|
|
17
17
|
decisions: readonly CoverageDecision[];
|
|
18
18
|
harness: CoverageTier;
|
|
19
19
|
evals: CoverageTier;
|
|
20
|
+
/** Surfaces opted out via `vigiles:ignore-test`; still part of the inventory. */
|
|
21
|
+
exemptSurfaces: readonly Surface[];
|
|
20
22
|
};
|
|
21
23
|
//# sourceMappingURL=test-coverage-files.d.ts.map
|
|
@@ -34,6 +34,7 @@ exports.findUntestedSurfacesInFiles = findUntestedSurfacesInFiles;
|
|
|
34
34
|
const posix_path_js_1 = require("./posix-path.js");
|
|
35
35
|
const layout_js_1 = require("./core/layout.js");
|
|
36
36
|
const coverage_evidence_js_1 = require("./coverage-evidence.js");
|
|
37
|
+
const output_style_js_1 = require("./core/output-style.js");
|
|
37
38
|
const source_kinds_js_1 = require("./source-kinds.js");
|
|
38
39
|
const IGNORE_MARKER = "vigiles:ignore-test";
|
|
39
40
|
/** Mirror of the DEFAULT_IGNORE globs (root-anchored), as a key predicate. */
|
|
@@ -136,6 +137,13 @@ function discoverAgents(files, layout) {
|
|
|
136
137
|
}
|
|
137
138
|
return out;
|
|
138
139
|
}
|
|
140
|
+
/** Twin of test-coverage.ts `discoverOutputStyles`: the harness decides, over the map. */
|
|
141
|
+
function discoverOutputStyles(files, layout) {
|
|
142
|
+
const scan = (0, output_style_js_1.findOutputStyles)(layout, new Map(Object.entries(files).filter(([path]) => !isIgnored(path))));
|
|
143
|
+
return scan.kind === "not-supported"
|
|
144
|
+
? []
|
|
145
|
+
: scan.styles.map((style) => (0, coverage_evidence_js_1.outputStyleSurface)(style.path, files[style.path] ?? ""));
|
|
146
|
+
}
|
|
139
147
|
/**
|
|
140
148
|
* Hook-script paths referenced from a manifest's `hooks` block (file hooks only).
|
|
141
149
|
*
|
|
@@ -226,8 +234,17 @@ function tierOf(considered, tests, globs) {
|
|
|
226
234
|
}
|
|
227
235
|
return { covered, untested, decisions };
|
|
228
236
|
}
|
|
237
|
+
/** Every surface kind in the map — twin of test-coverage.ts `discoverSurfaces`. */
|
|
238
|
+
function discoverSurfaces(files, layout, repoName) {
|
|
239
|
+
return [
|
|
240
|
+
...discoverSkills(files, layout, repoName),
|
|
241
|
+
...discoverAgents(files, layout),
|
|
242
|
+
...discoverHooks(files, layout),
|
|
243
|
+
...discoverOutputStyles(files, layout),
|
|
244
|
+
];
|
|
245
|
+
}
|
|
229
246
|
/**
|
|
230
|
-
* The untested harness surfaces (skills / agents / hooks) in a file map — the
|
|
247
|
+
* The untested harness surfaces (skills / agents / hooks / output styles) in a file map — the
|
|
231
248
|
* browser-safe twin of `findUntestedSurfaces`, returning the union `untested` list
|
|
232
249
|
* plus the same per-tier split (`scanFiles` needs the counts). Same coverage rules
|
|
233
250
|
* as the disk detector; the tier split is by suffix, exactly as on disk.
|
|
@@ -239,11 +256,7 @@ function findUntestedSurfacesInFiles(files, layout,
|
|
|
239
256
|
* `name:`. Same value `scanFiles` already threads for the skill inventory.
|
|
240
257
|
*/
|
|
241
258
|
repoName) {
|
|
242
|
-
const surfaces =
|
|
243
|
-
...discoverSkills(files, layout, repoName),
|
|
244
|
-
...discoverAgents(files, layout),
|
|
245
|
-
...discoverHooks(files, layout),
|
|
246
|
-
];
|
|
259
|
+
const surfaces = discoverSurfaces(files, layout, repoName);
|
|
247
260
|
const considered = surfaces.filter((s) => !s.ignored);
|
|
248
261
|
const tests = discoverTests(files);
|
|
249
262
|
// NO configured `{surface}` globs here, and that is a property of this twin
|
|
@@ -257,6 +270,7 @@ repoName) {
|
|
|
257
270
|
return {
|
|
258
271
|
untested: [...union.untested],
|
|
259
272
|
decisions: union.decisions,
|
|
273
|
+
exemptSurfaces: surfaces.filter((s) => s.ignored),
|
|
260
274
|
harness: tierOf(considered, tests.filter((t) => !(0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path))), noConfiguredGlobs),
|
|
261
275
|
evals: tierOf(considered, tests.filter((t) => (0, coverage_evidence_js_1.isEvalScript)((0, posix_path_js_1.basename)(t.path))), noConfiguredGlobs),
|
|
262
276
|
};
|
package/dist/test-coverage.d.ts
CHANGED
|
@@ -64,7 +64,7 @@ import { type ExcludeSet } from "./exclude.js";
|
|
|
64
64
|
import type { RepoPath } from "./core/frame.js";
|
|
65
65
|
import { type PluginLayout } from "./core/layout.js";
|
|
66
66
|
import { type CoverageEvidence, type EvidenceCounts } from "./coverage-evidence.js";
|
|
67
|
-
export type SurfaceKind = "skill" | "agent" | "hook";
|
|
67
|
+
export type SurfaceKind = "skill" | "agent" | "hook" | "output-style";
|
|
68
68
|
export interface Surface {
|
|
69
69
|
readonly kind: SurfaceKind;
|
|
70
70
|
/** Repo-relative path to the surface file (SKILL.md / agent .md / hook script). */
|
|
@@ -119,6 +119,11 @@ export interface UntestedReport {
|
|
|
119
119
|
readonly untested: readonly Surface[];
|
|
120
120
|
/** Surfaces explicitly opted out via `vigiles:ignore-test`. */
|
|
121
121
|
readonly exempt: number;
|
|
122
|
+
/**
|
|
123
|
+
* The exempt surfaces themselves. The marker waives a surface's test, not its
|
|
124
|
+
* existence, so an inventory counts them (an exempt style still ships).
|
|
125
|
+
*/
|
|
126
|
+
readonly exemptSurfaces: readonly Surface[];
|
|
122
127
|
/**
|
|
123
128
|
* Test files still carrying the RETIRED `vigiles:covers` marker.
|
|
124
129
|
*
|
|
@@ -201,6 +206,18 @@ export interface TestCoverageOptions {
|
|
|
201
206
|
readonly agents?: boolean;
|
|
202
207
|
/** Scan hook scripts referenced from plugin.json / settings.json. Default true. */
|
|
203
208
|
readonly hooks?: boolean;
|
|
209
|
+
/**
|
|
210
|
+
* Scan output styles, in every folder the layout's `outputStyles` names.
|
|
211
|
+
* Default true; a layout with no output styles finds none either way.
|
|
212
|
+
*/
|
|
213
|
+
readonly outputStyles?: boolean;
|
|
214
|
+
/**
|
|
215
|
+
* Every harness the repository declares, when it declares more than one.
|
|
216
|
+
* Surfaces are found in each of them, one per file: a repository can list a
|
|
217
|
+
* harness first and keep its skills, agents or styles in another's folder.
|
|
218
|
+
* Defaults to `[layout]`.
|
|
219
|
+
*/
|
|
220
|
+
readonly harnessLayouts?: readonly PluginLayout[];
|
|
204
221
|
/** Globs of test files that count as coverage. */
|
|
205
222
|
readonly include?: readonly string[];
|
|
206
223
|
/**
|
package/dist/test-coverage.js
CHANGED
|
@@ -78,6 +78,7 @@ const ts_runner_caps_js_1 = require("./ts-runner-caps.js");
|
|
|
78
78
|
const glob_1 = require("glob");
|
|
79
79
|
const glob_ignore_js_1 = require("./core/glob-ignore.js");
|
|
80
80
|
const exclude_js_1 = require("./exclude.js");
|
|
81
|
+
const output_style_js_1 = require("./core/output-style.js");
|
|
81
82
|
const source_kinds_js_1 = require("./source-kinds.js");
|
|
82
83
|
const layout_js_1 = require("./core/layout.js");
|
|
83
84
|
const coverage_evidence_js_1 = require("./coverage-evidence.js");
|
|
@@ -249,6 +250,46 @@ function discoverAgents(basePath, ignore, layout) {
|
|
|
249
250
|
}
|
|
250
251
|
return out;
|
|
251
252
|
}
|
|
253
|
+
/**
|
|
254
|
+
* Output styles, by the layout's own rules: every file in a style home that
|
|
255
|
+
* the harness would load. {@link findOutputStyles} decides; this only reads the
|
|
256
|
+
* candidates off the disk, so it and the browser twin cannot disagree on which
|
|
257
|
+
* files are styles.
|
|
258
|
+
*/
|
|
259
|
+
function discoverOutputStyles(basePath, ignore, layout) {
|
|
260
|
+
const candidates = new Map((0, output_style_js_1.outputStyleHomes)(layout).flatMap((home) => (0, glob_1.globSync)(`${home}/**/*`, {
|
|
261
|
+
cwd: basePath,
|
|
262
|
+
ignore: [...ignore],
|
|
263
|
+
dot: true,
|
|
264
|
+
nodir: true,
|
|
265
|
+
}).map((path) => [path.split(node_path_1.sep).join("/"), ""])));
|
|
266
|
+
const scan = (0, output_style_js_1.findOutputStyles)(layout, candidates);
|
|
267
|
+
return scan.kind === "not-supported"
|
|
268
|
+
? []
|
|
269
|
+
: scan.styles.map((style) => (0, coverage_evidence_js_1.outputStyleSurface)(style.path, read((0, node_path_1.join)(basePath, style.path))));
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Every kind the options switch on, found under the bundle. Discovery gets only
|
|
273
|
+
* the frame-free ignore floor; the rule's and the repo's excludes are applied
|
|
274
|
+
* later, from `root`, by the caller.
|
|
275
|
+
*/
|
|
276
|
+
function discoverSurfaces(basePath, layout, options) {
|
|
277
|
+
const on = (flag) => flag !== false;
|
|
278
|
+
const found = (options.harnessLayouts ?? [layout]).flatMap((each) => [
|
|
279
|
+
...(on(options.skills)
|
|
280
|
+
? discoverSkills(basePath, [...DEFAULT_IGNORE], each)
|
|
281
|
+
: []),
|
|
282
|
+
...(on(options.agents)
|
|
283
|
+
? discoverAgents(basePath, [...DEFAULT_IGNORE], each)
|
|
284
|
+
: []),
|
|
285
|
+
...(on(options.hooks) ? discoverHooks(basePath, each) : []),
|
|
286
|
+
...(on(options.outputStyles)
|
|
287
|
+
? discoverOutputStyles(basePath, DEFAULT_IGNORE, each)
|
|
288
|
+
: []),
|
|
289
|
+
]);
|
|
290
|
+
// Two declared harnesses can read the same file: one surface per file.
|
|
291
|
+
return [...new Map(found.map((s) => [`${s.kind}\0${s.path}`, s])).values()];
|
|
292
|
+
}
|
|
252
293
|
/**
|
|
253
294
|
* Hook-script paths referenced from a manifest's `hooks` block (file hooks only).
|
|
254
295
|
*
|
|
@@ -419,6 +460,11 @@ function tierOf(considered, tests, index, tier, globs) {
|
|
|
419
460
|
// ---------------------------------------------------------------------------
|
|
420
461
|
// Public API
|
|
421
462
|
// ---------------------------------------------------------------------------
|
|
463
|
+
/** The exempt surfaces and their count: waived from testing, still in the inventory. */
|
|
464
|
+
function exemptOf(surfaces) {
|
|
465
|
+
const exemptSurfaces = surfaces.filter((s) => s.ignored);
|
|
466
|
+
return { exempt: exemptSurfaces.length, exemptSurfaces };
|
|
467
|
+
}
|
|
422
468
|
/**
|
|
423
469
|
* Find harness surfaces (skills / agents / hooks) that no test or eval covers.
|
|
424
470
|
* A surface is covered by a colocated `*.{harness,eval}.mjs` OR any discovered
|
|
@@ -449,24 +495,15 @@ function findUntestedSurfaces(options) {
|
|
|
449
495
|
// Discovery globs from the BUNDLE, so it gets only the frame-free floor; the
|
|
450
496
|
// rule's patterns are root-relative and the repo's carry their own root, so
|
|
451
497
|
// both are applied below, to the root-relative path.
|
|
452
|
-
const
|
|
453
|
-
const found = [];
|
|
454
|
-
if (options.skills !== false)
|
|
455
|
-
found.push(...discoverSkills(basePath, floor, layout));
|
|
456
|
-
if (options.agents !== false)
|
|
457
|
-
found.push(...discoverAgents(basePath, floor, layout));
|
|
458
|
-
if (options.hooks !== false)
|
|
459
|
-
found.push(...discoverHooks(basePath, layout));
|
|
498
|
+
const found = discoverSurfaces(basePath, layout, options);
|
|
460
499
|
// Hooks were never subject to `ignore` (a compiled hook lives under the
|
|
461
500
|
// default-ignored `.vigiles/`); keep that, and re-apply exclude to the kinds
|
|
462
501
|
// whose discovery glob used it — now from `root`, where the patterns live.
|
|
463
502
|
const surfaces = found
|
|
464
503
|
.map(toRoot)
|
|
465
504
|
.filter((s) => s.kind === "hook" || !excluded(s.path));
|
|
466
|
-
//
|
|
467
|
-
// `vigiles:ignore-test` marker exempts a surface (a visible, deliberate skip).
|
|
505
|
+
// Only an explicit `vigiles:ignore-test` marker exempts a surface.
|
|
468
506
|
const considered = surfaces.filter((s) => !s.ignored);
|
|
469
|
-
const exempt = surfaces.length - considered.length;
|
|
470
507
|
const tests = discoverTests(root, globs, ignore);
|
|
471
508
|
const split = partitionTests(tests);
|
|
472
509
|
// The run record, if there is one. NO artifact ⇒ an empty index ⇒ every
|
|
@@ -487,7 +524,7 @@ function findUntestedSurfaces(options) {
|
|
|
487
524
|
total: considered.length,
|
|
488
525
|
covered: union.covered,
|
|
489
526
|
untested: union.untested,
|
|
490
|
-
|
|
527
|
+
...exemptOf(surfaces),
|
|
491
528
|
staleRuns: staleRunsFor(considered, runIndex),
|
|
492
529
|
testExt: (0, test_file_ext_js_1.testFileExt)({
|
|
493
530
|
configured: options.testExtension,
|
|
@@ -683,6 +720,19 @@ function evalTierQuestion(kind) {
|
|
|
683
720
|
`the tool contract, \`assertAgentOk\` for the typed result. Those two ` +
|
|
684
721
|
`calls drive Claude Code and take no \`evalDriver\`, so on another ` +
|
|
685
722
|
`harness this tier has no public dispatch yet.`);
|
|
723
|
+
case "output-style":
|
|
724
|
+
// A style is not selected by the model, so there is no firing to
|
|
725
|
+
// measure; the real-model question is whether a model handed the style
|
|
726
|
+
// FOLLOWS it. The deterministic test that covers it drove a scripted
|
|
727
|
+
// model, which proves delivery and nothing about following.
|
|
728
|
+
return (`A deterministic test covers it, but that test drove a SCRIPTED model: ` +
|
|
729
|
+
`it can show the style reached the model, never that a real model ` +
|
|
730
|
+
`follows it.\n` +
|
|
731
|
+
`See the \`test-harness\` skill: compare runs with and without the style ` +
|
|
732
|
+
`(\`measureArms\`), check the output with deterministic checks where the ` +
|
|
733
|
+
`rule has a checkable shape and \`judged\` where it does not, and gate ` +
|
|
734
|
+
`with \`assertRates\`. NOT \`measureTriggerRate\`: a style is not ` +
|
|
735
|
+
`selected by the model, so there is no trigger to measure.`);
|
|
686
736
|
case "hook":
|
|
687
737
|
// Nothing. A hook is deterministic by construction: `runHook` answers
|
|
688
738
|
// "does it block event X?" at the unit tier, free, for every event type —
|
|
@@ -859,8 +909,8 @@ function formatUntestedReport(report) {
|
|
|
859
909
|
// model budget.
|
|
860
910
|
lines.push(` Two gaps, two costs: ${String(report.harness.untested.length)} with no ` +
|
|
861
911
|
`deterministic harness (free, every push) · ` +
|
|
862
|
-
`${String(report.evals.untested.length)}
|
|
863
|
-
`(
|
|
912
|
+
`${String(report.evals.untested.length)} never measured with a real model ` +
|
|
913
|
+
`(paid, run on a schedule).`);
|
|
864
914
|
// What the surfaces that DID pass are resting on.
|
|
865
915
|
if (provenance)
|
|
866
916
|
lines.push(` ${provenance}`);
|
package/dist/test.d.ts
CHANGED
|
@@ -62,7 +62,7 @@ export { loadHook } from "./load-hook.js";
|
|
|
62
62
|
export { experimental_hookState } from "./hook-state-store.js";
|
|
63
63
|
export type { HookStateHandle, SeedStateOptions } from "./hook-state-store.js";
|
|
64
64
|
export type { StateFact, Duration } from "./core/hook-state.js";
|
|
65
|
-
export { evalChecks, assertChecks, tool, toolWith, notTool, onlyTools, skill, output, hookFired, received, turns, wrote, didNotWrite, subagent, blocked, allowed, mcp, cost, latency, tokens, inputTokens, outputTokens, cacheTokens, } from "./check.js";
|
|
65
|
+
export { evalChecks, assertChecks, tool, toolWith, notTool, onlyTools, skill, output, experimental_eachReply, experimental_replyCount, hookFired, received, turns, wrote, didNotWrite, subagent, blocked, allowed, mcp, cost, latency, tokens, inputTokens, outputTokens, cacheTokens, } from "./check.js";
|
|
66
66
|
export type { ArgMatcher, Check, CheckJSON, CheckResult, JudgeFn, } from "./check.js";
|
|
67
67
|
export { DISASTER_CATALOG, verifyGuardrail, unblockedDisasters, assertBlocksDisasters, formatGuardrailReport, experimental_alternateSpellings, } from "./guardrail-check.js";
|
|
68
68
|
export type { DisasterEvent, DisasterCategory, GuardrailResult, VerifyGuardrailOptions, } from "./guardrail-check.js";
|
|
@@ -70,6 +70,8 @@ export { experimental_verifyPluginGuards, experimental_formatPluginGuardReport,
|
|
|
70
70
|
export type { PluginGuardReport, SweptHook, SweptHookOutcome, NonCommandHookAction, VerifyPluginGuardsOptions, } from "./verify-plugin-guards.js";
|
|
71
71
|
export * from "./tool-stub.js";
|
|
72
72
|
export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
|
|
73
|
+
export { experimental_outputStyleArms } from "./output-style-arms.js";
|
|
74
|
+
export type { OutputStyleArms } from "./output-style-arms.js";
|
|
73
75
|
export type { HarnessTestSpec, Trace, SubagentTrace, HarnessTestResult, RunHarnessTestOptions, ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, SandboxMode, } from "./harness-test.js";
|
|
74
76
|
export { commandsIn, mustInclude, mustNotInclude } from "./doc-commands.js";
|
|
75
77
|
export type { DocCommand } from "./doc-commands.js";
|
package/dist/test.js
CHANGED
|
@@ -71,8 +71,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
71
71
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
72
72
|
};
|
|
73
73
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
74
|
-
exports.
|
|
75
|
-
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = void 0;
|
|
74
|
+
exports.runHarness = exports.runHarnessTest = exports.experimental_formatPluginGuardReport = exports.experimental_verifyPluginGuards = exports.experimental_alternateSpellings = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.experimental_replyCount = exports.experimental_eachReply = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.experimental_hookState = exports.loadHook = exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.cleanupTmpDir = exports.makeTmpDir = exports.recordCheck = void 0;
|
|
75
|
+
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.experimental_outputStyleArms = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = void 0;
|
|
76
76
|
// --- reporting: how much did this script actually do? ---
|
|
77
77
|
// `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
|
|
78
78
|
// prints the same `✓` as one that ran and passed (measured 2026-08-08 on a file
|
|
@@ -159,6 +159,8 @@ Object.defineProperty(exports, "notTool", { enumerable: true, get: function () {
|
|
|
159
159
|
Object.defineProperty(exports, "onlyTools", { enumerable: true, get: function () { return check_js_1.onlyTools; } });
|
|
160
160
|
Object.defineProperty(exports, "skill", { enumerable: true, get: function () { return check_js_1.skill; } });
|
|
161
161
|
Object.defineProperty(exports, "output", { enumerable: true, get: function () { return check_js_1.output; } });
|
|
162
|
+
Object.defineProperty(exports, "experimental_eachReply", { enumerable: true, get: function () { return check_js_1.experimental_eachReply; } });
|
|
163
|
+
Object.defineProperty(exports, "experimental_replyCount", { enumerable: true, get: function () { return check_js_1.experimental_replyCount; } });
|
|
162
164
|
Object.defineProperty(exports, "hookFired", { enumerable: true, get: function () { return check_js_1.hookFired; } });
|
|
163
165
|
Object.defineProperty(exports, "received", { enumerable: true, get: function () { return check_js_1.received; } });
|
|
164
166
|
Object.defineProperty(exports, "turns", { enumerable: true, get: function () { return check_js_1.turns; } });
|
|
@@ -213,6 +215,10 @@ Object.defineProperty(exports, "parseHooks", { enumerable: true, get: function (
|
|
|
213
215
|
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return harness_test_js_1.decideSandbox; } });
|
|
214
216
|
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return harness_test_js_1.specTrusted; } });
|
|
215
217
|
Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return harness_test_js_1.sandboxAvailable; } });
|
|
218
|
+
// The two arms of an output-style eval, after one free run proves the style
|
|
219
|
+
// reaches the model. Free: the run uses the scripted model.
|
|
220
|
+
var output_style_arms_js_1 = require("./output-style-arms.js");
|
|
221
|
+
Object.defineProperty(exports, "experimental_outputStyleArms", { enumerable: true, get: function () { return output_style_arms_js_1.experimental_outputStyleArms; } });
|
|
216
222
|
// A document's own rule about its own COMMANDS, made checkable. vigiles cannot
|
|
217
223
|
// infer "always pass -g" from prose; what was missing was a cheap way to declare
|
|
218
224
|
// it — measured at ~95 lines of markdown parsing around ~5 lines of rule, which
|
|
@@ -728,6 +728,140 @@
|
|
|
728
728
|
}
|
|
729
729
|
]
|
|
730
730
|
},
|
|
731
|
+
"untested-output-style": {
|
|
732
|
+
"anyOf": [
|
|
733
|
+
{
|
|
734
|
+
"anyOf": [
|
|
735
|
+
{
|
|
736
|
+
"type": "string",
|
|
737
|
+
"const": "warn"
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
"type": "string",
|
|
741
|
+
"const": "error"
|
|
742
|
+
},
|
|
743
|
+
{
|
|
744
|
+
"type": "boolean",
|
|
745
|
+
"const": false
|
|
746
|
+
},
|
|
747
|
+
{
|
|
748
|
+
"type": "string",
|
|
749
|
+
"const": "off"
|
|
750
|
+
},
|
|
751
|
+
{
|
|
752
|
+
"type": "number",
|
|
753
|
+
"const": 0
|
|
754
|
+
},
|
|
755
|
+
{
|
|
756
|
+
"type": "number",
|
|
757
|
+
"const": 1
|
|
758
|
+
},
|
|
759
|
+
{
|
|
760
|
+
"type": "number",
|
|
761
|
+
"const": 2
|
|
762
|
+
},
|
|
763
|
+
{
|
|
764
|
+
"type": "boolean",
|
|
765
|
+
"const": true
|
|
766
|
+
}
|
|
767
|
+
]
|
|
768
|
+
},
|
|
769
|
+
{
|
|
770
|
+
"type": "array",
|
|
771
|
+
"prefixItems": [
|
|
772
|
+
{
|
|
773
|
+
"type": "string",
|
|
774
|
+
"const": "warn"
|
|
775
|
+
},
|
|
776
|
+
{
|
|
777
|
+
"type": "object",
|
|
778
|
+
"properties": {
|
|
779
|
+
"include": {
|
|
780
|
+
"anyOf": [
|
|
781
|
+
{
|
|
782
|
+
"type": "string"
|
|
783
|
+
},
|
|
784
|
+
{
|
|
785
|
+
"type": "array",
|
|
786
|
+
"items": {
|
|
787
|
+
"type": "string"
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
]
|
|
791
|
+
},
|
|
792
|
+
"exclude": {
|
|
793
|
+
"anyOf": [
|
|
794
|
+
{
|
|
795
|
+
"type": "string"
|
|
796
|
+
},
|
|
797
|
+
{
|
|
798
|
+
"type": "array",
|
|
799
|
+
"items": {
|
|
800
|
+
"type": "string"
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
]
|
|
804
|
+
},
|
|
805
|
+
"testExtension": {
|
|
806
|
+
"type": "string"
|
|
807
|
+
}
|
|
808
|
+
},
|
|
809
|
+
"additionalProperties": false
|
|
810
|
+
}
|
|
811
|
+
],
|
|
812
|
+
"items": false,
|
|
813
|
+
"minItems": 2,
|
|
814
|
+
"maxItems": 2
|
|
815
|
+
},
|
|
816
|
+
{
|
|
817
|
+
"type": "array",
|
|
818
|
+
"prefixItems": [
|
|
819
|
+
{
|
|
820
|
+
"type": "string",
|
|
821
|
+
"const": "error"
|
|
822
|
+
},
|
|
823
|
+
{
|
|
824
|
+
"type": "object",
|
|
825
|
+
"properties": {
|
|
826
|
+
"include": {
|
|
827
|
+
"anyOf": [
|
|
828
|
+
{
|
|
829
|
+
"type": "string"
|
|
830
|
+
},
|
|
831
|
+
{
|
|
832
|
+
"type": "array",
|
|
833
|
+
"items": {
|
|
834
|
+
"type": "string"
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
]
|
|
838
|
+
},
|
|
839
|
+
"exclude": {
|
|
840
|
+
"anyOf": [
|
|
841
|
+
{
|
|
842
|
+
"type": "string"
|
|
843
|
+
},
|
|
844
|
+
{
|
|
845
|
+
"type": "array",
|
|
846
|
+
"items": {
|
|
847
|
+
"type": "string"
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
]
|
|
851
|
+
},
|
|
852
|
+
"testExtension": {
|
|
853
|
+
"type": "string"
|
|
854
|
+
}
|
|
855
|
+
},
|
|
856
|
+
"additionalProperties": false
|
|
857
|
+
}
|
|
858
|
+
],
|
|
859
|
+
"items": false,
|
|
860
|
+
"minItems": 2,
|
|
861
|
+
"maxItems": 2
|
|
862
|
+
}
|
|
863
|
+
]
|
|
864
|
+
},
|
|
731
865
|
"unmarked-refs": {
|
|
732
866
|
"anyOf": [
|
|
733
867
|
{
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "33.
|
|
3
|
+
"version": "33.2.0",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|