vigiles 4.0.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -8
- package/dist/adapter-registry.d.ts +45 -1
- package/dist/adapter-registry.js +78 -3
- package/dist/adapters/claude-code/agent-runtime.d.ts +1 -1
- package/dist/adapters/claude-code/agent-runtime.js +1 -1
- package/dist/arg-match.d.ts +28 -0
- package/dist/arg-match.js +61 -0
- package/dist/check.d.ts +52 -1
- package/dist/check.js +121 -0
- package/dist/claude-code.d.ts +1 -0
- package/dist/claude-code.js +11 -0
- package/dist/cli.js +211 -35
- package/dist/core/types.d.ts +27 -12
- package/dist/core/validate.js +14 -3
- package/dist/eval-cache.d.ts +49 -1
- package/dist/eval-cache.js +84 -4
- package/dist/eval.d.ts +243 -14
- package/dist/eval.js +540 -28
- package/dist/integration.d.ts +15 -10
- package/dist/integration.js +27 -10
- package/dist/linting.d.ts +4 -3
- package/dist/linting.js +4 -3
- package/dist/setup-plan.d.ts +11 -0
- package/dist/setup-plan.js +29 -0
- package/dist/skill-harness.d.ts +25 -0
- package/dist/skill-harness.js +40 -0
- package/dist/test-coverage.d.ts +11 -12
- package/dist/test-coverage.js +14 -19
- package/dist/testing.d.ts +4 -1
- package/dist/testing.js +24 -2
- package/dist/tool-intercept.d.ts +101 -0
- package/dist/tool-intercept.js +165 -0
- package/dist/tool-stub.d.ts +35 -0
- package/dist/tool-stub.js +92 -0
- package/package.json +11 -17
- package/skills/test-harness/SKILL.md +56 -3
package/dist/integration.d.ts
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `vigiles/integration` — the **deterministic, assembled-machine** tier.
|
|
3
3
|
*
|
|
4
|
-
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
4
|
+
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the
|
|
5
|
+
* harness-agnostic runners for the assembled machine: `runHarnessTest` /
|
|
6
|
+
* `runHarness` (real hooks fire, outcome reproducible) plus the generic trace
|
|
7
|
+
* parsers and the sandbox policy. Capability contract: needs the **`claude`
|
|
8
|
+
* binary and bubblewrap**, but **no API key and no network**. A
|
|
9
|
+
* `*.integration.test.ts` imports from here.
|
|
10
|
+
*
|
|
11
|
+
* The Claude-Code **transport** — the scripted mock (`scriptModel`) and the
|
|
12
|
+
* plugin loader (`loadPlugin` / `resolveHarness`) — is harness-specific, so it is
|
|
13
|
+
* NOT re-exported here (this surface stays agnostic, enforced by the
|
|
14
|
+
* `agnostic-surface ⊄ adapter` boundary lint). Import it from
|
|
15
|
+
* [`vigiles/claude-code`](./claude-code.ts):
|
|
16
|
+
* `import { scriptModel, loadPlugin } from "vigiles/claude-code"`.
|
|
11
17
|
*
|
|
12
18
|
* Real **egress** is a CAPABILITY of this scope (the former `e2e` tier), not a
|
|
13
19
|
* separate tier: `egressRoutes()` probes whether allowlisted egress can route,
|
|
@@ -17,8 +23,7 @@
|
|
|
17
23
|
* `research/testing-api-design.md` Part 4.
|
|
18
24
|
*/
|
|
19
25
|
export * from "./unit.js";
|
|
20
|
-
export
|
|
21
|
-
export
|
|
22
|
-
export type { LoadedPlugin } from "./plugin-loader.js";
|
|
26
|
+
export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
|
|
27
|
+
export type { HarnessTestSpec, Trace, SubagentTrace, HarnessTestResult, RunHarnessTestOptions, ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, SandboxMode, } from "./harness-test.js";
|
|
23
28
|
export { egressRoutes } from "./run-hook.js";
|
|
24
29
|
//# sourceMappingURL=integration.d.ts.map
|
package/dist/integration.js
CHANGED
|
@@ -14,17 +14,23 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.egressRoutes = void 0;
|
|
17
|
+
exports.egressRoutes = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = exports.runHarnessTest = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/integration` — the **deterministic, assembled-machine** tier.
|
|
20
20
|
*
|
|
21
|
-
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
21
|
+
* Re-exports everything in [`vigiles/unit`](./unit.ts) and adds the
|
|
22
|
+
* harness-agnostic runners for the assembled machine: `runHarnessTest` /
|
|
23
|
+
* `runHarness` (real hooks fire, outcome reproducible) plus the generic trace
|
|
24
|
+
* parsers and the sandbox policy. Capability contract: needs the **`claude`
|
|
25
|
+
* binary and bubblewrap**, but **no API key and no network**. A
|
|
26
|
+
* `*.integration.test.ts` imports from here.
|
|
27
|
+
*
|
|
28
|
+
* The Claude-Code **transport** — the scripted mock (`scriptModel`) and the
|
|
29
|
+
* plugin loader (`loadPlugin` / `resolveHarness`) — is harness-specific, so it is
|
|
30
|
+
* NOT re-exported here (this surface stays agnostic, enforced by the
|
|
31
|
+
* `agnostic-surface ⊄ adapter` boundary lint). Import it from
|
|
32
|
+
* [`vigiles/claude-code`](./claude-code.ts):
|
|
33
|
+
* `import { scriptModel, loadPlugin } from "vigiles/claude-code"`.
|
|
28
34
|
*
|
|
29
35
|
* Real **egress** is a CAPABILITY of this scope (the former `e2e` tier), not a
|
|
30
36
|
* separate tier: `egressRoutes()` probes whether allowlisted egress can route,
|
|
@@ -34,8 +40,19 @@ exports.egressRoutes = void 0;
|
|
|
34
40
|
* `research/testing-api-design.md` Part 4.
|
|
35
41
|
*/
|
|
36
42
|
__exportStar(require("./unit.js"), exports);
|
|
37
|
-
|
|
38
|
-
|
|
43
|
+
// The harness-test tier — AGNOSTIC SURFACE ONLY (CC transport lives in
|
|
44
|
+
// vigiles/claude-code; see the module doc above).
|
|
45
|
+
var harness_test_js_1 = require("./harness-test.js");
|
|
46
|
+
Object.defineProperty(exports, "runHarnessTest", { enumerable: true, get: function () { return harness_test_js_1.runHarnessTest; } });
|
|
47
|
+
Object.defineProperty(exports, "runHarness", { enumerable: true, get: function () { return harness_test_js_1.runHarness; } });
|
|
48
|
+
Object.defineProperty(exports, "parseToolCalls", { enumerable: true, get: function () { return harness_test_js_1.parseToolCalls; } });
|
|
49
|
+
Object.defineProperty(exports, "parseSubagents", { enumerable: true, get: function () { return harness_test_js_1.parseSubagents; } });
|
|
50
|
+
Object.defineProperty(exports, "parseResultEvent", { enumerable: true, get: function () { return harness_test_js_1.parseResultEvent; } });
|
|
51
|
+
Object.defineProperty(exports, "parseOutput", { enumerable: true, get: function () { return harness_test_js_1.parseOutput; } });
|
|
52
|
+
Object.defineProperty(exports, "parseHooks", { enumerable: true, get: function () { return harness_test_js_1.parseHooks; } });
|
|
53
|
+
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return harness_test_js_1.decideSandbox; } });
|
|
54
|
+
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return harness_test_js_1.specTrusted; } });
|
|
55
|
+
Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return harness_test_js_1.sandboxAvailable; } });
|
|
39
56
|
var run_hook_js_1 = require("./run-hook.js");
|
|
40
57
|
Object.defineProperty(exports, "egressRoutes", { enumerable: true, get: function () { return run_hook_js_1.egressRoutes; } });
|
|
41
58
|
//# sourceMappingURL=integration.js.map
|
package/dist/linting.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
|
|
3
|
-
* files. Re-exports the spec builders/types
|
|
4
|
-
* concern-named import.
|
|
5
|
-
*
|
|
3
|
+
* files. Re-exports the spec builders/types, the compiler, and the linter
|
|
4
|
+
* cross-referencing engine under one concern-named import. This is the canonical
|
|
5
|
+
* pillar-1 surface; the spec builders are also at the package root (`vigiles`).
|
|
6
6
|
*/
|
|
7
7
|
export * from "./core/spec.js";
|
|
8
8
|
export * from "./core/compile.js";
|
|
9
|
+
export * from "./core/linters.js";
|
|
9
10
|
//# sourceMappingURL=linting.d.ts.map
|
package/dist/linting.js
CHANGED
|
@@ -16,10 +16,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/**
|
|
18
18
|
* `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
|
|
19
|
-
* files. Re-exports the spec builders/types
|
|
20
|
-
* concern-named import.
|
|
21
|
-
*
|
|
19
|
+
* files. Re-exports the spec builders/types, the compiler, and the linter
|
|
20
|
+
* cross-referencing engine under one concern-named import. This is the canonical
|
|
21
|
+
* pillar-1 surface; the spec builders are also at the package root (`vigiles`).
|
|
22
22
|
*/
|
|
23
23
|
__exportStar(require("./core/spec.js"), exports);
|
|
24
24
|
__exportStar(require("./core/compile.js"), exports);
|
|
25
|
+
__exportStar(require("./core/linters.js"), exports);
|
|
25
26
|
//# sourceMappingURL=linting.js.map
|
package/dist/setup-plan.d.ts
CHANGED
|
@@ -44,6 +44,17 @@ export interface ParsedSetupArgs {
|
|
|
44
44
|
export declare function parseSetupArgs(args: readonly string[]): ParsedSetupArgs;
|
|
45
45
|
/** The non-interactive defaults: both pillars, CI, and the plugin. */
|
|
46
46
|
export declare function defaultPlan(strict?: boolean): SetupPlan;
|
|
47
|
+
/**
|
|
48
|
+
* Pure config-merge for what `vigiles init` writes to `.vigilesrc.json`: record
|
|
49
|
+
* the `harness` if absent, add strict rule severities if `--strict`, NEVER
|
|
50
|
+
* clobber an existing key. Returns the merged config, or `null` when nothing
|
|
51
|
+
* changed (so the IO layer skips the write). The IO (read/parse/write + the
|
|
52
|
+
* malformed-file guard) stays in cli.ts.
|
|
53
|
+
*/
|
|
54
|
+
export declare function mergeProjectConfig(existing: Record<string, unknown>, opts: {
|
|
55
|
+
harness: string | string[];
|
|
56
|
+
strict: boolean;
|
|
57
|
+
}): Record<string, unknown> | null;
|
|
47
58
|
/**
|
|
48
59
|
* Whether to drop into interactive prompts: a human at a TTY who passed neither
|
|
49
60
|
* `--yes` nor an explicit `--target`, and who hasn't already pinned every choice
|
package/dist/setup-plan.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.parseSetupArgs = parseSetupArgs;
|
|
14
14
|
exports.defaultPlan = defaultPlan;
|
|
15
|
+
exports.mergeProjectConfig = mergeProjectConfig;
|
|
15
16
|
exports.shouldPrompt = shouldPrompt;
|
|
16
17
|
exports.planPluginInstall = planPluginInstall;
|
|
17
18
|
exports.resolvePlan = resolvePlan;
|
|
@@ -52,6 +53,34 @@ function defaultPlan(strict = false) {
|
|
|
52
53
|
force: false,
|
|
53
54
|
};
|
|
54
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* Pure config-merge for what `vigiles init` writes to `.vigilesrc.json`: record
|
|
58
|
+
* the `harness` if absent, add strict rule severities if `--strict`, NEVER
|
|
59
|
+
* clobber an existing key. Returns the merged config, or `null` when nothing
|
|
60
|
+
* changed (so the IO layer skips the write). The IO (read/parse/write + the
|
|
61
|
+
* malformed-file guard) stays in cli.ts.
|
|
62
|
+
*/
|
|
63
|
+
function mergeProjectConfig(existing, opts) {
|
|
64
|
+
const config = { ...existing };
|
|
65
|
+
let changed = false;
|
|
66
|
+
if (config.harness === undefined) {
|
|
67
|
+
config.harness = opts.harness;
|
|
68
|
+
changed = true;
|
|
69
|
+
}
|
|
70
|
+
if (opts.strict) {
|
|
71
|
+
const rules = { ...config.rules };
|
|
72
|
+
// `require-skill-spec` is deprecated (skills can be hand-written), so --strict
|
|
73
|
+
// no longer promotes it; it tightens only `require-spec` (instruction files).
|
|
74
|
+
for (const r of ["require-spec"]) {
|
|
75
|
+
if (rules[r] === undefined) {
|
|
76
|
+
rules[r] = "error";
|
|
77
|
+
changed = true;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
config.rules = rules;
|
|
81
|
+
}
|
|
82
|
+
return changed ? config : null;
|
|
83
|
+
}
|
|
55
84
|
/**
|
|
56
85
|
* Whether to drop into interactive prompts: a human at a TTY who passed neither
|
|
57
86
|
* `--yes` nor an explicit `--target`, and who hasn't already pinned every choice
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-harness skill-frontmatter verification (slice 3 of
|
|
3
|
+
* research/multi-harness-compile.md, the *verify* half).
|
|
4
|
+
*
|
|
5
|
+
* A skill's `SKILL.md` references are harness-agnostic; the one harness-specific
|
|
6
|
+
* surface is the frontmatter PROFILE. The `claude-code` profile emits CC-only
|
|
7
|
+
* keys (`disable-model-invocation`, `argument-hint`); the `minimal` profile
|
|
8
|
+
* (Codex, OpenCode) omits them. So a skill that sets those keys, in a repo that
|
|
9
|
+
* also targets a minimal-profile harness, has a silent semantic gap: the
|
|
10
|
+
* constraint the author expressed won't take effect there.
|
|
11
|
+
*
|
|
12
|
+
* This reports that gap. It is ASSUMPTION-FREE — the minimal profile *drops* the
|
|
13
|
+
* keys, so the warning states a fact about vigiles's own output, not a guess
|
|
14
|
+
* about another tool's parser tolerance.
|
|
15
|
+
*/
|
|
16
|
+
import type { SkillSpec } from "./core/spec.js";
|
|
17
|
+
/** The Claude-Code-only frontmatter keys a skill spec would emit. */
|
|
18
|
+
export declare function claudeOnlyFrontmatterKeys(spec: SkillSpec): string[];
|
|
19
|
+
/**
|
|
20
|
+
* Warn for each declared harness whose `minimal` SKILL.md profile would DROP a
|
|
21
|
+
* skill's Claude-Code-only frontmatter. Empty when the skill uses no such keys or
|
|
22
|
+
* no declared harness is minimal-profile.
|
|
23
|
+
*/
|
|
24
|
+
export declare function skillFrontmatterDropWarnings(spec: SkillSpec, harnessNames: readonly string[]): string[];
|
|
25
|
+
//# sourceMappingURL=skill-harness.d.ts.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.claudeOnlyFrontmatterKeys = claudeOnlyFrontmatterKeys;
|
|
4
|
+
exports.skillFrontmatterDropWarnings = skillFrontmatterDropWarnings;
|
|
5
|
+
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
6
|
+
/** The Claude-Code-only frontmatter keys a skill spec would emit. */
|
|
7
|
+
function claudeOnlyFrontmatterKeys(spec) {
|
|
8
|
+
const keys = [];
|
|
9
|
+
if (spec.disableModelInvocation !== undefined) {
|
|
10
|
+
keys.push("disable-model-invocation");
|
|
11
|
+
}
|
|
12
|
+
if (spec.argumentHint || (spec.inputs && spec.inputs.length > 0)) {
|
|
13
|
+
keys.push("argument-hint");
|
|
14
|
+
}
|
|
15
|
+
return keys;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Warn for each declared harness whose `minimal` SKILL.md profile would DROP a
|
|
19
|
+
* skill's Claude-Code-only frontmatter. Empty when the skill uses no such keys or
|
|
20
|
+
* no declared harness is minimal-profile.
|
|
21
|
+
*/
|
|
22
|
+
function skillFrontmatterDropWarnings(spec, harnessNames) {
|
|
23
|
+
const ccKeys = claudeOnlyFrontmatterKeys(spec);
|
|
24
|
+
if (ccKeys.length === 0)
|
|
25
|
+
return [];
|
|
26
|
+
const warnings = [];
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
for (const name of harnessNames) {
|
|
29
|
+
const adapter = (0, adapter_registry_js_1.getAdapter)(name);
|
|
30
|
+
if (!adapter || seen.has(adapter.name))
|
|
31
|
+
continue;
|
|
32
|
+
seen.add(adapter.name);
|
|
33
|
+
if (adapter.dialect.skillFrontmatter === "minimal") {
|
|
34
|
+
const one = ccKeys.length === 1;
|
|
35
|
+
warnings.push(`skill "${spec.name}": ${ccKeys.join(", ")} ${one ? "is" : "are"} Claude-Code-only — declared harness "${adapter.name}" drops ${one ? "it" : "them"}.`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return warnings;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=skill-harness.js.map
|
package/dist/test-coverage.d.ts
CHANGED
|
@@ -15,10 +15,11 @@
|
|
|
15
15
|
* surface by PATH (`skills/foo`, `hooks/pre-edit.sh`) or NAMESPACE
|
|
16
16
|
* (`vigiles:foo`). Not bare-name — too fuzzy.
|
|
17
17
|
*
|
|
18
|
-
* Warning-by-default (a nudge, not a gate).
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* Warning-by-default (a nudge, not a gate). EVERY skill, agent, and hook is held
|
|
19
|
+
* to the requirement — invocation mode does NOT exempt anything (a command-only
|
|
20
|
+
* skill still DOES something when invoked, and that behaviour is worth a test).
|
|
21
|
+
* The only opt-out is explicit: a `vigiles:ignore-test` marker in the surface
|
|
22
|
+
* file, which is reported as `exempt` so the skip is visible, never silent.
|
|
22
23
|
*/
|
|
23
24
|
export type SurfaceKind = "skill" | "agent" | "hook";
|
|
24
25
|
export interface Surface {
|
|
@@ -29,15 +30,15 @@ export interface Surface {
|
|
|
29
30
|
readonly name: string;
|
|
30
31
|
/** Substrings a test may reference to "cover" this surface (path / namespace). */
|
|
31
32
|
readonly tokens: readonly string[];
|
|
32
|
-
/**
|
|
33
|
-
readonly
|
|
33
|
+
/** Explicitly opted out of the test requirement via `vigiles:ignore-test`. */
|
|
34
|
+
readonly ignored: boolean;
|
|
34
35
|
}
|
|
35
36
|
export interface UntestedReport {
|
|
36
37
|
/** Total surfaces considered (after exemptions). */
|
|
37
38
|
readonly total: number;
|
|
38
39
|
readonly covered: readonly Surface[];
|
|
39
40
|
readonly untested: readonly Surface[];
|
|
40
|
-
/** Surfaces
|
|
41
|
+
/** Surfaces explicitly opted out via `vigiles:ignore-test`. */
|
|
41
42
|
readonly exempt: number;
|
|
42
43
|
}
|
|
43
44
|
export interface TestCoverageOptions {
|
|
@@ -49,8 +50,6 @@ export interface TestCoverageOptions {
|
|
|
49
50
|
readonly agents?: boolean;
|
|
50
51
|
/** Scan hook scripts referenced from plugin.json / settings.json. Default true. */
|
|
51
52
|
readonly hooks?: boolean;
|
|
52
|
-
/** Require a test for user-invoked (disable-model-invocation) skills. Default false. */
|
|
53
|
-
readonly includeUserInvokedSkills?: boolean;
|
|
54
53
|
/** Globs of test files that count as coverage. */
|
|
55
54
|
readonly testGlobs?: readonly string[];
|
|
56
55
|
/** Extra ignore globs (added to node_modules/dist/.git/.vigiles). */
|
|
@@ -59,9 +58,9 @@ export interface TestCoverageOptions {
|
|
|
59
58
|
/**
|
|
60
59
|
* Find harness surfaces (skills / agents / hooks) that no test or eval covers.
|
|
61
60
|
* A surface is covered by a colocated `*.{harness,eval}.mjs` OR any discovered
|
|
62
|
-
* test that references its path/namespace.
|
|
63
|
-
*
|
|
64
|
-
*
|
|
61
|
+
* test that references its path/namespace. EVERY skill, agent, and hook is held
|
|
62
|
+
* to this — the only exemption is an explicit `vigiles:ignore-test` marker in the
|
|
63
|
+
* surface file (counted as `exempt`).
|
|
65
64
|
*/
|
|
66
65
|
export declare function findUntestedSurfaces(options?: TestCoverageOptions): UntestedReport;
|
|
67
66
|
/** Suggested colocated test path for an untested surface (shown in the warning). */
|
package/dist/test-coverage.js
CHANGED
|
@@ -16,10 +16,11 @@
|
|
|
16
16
|
* surface by PATH (`skills/foo`, `hooks/pre-edit.sh`) or NAMESPACE
|
|
17
17
|
* (`vigiles:foo`). Not bare-name — too fuzzy.
|
|
18
18
|
*
|
|
19
|
-
* Warning-by-default (a nudge, not a gate).
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
19
|
+
* Warning-by-default (a nudge, not a gate). EVERY skill, agent, and hook is held
|
|
20
|
+
* to the requirement — invocation mode does NOT exempt anything (a command-only
|
|
21
|
+
* skill still DOES something when invoked, and that behaviour is worth a test).
|
|
22
|
+
* The only opt-out is explicit: a `vigiles:ignore-test` marker in the surface
|
|
23
|
+
* file, which is reported as `exempt` so the skip is visible, never silent.
|
|
23
24
|
*/
|
|
24
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
26
|
exports.findUntestedSurfaces = findUntestedSurfaces;
|
|
@@ -57,10 +58,6 @@ function read(path) {
|
|
|
57
58
|
return "";
|
|
58
59
|
}
|
|
59
60
|
}
|
|
60
|
-
/** A SKILL.md is user-invoked when its frontmatter sets disable-model-invocation. */
|
|
61
|
-
function isUserInvoked(content) {
|
|
62
|
-
return /^\s*disable-model-invocation:\s*true\s*$/m.test(content);
|
|
63
|
-
}
|
|
64
61
|
function discoverSkills(basePath, ignore) {
|
|
65
62
|
const out = [];
|
|
66
63
|
const found = (0, glob_1.globSync)(["skills/*/SKILL.md", ".claude/skills/*/SKILL.md"], {
|
|
@@ -70,14 +67,12 @@ function discoverSkills(basePath, ignore) {
|
|
|
70
67
|
for (const path of found.sort()) {
|
|
71
68
|
const name = (0, node_path_1.basename)((0, node_path_1.dirname)(path));
|
|
72
69
|
const content = read((0, node_path_1.join)(basePath, path));
|
|
73
|
-
if (content.includes(IGNORE_MARKER))
|
|
74
|
-
continue;
|
|
75
70
|
out.push({
|
|
76
71
|
kind: "skill",
|
|
77
72
|
path,
|
|
78
73
|
name,
|
|
79
74
|
tokens: [`skills/${name}`, `:${name}`],
|
|
80
|
-
|
|
75
|
+
ignored: content.includes(IGNORE_MARKER),
|
|
81
76
|
});
|
|
82
77
|
}
|
|
83
78
|
return out;
|
|
@@ -92,8 +87,6 @@ function discoverAgents(basePath, ignore) {
|
|
|
92
87
|
if (path.endsWith(".spec.ts"))
|
|
93
88
|
continue;
|
|
94
89
|
const content = read((0, node_path_1.join)(basePath, path));
|
|
95
|
-
if (content.includes(IGNORE_MARKER))
|
|
96
|
-
continue;
|
|
97
90
|
const name = (0, node_path_1.basename)(path, ".md");
|
|
98
91
|
const dir = (0, node_path_1.dirname)(path);
|
|
99
92
|
out.push({
|
|
@@ -101,7 +94,7 @@ function discoverAgents(basePath, ignore) {
|
|
|
101
94
|
path,
|
|
102
95
|
name,
|
|
103
96
|
tokens: [`${dir}/${name}`],
|
|
104
|
-
|
|
97
|
+
ignored: content.includes(IGNORE_MARKER),
|
|
105
98
|
});
|
|
106
99
|
}
|
|
107
100
|
return out;
|
|
@@ -147,7 +140,7 @@ function discoverHooks(basePath) {
|
|
|
147
140
|
path,
|
|
148
141
|
name: (0, node_path_1.basename)(path).replace(/\.[^.]+$/, ""),
|
|
149
142
|
tokens: [path],
|
|
150
|
-
|
|
143
|
+
ignored: false,
|
|
151
144
|
}));
|
|
152
145
|
}
|
|
153
146
|
function discoverTests(basePath, globs, ignore) {
|
|
@@ -186,9 +179,9 @@ function isCovered(surface, tests) {
|
|
|
186
179
|
/**
|
|
187
180
|
* Find harness surfaces (skills / agents / hooks) that no test or eval covers.
|
|
188
181
|
* A surface is covered by a colocated `*.{harness,eval}.mjs` OR any discovered
|
|
189
|
-
* test that references its path/namespace.
|
|
190
|
-
*
|
|
191
|
-
*
|
|
182
|
+
* test that references its path/namespace. EVERY skill, agent, and hook is held
|
|
183
|
+
* to this — the only exemption is an explicit `vigiles:ignore-test` marker in the
|
|
184
|
+
* surface file (counted as `exempt`).
|
|
192
185
|
*/
|
|
193
186
|
function findUntestedSurfaces(options = {}) {
|
|
194
187
|
const basePath = options.basePath ?? process.cwd();
|
|
@@ -201,7 +194,9 @@ function findUntestedSurfaces(options = {}) {
|
|
|
201
194
|
surfaces.push(...discoverAgents(basePath, ignore));
|
|
202
195
|
if (options.hooks !== false)
|
|
203
196
|
surfaces.push(...discoverHooks(basePath));
|
|
204
|
-
|
|
197
|
+
// Every skill/agent/hook is held to the requirement — only an explicit
|
|
198
|
+
// `vigiles:ignore-test` marker exempts a surface (a visible, deliberate skip).
|
|
199
|
+
const considered = surfaces.filter((s) => !s.ignored);
|
|
205
200
|
const exempt = surfaces.length - considered.length;
|
|
206
201
|
const tests = discoverTests(basePath, globs, ignore);
|
|
207
202
|
const covered = [];
|
package/dist/testing.d.ts
CHANGED
|
@@ -11,9 +11,12 @@
|
|
|
11
11
|
* `research/adapter-api-design.md`.
|
|
12
12
|
*/
|
|
13
13
|
export * from "./run-hook.js";
|
|
14
|
-
export * from "./harness-test.js";
|
|
15
14
|
export * from "./eval.js";
|
|
16
15
|
export * from "./harness-assert.js";
|
|
17
16
|
export * from "./check.js";
|
|
18
17
|
export { hookFired } from "./check.js";
|
|
18
|
+
export { judge } from "./judge.js";
|
|
19
|
+
export * from "./tool-stub.js";
|
|
20
|
+
export { runHarnessTest, runHarness, parseToolCalls, parseSubagents, parseResultEvent, parseOutput, parseHooks, decideSandbox, specTrusted, sandboxAvailable, } from "./harness-test.js";
|
|
21
|
+
export type { HarnessTestSpec, Trace, SubagentTrace, HarnessTestResult, RunHarnessTestOptions, ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, SandboxMode, } from "./harness-test.js";
|
|
19
22
|
//# sourceMappingURL=testing.d.ts.map
|
package/dist/testing.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.hookFired = void 0;
|
|
17
|
+
exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = exports.runHarnessTest = exports.judge = exports.hookFired = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/testing` — Pillar 2 entry point: the **harness-testing** API. Re-exports
|
|
20
20
|
* the three tiers — `runHook` (unit), `runHarnessTest` (deterministic), `runEval`
|
|
@@ -28,7 +28,6 @@ exports.hookFired = void 0;
|
|
|
28
28
|
* `research/adapter-api-design.md`.
|
|
29
29
|
*/
|
|
30
30
|
__exportStar(require("./run-hook.js"), exports);
|
|
31
|
-
__exportStar(require("./harness-test.js"), exports);
|
|
32
31
|
__exportStar(require("./eval.js"), exports);
|
|
33
32
|
__exportStar(require("./harness-assert.js"), exports);
|
|
34
33
|
// The declarative check vocabulary is now first-class at the front door. Its
|
|
@@ -37,4 +36,27 @@ __exportStar(require("./harness-assert.js"), exports);
|
|
|
37
36
|
__exportStar(require("./check.js"), exports);
|
|
38
37
|
var check_js_1 = require("./check.js");
|
|
39
38
|
Object.defineProperty(exports, "hookFired", { enumerable: true, get: function () { return check_js_1.hookFired; } });
|
|
39
|
+
// The model-graded judge (agnostic — grades text against a rubric). Public here
|
|
40
|
+
// now that `vigiles/judge` is no longer a standalone subpath.
|
|
41
|
+
var judge_js_1 = require("./judge.js");
|
|
42
|
+
Object.defineProperty(exports, "judge", { enumerable: true, get: function () { return judge_js_1.judge; } });
|
|
43
|
+
// Tool stubs on PATH (rung R2): shadow a CLI tool with a recorded canned result.
|
|
44
|
+
__exportStar(require("./tool-stub.js"), exports);
|
|
45
|
+
// The harness-test tier — AGNOSTIC SURFACE ONLY. The Claude-Code transport
|
|
46
|
+
// (`scriptModel`, `claudeCodeDriver`, `buildClaudeArgs`, `parseClaudeRun`,
|
|
47
|
+
// `claudeAvailable`, `loadPlugin`, `resolveHarness`) is deliberately NOT
|
|
48
|
+
// re-exported here, so `vigiles/testing` stays harness-agnostic — import those
|
|
49
|
+
// from `vigiles/claude-code` (or your harness's package). See
|
|
50
|
+
// `research/code-adapter-architecture.md`.
|
|
51
|
+
var harness_test_js_1 = require("./harness-test.js");
|
|
52
|
+
Object.defineProperty(exports, "runHarnessTest", { enumerable: true, get: function () { return harness_test_js_1.runHarnessTest; } });
|
|
53
|
+
Object.defineProperty(exports, "runHarness", { enumerable: true, get: function () { return harness_test_js_1.runHarness; } });
|
|
54
|
+
Object.defineProperty(exports, "parseToolCalls", { enumerable: true, get: function () { return harness_test_js_1.parseToolCalls; } });
|
|
55
|
+
Object.defineProperty(exports, "parseSubagents", { enumerable: true, get: function () { return harness_test_js_1.parseSubagents; } });
|
|
56
|
+
Object.defineProperty(exports, "parseResultEvent", { enumerable: true, get: function () { return harness_test_js_1.parseResultEvent; } });
|
|
57
|
+
Object.defineProperty(exports, "parseOutput", { enumerable: true, get: function () { return harness_test_js_1.parseOutput; } });
|
|
58
|
+
Object.defineProperty(exports, "parseHooks", { enumerable: true, get: function () { return harness_test_js_1.parseHooks; } });
|
|
59
|
+
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return harness_test_js_1.decideSandbox; } });
|
|
60
|
+
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return harness_test_js_1.specTrusted; } });
|
|
61
|
+
Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return harness_test_js_1.sandboxAvailable; } });
|
|
40
62
|
//# sourceMappingURL=testing.js.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool interception — the eval-tier half of the tool-call spy.
|
|
3
|
+
*
|
|
4
|
+
* The eval tier drives the REAL model, so the agent's tool decisions are genuine.
|
|
5
|
+
* But letting a side-effecting tool actually run — a paid image API, `git push`,
|
|
6
|
+
* spawning a paid subagent — makes the eval expensive and dangerous. An
|
|
7
|
+
* **intercept** lets the model emit the `tool_use` (so its arguments are captured
|
|
8
|
+
* in the `Trace`, where `toolWith` / `notTool` assert on them) while a PreToolUse
|
|
9
|
+
* hook **denies the real execution** with a block message. The run stays cheap and
|
|
10
|
+
* side-effect-free, but the agent's *decision* — the thing a completion grader
|
|
11
|
+
* can't see — is fully observable.
|
|
12
|
+
*
|
|
13
|
+
* This module is the PURE decision + wiring core, mirroring the agent-contract
|
|
14
|
+
* rail (`src/adapters/claude-code/agent-runtime.ts`):
|
|
15
|
+
*
|
|
16
|
+
* - `decideIntercept` — does this call get intercepted, and with what deny reason;
|
|
17
|
+
* - `buildInterceptSettings` — the PreToolUse hook fragment routing matched tools
|
|
18
|
+
* through `vigiles intercept-tool-hook`;
|
|
19
|
+
* - `serializeIntercepts` / `parseIntercepts` — the env round-trip (incl. RegExp
|
|
20
|
+
* matchers) the hook subprocess reads back.
|
|
21
|
+
*
|
|
22
|
+
* The hook denies via exit 2 + a stderr message (the same block mechanism the
|
|
23
|
+
* agent rail uses). IMPORTANT — Claude Code surfaces that to the model as a
|
|
24
|
+
* *blocked* call: the tool is intercepted (prevented), NOT executed, and the model
|
|
25
|
+
* is NOT handed a faked successful return. So this is **intercept-and-prevent +
|
|
26
|
+
* observe the attempt**, not a faithful tool mock: it's sound for "did the agent
|
|
27
|
+
* ATTEMPT X / call it with these args / push to the wrong branch" (first-attempt
|
|
28
|
+
* questions, where what happens after doesn't matter), and unsound for "stub the
|
|
29
|
+
* tool and let the trajectory continue as if it succeeded" (the model is told it
|
|
30
|
+
* was blocked, so a multi-step flow that needs the real result will derail). CC
|
|
31
|
+
* exposes no "skip execution but return this as success" primitive for arbitrary
|
|
32
|
+
* tools, so deny+reason is the closest available — with that ceiling.
|
|
33
|
+
*/
|
|
34
|
+
import { type ArgMatcher } from "./arg-match.js";
|
|
35
|
+
/** Env var the spawned `vigiles intercept-tool-hook` reads its intercept list from. */
|
|
36
|
+
export declare const INTERCEPT_TOOLS_ENV = "VIGILES_INTERCEPT_TOOLS";
|
|
37
|
+
/** Declare a tool to intercept: deny its real execution with a block message. */
|
|
38
|
+
export interface ToolIntercept {
|
|
39
|
+
/** The tool to intercept (e.g. `"Bash"`, `"WebFetch"`, `"Task"`, `"mcp__img__gen"`). */
|
|
40
|
+
readonly tool: string;
|
|
41
|
+
/**
|
|
42
|
+
* Only intercept calls whose `input` matches (so you can intercept `git push`
|
|
43
|
+
* while letting other `Bash` through). Omit to intercept every call to `tool`.
|
|
44
|
+
*/
|
|
45
|
+
readonly when?: ArgMatcher;
|
|
46
|
+
/**
|
|
47
|
+
* The denial reason shown to the model when the call is intercepted. This is a
|
|
48
|
+
* *block* message: the call is intercepted (prevented), NOT executed — so phrase
|
|
49
|
+
* it as "this was prevented; don't retry", not "here's your result".
|
|
50
|
+
* Defaults to {@link DEFAULT_INTERCEPT_REASON}.
|
|
51
|
+
*/
|
|
52
|
+
readonly denyReason?: string;
|
|
53
|
+
}
|
|
54
|
+
/** The default denial reason — honest that the call was intercepted (prevented), NOT executed. */
|
|
55
|
+
export declare const DEFAULT_INTERCEPT_REASON: string;
|
|
56
|
+
/** The decision for one tool call: intercept it (deny + reason), or let it run. */
|
|
57
|
+
export type InterceptDecision = {
|
|
58
|
+
readonly intercept: true;
|
|
59
|
+
readonly denyReason: string;
|
|
60
|
+
} | {
|
|
61
|
+
readonly intercept: false;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Decide whether a tool call should be intercepted. Returns the first matching
|
|
65
|
+
* intercept's denial reason (preventing real execution), or `{ intercept: false }`
|
|
66
|
+
* to let the call run for real. Pure — the same logic `vigiles intercept-tool-hook`
|
|
67
|
+
* runs.
|
|
68
|
+
*/
|
|
69
|
+
export declare function decideIntercept(toolName: string, input: unknown, intercepts: readonly ToolIntercept[]): InterceptDecision;
|
|
70
|
+
/**
|
|
71
|
+
* Decide from a raw PreToolUse event JSON (the hook's stdin). Parses `tool_name`
|
|
72
|
+
* + `tool_input`, then defers to {@link decideIntercept}. Malformed input or a
|
|
73
|
+
* missing tool name is a no-op (let it run) — never fail closed on a parse error.
|
|
74
|
+
*/
|
|
75
|
+
export declare function interceptHookDecision(rawEvent: string, intercepts: readonly ToolIntercept[]): InterceptDecision;
|
|
76
|
+
/** A `.claude/settings.json` fragment registering the intercept PreToolUse hook. */
|
|
77
|
+
export interface InterceptSettings {
|
|
78
|
+
readonly hooks: {
|
|
79
|
+
readonly PreToolUse: readonly unknown[];
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Build the PreToolUse hook fragment that routes every intercepted tool through
|
|
84
|
+
* `vigiles intercept-tool-hook`. The `matcher` is a CC tool-name regex over the
|
|
85
|
+
* union of intercepted tool names (each escaped), so unrelated tools are never
|
|
86
|
+
* intercepted. Merge the result into an arm's `settings`; the intercept list
|
|
87
|
+
* itself travels in the {@link INTERCEPT_TOOLS_ENV} env var (see
|
|
88
|
+
* {@link serializeIntercepts}).
|
|
89
|
+
*/
|
|
90
|
+
export declare function buildInterceptSettings(intercepts: readonly ToolIntercept[], opts?: {
|
|
91
|
+
command?: string;
|
|
92
|
+
}): InterceptSettings;
|
|
93
|
+
/**
|
|
94
|
+
* Serialize an intercept list to a JSON string for {@link INTERCEPT_TOOLS_ENV}.
|
|
95
|
+
* RegExp matchers are encoded as `{ re, flags }` so they round-trip exactly (a
|
|
96
|
+
* plain `JSON.stringify` would drop them to `{}`).
|
|
97
|
+
*/
|
|
98
|
+
export declare function serializeIntercepts(intercepts: readonly ToolIntercept[]): string;
|
|
99
|
+
/** Parse an intercept list from the env JSON (tolerant — a bad entry is skipped). */
|
|
100
|
+
export declare function parseIntercepts(json: string): ToolIntercept[];
|
|
101
|
+
//# sourceMappingURL=tool-intercept.d.ts.map
|