vigiles 27.2.0 โ 27.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-conformance.js +2 -2
- package/dist/adapters/claude-code/run-scripts.js +47 -8
- package/dist/cli-main.js +42 -4
- package/dist/core/linters.js +3 -3
- package/dist/core/test-utils.d.ts +1 -2
- package/dist/core/test-utils.js +7 -9
- package/dist/core/tmp-root.d.ts +12 -0
- package/dist/core/tmp-root.js +59 -0
- package/dist/eval.js +6 -5
- package/dist/harness-test.js +2 -2
- package/dist/run-script.js +3 -3
- package/dist/sandbox.js +2 -2
- package/dist/scan-behavioral.js +2 -2
- package/dist/test.d.ts +1 -0
- package/dist/test.js +17 -2
- package/package.json +1 -1
|
@@ -13,13 +13,13 @@ exports.assertAdapterLoadsHooks = assertAdapterLoadsHooks;
|
|
|
13
13
|
* in their test suite. See `docs/authoring-an-adapter.md`.
|
|
14
14
|
*/
|
|
15
15
|
const node_fs_1 = require("node:fs");
|
|
16
|
-
const node_os_1 = require("node:os");
|
|
17
16
|
const node_path_1 = require("node:path");
|
|
18
17
|
const compile_js_1 = require("./core/compile.js");
|
|
19
18
|
const spec_js_1 = require("./core/spec.js");
|
|
20
19
|
const plugin_loader_js_1 = require("./plugin-loader.js");
|
|
21
20
|
const vocabulary_consistency_js_1 = require("./core/vocabulary-consistency.js");
|
|
22
21
|
const event_capability_js_1 = require("./core/event-capability.js");
|
|
22
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
23
23
|
/** Check an adapter against the port contracts; returns the (possibly empty) failure list. */
|
|
24
24
|
function checkAdapterConformance(adapter) {
|
|
25
25
|
const failures = [];
|
|
@@ -166,7 +166,7 @@ function assertHarnessTestable(adapter) {
|
|
|
166
166
|
* run with zero hooks. Does filesystem IO, so it's a separate opt-in assert.
|
|
167
167
|
*/
|
|
168
168
|
function assertAdapterLoadsHooks(adapter) {
|
|
169
|
-
const dir = (0,
|
|
169
|
+
const dir = (0, tmp_root_js_1.makeTmpDir)("conformance");
|
|
170
170
|
try {
|
|
171
171
|
const settingsAbs = (0, node_path_1.join)(dir, adapter.layout.settingsPath);
|
|
172
172
|
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(settingsAbs), { recursive: true });
|
|
@@ -26,7 +26,6 @@ const node_os_1 = require("node:os");
|
|
|
26
26
|
const node_path_1 = require("node:path");
|
|
27
27
|
const node_url_1 = require("node:url");
|
|
28
28
|
const node_fs_1 = require("node:fs");
|
|
29
|
-
const node_os_2 = require("node:os");
|
|
30
29
|
const glob_1 = require("glob");
|
|
31
30
|
const check_count_js_1 = require("../../check-count.js");
|
|
32
31
|
/**
|
|
@@ -64,8 +63,21 @@ exports.SKIP_EXIT_CODE = 77;
|
|
|
64
63
|
function statusFor(code, checks, output) {
|
|
65
64
|
if (code === exports.SKIP_EXIT_CODE)
|
|
66
65
|
return "skip";
|
|
66
|
+
// ๐ด `checks === undefined` GUARDS THE TEXT MATCH, and it is the load-bearing
|
|
67
|
+
// half. A script that REPORTED a count executed: the counter is written by an
|
|
68
|
+
// exit handler that exists only once the module was linked and run
|
|
69
|
+
// (`check-count.ts`), so the count is a STRUCTURAL fact about the child, while
|
|
70
|
+
// `didNotLoad` is a guess about its text.
|
|
71
|
+
//
|
|
72
|
+
// MEASURED: `statusFor(1, 3, "<hook stderr: Cannot find module โฆ>\nAssertionError")`
|
|
73
|
+
// returned `"skip"` and the run exited 0 โ a harness that ran, recorded three
|
|
74
|
+
// checks and FAILED an assertion, reported as skipped. That is not an exotic
|
|
75
|
+
// input: vigiles harnesses drive hooks and print their transcripts, so a
|
|
76
|
+
// loader phrase in the output is ordinary EVIDENCE about the thing under test,
|
|
77
|
+
// not a diagnosis of the harness. Watching the child from outside cannot tell
|
|
78
|
+
// those apart. The count can, and it was already in hand.
|
|
67
79
|
if (code !== 0)
|
|
68
|
-
return didNotLoad(output) ? "skip" : "fail";
|
|
80
|
+
return checks === undefined && didNotLoad(output) ? "skip" : "fail";
|
|
69
81
|
return checks === 0 ? "vacuous" : "pass";
|
|
70
82
|
}
|
|
71
83
|
/**
|
|
@@ -83,10 +95,17 @@ function statusFor(code, checks, output) {
|
|
|
83
95
|
* 'recordCheck' not found`, and the ledger dropped from 48 records to 34 and from
|
|
84
96
|
* 47 to 33. Nothing about those surfaces had changed โ the machine had.
|
|
85
97
|
*
|
|
86
|
-
* โ ๏ธ This does NOT make a broken environment quiet
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
* measurement taken on a machine
|
|
98
|
+
* โ ๏ธ This does NOT make a broken environment quiet: a script classified here
|
|
99
|
+
* fails the run by default (`cli-main.ts`, right after `anyFailed`), because a
|
|
100
|
+
* skip the AUTHOR never declared is not a skip. What this classification buys is
|
|
101
|
+
* only that a machine problem may not delete a measurement taken on a machine
|
|
102
|
+
* that worked.
|
|
103
|
+
*
|
|
104
|
+
* ๐ด That default is new, and the sentence it replaces was false. It read
|
|
105
|
+
* ยซ`--no-skip` โ which this repo's own CI passes โ still fails the runยป.
|
|
106
|
+
* Measured: `--no-skip` appears ZERO times under `.github/`, `package.json`,
|
|
107
|
+
* `scripts/` and `.claude/`; CI passes `--min=14` and nothing else. The
|
|
108
|
+
* safety net the non-fatal classification leaned on was never strung.
|
|
90
109
|
*
|
|
91
110
|
* Deliberately literal, and only the loader's own vocabulary: these strings come
|
|
92
111
|
* from Node's module resolution, not from user code. A test that legitimately
|
|
@@ -99,6 +118,13 @@ function didNotLoad(output) {
|
|
|
99
118
|
output.includes("Cannot find package") ||
|
|
100
119
|
output.includes("Cannot find module") ||
|
|
101
120
|
/SyntaxError: Named export '[^']*' not found/.test(output) ||
|
|
121
|
+
// Same event, ESM spelling. Node phrases a missing named export one way for
|
|
122
|
+
// a CommonJS target and another for an ES module, and only the first was
|
|
123
|
+
// listed โ so the 2026-08-20 class below still RETRACTED coverage whenever
|
|
124
|
+
// the dependency happened to be ESM. Measured on Node 22:
|
|
125
|
+
// CJS: SyntaxError: Named export 'recordCheck' not found. The requested module โฆ
|
|
126
|
+
// ESM: SyntaxError: The requested module './x.mjs' does not provide an export named 'recordCheck'
|
|
127
|
+
/SyntaxError: The requested module '[^']*' does not provide an export named/.test(output) ||
|
|
102
128
|
output.includes("ERR_UNSUPPORTED_DIR_IMPORT") ||
|
|
103
129
|
output.includes("ERR_PACKAGE_PATH_NOT_EXPORTED"));
|
|
104
130
|
}
|
|
@@ -118,6 +144,7 @@ var ts_runner_caps_js_1 = require("../../ts-runner-caps.js");
|
|
|
118
144
|
Object.defineProperty(exports, "detectNodeCaps", { enumerable: true, get: function () { return ts_runner_caps_js_1.detectNodeCaps; } });
|
|
119
145
|
Object.defineProperty(exports, "canRunTypeScript", { enumerable: true, get: function () { return ts_runner_caps_js_1.canRunTypeScript; } });
|
|
120
146
|
const ts_runner_caps_js_2 = require("../../ts-runner-caps.js");
|
|
147
|
+
const tmp_root_js_1 = require("../../core/tmp-root.js");
|
|
121
148
|
/**
|
|
122
149
|
* The `node` argv (after the binary) to run a single script. Plain JS runs
|
|
123
150
|
* directly; a TypeScript script picks `tsx` when available, else Node's native
|
|
@@ -159,7 +186,13 @@ function discoverScripts(patterns, defaultGlob, cwd, ignore) {
|
|
|
159
186
|
const globs = patterns.length > 0 ? patterns : [defaultGlob];
|
|
160
187
|
const found = new Set();
|
|
161
188
|
for (const p of globs) {
|
|
162
|
-
|
|
189
|
+
// ๐ด `isFile`, not `existsSync`: a DIRECTORY exists too. `vigiles test .`
|
|
190
|
+
// therefore passed `.` through as a script, `spawn("node", ["."])` died with
|
|
191
|
+
// Node's `ERR_UNSUPPORTED_DIR_IMPORT` stack, and the classifier downstream
|
|
192
|
+
// read that stack as "did not load" โ a crash reported as a skip, exit 0.
|
|
193
|
+
// A directory now contributes no files, so the caller's own loud
|
|
194
|
+
// nothing-matched path owns the message (see `cli-main.ts`).
|
|
195
|
+
if ((0, node_fs_1.statSync)((0, node_path_1.resolve)(cwd, p), { throwIfNoEntry: false })?.isFile()) {
|
|
163
196
|
found.add(p);
|
|
164
197
|
continue;
|
|
165
198
|
}
|
|
@@ -174,10 +207,16 @@ function discoverScripts(patterns, defaultGlob, cwd, ignore) {
|
|
|
174
207
|
// `test-coverage.ts` and `cli.ts` both pass `dot: true` with comments saying why, and
|
|
175
208
|
// `test-coverage.test.ts` records "glob without `dot:true` never found it and the surface
|
|
176
209
|
// looked untested". Coverage learned it; the runner did not.
|
|
210
|
+
// ๐ด `nodir: true` for the same reason as the `isFile` guard above, and the
|
|
211
|
+
// guard alone was NOT enough โ caught by its own test. A pattern that names
|
|
212
|
+
// an existing directory (`sub`, `.`) skips the fast path and then comes back
|
|
213
|
+
// out of the globber, because a directory matches a glob perfectly well.
|
|
214
|
+
// Asked of the globber rather than filtered afterwards: it already knows.
|
|
177
215
|
for (const m of (0, glob_1.globSync)(p, {
|
|
178
216
|
cwd,
|
|
179
217
|
ignore: [...ignore],
|
|
180
218
|
dot: true,
|
|
219
|
+
nodir: true,
|
|
181
220
|
})) {
|
|
182
221
|
found.add(m);
|
|
183
222
|
}
|
|
@@ -219,7 +258,7 @@ function readCheckReport(path) {
|
|
|
219
258
|
*/
|
|
220
259
|
async function runScripts(files, cwd, env = {}, opts = {}) {
|
|
221
260
|
const caps = (0, ts_runner_caps_js_2.detectNodeCaps)(cwd);
|
|
222
|
-
const countDir = (0,
|
|
261
|
+
const countDir = (0, tmp_root_js_1.makeTmpDir)("checks");
|
|
223
262
|
// ๐ด THE DEFAULT IS DECIDED BY `entry`, NOT BY A FLAG, because the two commands
|
|
224
263
|
// that share this runner have OPPOSITE right answers and the caller already
|
|
225
264
|
// distinguishes them:
|
package/dist/cli-main.js
CHANGED
|
@@ -4936,10 +4936,19 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
|
|
|
4936
4936
|
// VALUE cannot tell them apart โ only the flag's presence can. (Caught by a control:
|
|
4937
4937
|
// the first version read `minRequired === 0` and made `--min=0` do nothing.)
|
|
4938
4938
|
if (restArgs.length > 0 && minFlag === undefined) {
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
|
|
4942
|
-
|
|
4939
|
+
// A DIRECTORY is the one stale-looking target whose cause we actually know,
|
|
4940
|
+
// so say it instead of listing the three guesses. `vigiles test .` used to
|
|
4941
|
+
// reach `spawn("node", ["."])` and surface Node's module-resolution stack;
|
|
4942
|
+
// the generic message above would now be true but unhelpful.
|
|
4943
|
+
const dirs = restArgs.filter((a) => (0, node_fs_1.lstatSync)(a, { throwIfNoEntry: false })?.isDirectory() === true);
|
|
4944
|
+
console.error(dirs.length > 0
|
|
4945
|
+
? `โ vigiles ${kind}: ${dirs.join(", ")} ${dirs.length === 1 ? "is a directory" : "are directories"} โ ` +
|
|
4946
|
+
`pass a file, or a glob like "${defaultGlob}".\n` +
|
|
4947
|
+
` A directory is not a script; nothing ran.`
|
|
4948
|
+
: `โ vigiles ${kind}: ${String(restArgs.length)} target(s) given and NOTHING matched โ ` +
|
|
4949
|
+
`${restArgs.join(", ")}\n` +
|
|
4950
|
+
` Nothing ran. A stale path, a wrong glob, or a moved file all look like this.\n` +
|
|
4951
|
+
` If an empty match is expected here, say so with --min=0.`);
|
|
4943
4952
|
process.exit(1);
|
|
4944
4953
|
}
|
|
4945
4954
|
console.log(`No ${defaultGlob} files found.`);
|
|
@@ -5002,6 +5011,35 @@ async function handleRunScripts(kind, args, restArgs, excludes) {
|
|
|
5002
5011
|
console.log("\n" + (0, run_scripts_js_1.formatScriptSummary)(results));
|
|
5003
5012
|
if ((0, run_scripts_js_1.anyFailed)(results))
|
|
5004
5013
|
process.exit(1);
|
|
5014
|
+
// ๐ด A SKIP THE AUTHOR NEVER DECLARED IS NOT A SKIP โ and the discriminator was
|
|
5015
|
+
// already sitting in the result. `skip()` exits 77 (`SKIP_EXIT_CODE`); a script
|
|
5016
|
+
// the runtime could not evaluate exits with whatever the loader gave it, 1 in
|
|
5017
|
+
// practice. Both are classified `"skip"` so that neither RETRACTS coverage โ
|
|
5018
|
+
// which is right, a file that did not run proved nothing either way โ but only
|
|
5019
|
+
// the declared one is a reason to stay green.
|
|
5020
|
+
//
|
|
5021
|
+
// Reported as #243: `vigiles test .` printed a resolver stack over a `โ`, said
|
|
5022
|
+
// `0 passed, 1 skipped`, and exited 0. Downstream a consumer's README shipped
|
|
5023
|
+
// that exact command as its first setup step, so a new reader's suite silently
|
|
5024
|
+
// never ran. `--no-skip` would have caught it and is not the default; `--min=1`
|
|
5025
|
+
// does not, because it counts files MATCHED, not scripts executed.
|
|
5026
|
+
//
|
|
5027
|
+
// This is deliberately NOT a fifth `ScriptStatus`. Coverage retraction reads the
|
|
5028
|
+
// status as a bare STRING (`executedScripts`, `coverage-artifact.ts`, whose
|
|
5029
|
+
// parameter is typed `string`), so a new member would start retracting silently
|
|
5030
|
+
// with no type error โ breaking the one property the classification exists to
|
|
5031
|
+
// protect.
|
|
5032
|
+
const notEvaluated = results.filter((r) => r.status === "skip" && r.code !== run_scripts_js_1.SKIP_EXIT_CODE);
|
|
5033
|
+
if (notEvaluated.length > 0) {
|
|
5034
|
+
console.error(`\nโ vigiles ${kind}: ${String(notEvaluated.length)} script(s) never ran โ the runtime could not load them:\n` +
|
|
5035
|
+
notEvaluated
|
|
5036
|
+
.map((r) => ` ${r.file} (exit ${String(r.code)})`)
|
|
5037
|
+
.join("\n") +
|
|
5038
|
+
`\n Their previous coverage is kept, because a script that did not run retracts nothing.\n` +
|
|
5039
|
+
` If a missing dependency is expected here, import it dynamically and call skip() โ ` +
|
|
5040
|
+
`a declared skip stays green.`);
|
|
5041
|
+
process.exit(1);
|
|
5042
|
+
}
|
|
5005
5043
|
// `--no-skip`: in a context that ASSERTS the capability is present (a CI job),
|
|
5006
5044
|
// a skipped tier is untested surface โ fail loudly instead of passing green.
|
|
5007
5045
|
if (args.includes("--no-skip") && results.some((r) => r.status === "skip")) {
|
package/dist/core/linters.js
CHANGED
|
@@ -36,7 +36,6 @@ exports.parseGolangciEnabledLinters = parseGolangciEnabledLinters;
|
|
|
36
36
|
exports.clearCedarCache = clearCedarCache;
|
|
37
37
|
exports.checkLinterRule = checkLinterRule;
|
|
38
38
|
const node_fs_1 = require("node:fs");
|
|
39
|
-
const node_os_1 = require("node:os");
|
|
40
39
|
const node_path_1 = require("node:path");
|
|
41
40
|
const edit_distance_js_1 = require("./edit-distance.js");
|
|
42
41
|
Object.defineProperty(exports, "editDistance", { enumerable: true, get: function () { return edit_distance_js_1.editDistance; } });
|
|
@@ -72,6 +71,7 @@ function augmentToolPath() {
|
|
|
72
71
|
}
|
|
73
72
|
}
|
|
74
73
|
augmentToolPath();
|
|
74
|
+
const tmp_root_js_1 = require("./tmp-root.js");
|
|
75
75
|
// ---------------------------------------------------------------------------
|
|
76
76
|
// Parsing enforcement references
|
|
77
77
|
// ---------------------------------------------------------------------------
|
|
@@ -267,7 +267,7 @@ function getDetektDefaultRules() {
|
|
|
267
267
|
if (DETEKT_DEFAULT_RULE_CACHE)
|
|
268
268
|
return DETEKT_DEFAULT_RULE_CACHE;
|
|
269
269
|
let rules = new Set();
|
|
270
|
-
const tmp = (0,
|
|
270
|
+
const tmp = (0, tmp_root_js_1.makeTmpDir)("detekt");
|
|
271
271
|
try {
|
|
272
272
|
const target = (0, node_path_1.join)(tmp, "generated-default.yml");
|
|
273
273
|
(0, node_child_process_1.execSync)(`detekt --generate-config --config ${target}`, {
|
|
@@ -481,7 +481,7 @@ function runCheckstyleProbe(configPath, probePath) {
|
|
|
481
481
|
* either placement instantiates.
|
|
482
482
|
*/
|
|
483
483
|
function checkstyleModuleInstantiates(ruleName) {
|
|
484
|
-
const tmp = (0,
|
|
484
|
+
const tmp = (0, tmp_root_js_1.makeTmpDir)("checkstyle");
|
|
485
485
|
try {
|
|
486
486
|
const probe = (0, node_path_1.join)(tmp, "Probe.java");
|
|
487
487
|
(0, node_fs_1.writeFileSync)(probe, "class Probe {}\n");
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { ClaudeSpec } from "./spec.js";
|
|
2
|
-
export
|
|
3
|
-
export declare function cleanupTmpDir(dir: string): void;
|
|
2
|
+
export { makeTmpDir, cleanupTmpDir } from "./tmp-root.js";
|
|
4
3
|
export declare function makeSpec(overrides?: Partial<ClaudeSpec>): ClaudeSpec;
|
|
5
4
|
declare function git(cwd: string, cmd: string): string;
|
|
6
5
|
export declare function initGitRepo(dir: string): void;
|
package/dist/core/test-utils.js
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.makeTmpDir =
|
|
4
|
-
exports.cleanupTmpDir = cleanupTmpDir;
|
|
3
|
+
exports.cleanupTmpDir = exports.makeTmpDir = void 0;
|
|
5
4
|
exports.makeSpec = makeSpec;
|
|
6
5
|
exports.initGitRepo = initGitRepo;
|
|
7
6
|
exports.git = git;
|
|
8
7
|
const node_fs_1 = require("node:fs");
|
|
9
8
|
const node_path_1 = require("node:path");
|
|
10
|
-
const node_os_1 = require("node:os");
|
|
11
9
|
const node_child_process_1 = require("node:child_process");
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
}
|
|
10
|
+
// Re-exported, not redefined: the temp root lives in `tmp-root.ts` because the
|
|
11
|
+
// runtime modules that need one must not pull in `makeSpec`/`initGitRepo` and
|
|
12
|
+
// their dependencies. One definition, two doors.
|
|
13
|
+
var tmp_root_js_1 = require("./tmp-root.js");
|
|
14
|
+
Object.defineProperty(exports, "makeTmpDir", { enumerable: true, get: function () { return tmp_root_js_1.makeTmpDir; } });
|
|
15
|
+
Object.defineProperty(exports, "cleanupTmpDir", { enumerable: true, get: function () { return tmp_root_js_1.cleanupTmpDir; } });
|
|
18
16
|
function makeSpec(overrides) {
|
|
19
17
|
return {
|
|
20
18
|
_specType: "claude",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Create a temporary directory and return the path with every symlink resolved.
|
|
3
|
+
*
|
|
4
|
+
* The `realpathSync` wrapper is the whole point: it must stay OUTSIDE
|
|
5
|
+
* `mkdtempSync`, because the directory has to exist before it can be resolved.
|
|
6
|
+
*
|
|
7
|
+
* @param suffix distinguishes roots in a listing; the name is `vigiles-<suffix>-*`
|
|
8
|
+
*/
|
|
9
|
+
export declare function makeTmpDir(suffix?: string): string;
|
|
10
|
+
/** Remove a root made by {@link makeTmpDir}. Safe on a path that is already gone. */
|
|
11
|
+
export declare function cleanupTmpDir(dir: string): void;
|
|
12
|
+
//# sourceMappingURL=tmp-root.d.ts.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.makeTmpDir = makeTmpDir;
|
|
4
|
+
exports.cleanupTmpDir = cleanupTmpDir;
|
|
5
|
+
/**
|
|
6
|
+
* The ONE way to make a temporary fixture root, with its symlinks resolved.
|
|
7
|
+
*
|
|
8
|
+
* โโ THE TRAP THIS EXISTS TO ABOLISH (issue #241, measured) โโโโโโโโโโโโโโโโโโโโโโ
|
|
9
|
+
* On macOS `os.tmpdir()` returns a path under `/var/folders/โฆ`, and `/var` is
|
|
10
|
+
* itself a symlink to `/private/var`. Node resolves a module's own URL to the
|
|
11
|
+
* REALPATH but leaves `process.argv[1]`, and any path a test composed itself,
|
|
12
|
+
* exactly as typed. A fixture built under `tmpdir()` therefore carries two
|
|
13
|
+
* spellings of one directory, and anything comparing them is red on macOS and
|
|
14
|
+
* green on Linux:
|
|
15
|
+
*
|
|
16
|
+
* const d = mkdtempSync(join(tmpdir(), "probe-"));
|
|
17
|
+
* // import.meta.url โ "file:///private/var/folders/โฆ/probe.mjs"
|
|
18
|
+
* // process.argv[1] โ "/var/folders/โฆ/probe.mjs"
|
|
19
|
+
*
|
|
20
|
+
* A consumer hit that three times in one suite: a resolver's return value against
|
|
21
|
+
* a composed expectation, an `isMain` control case, and a git fixture whose
|
|
22
|
+
* repository root git reported realpath'd while the relative path was computed
|
|
23
|
+
* against the other spelling (`zernie/research-paper-pipeline#9`).
|
|
24
|
+
*
|
|
25
|
+
* ๐ด WHY A SHIPPED HELPER AND NOT THREE FIXED CALL SITES. Those call sites were
|
|
26
|
+
* hand-rolled because the product shipped nothing to roll: `mkdtempSync(join(
|
|
27
|
+
* tmpdir(), โฆ))` is the shape a harness author reaches for, and it is the shape
|
|
28
|
+
* that carries the trap. Fixing our own sites leaves every future author to
|
|
29
|
+
* rediscover it. So this is exported from the harness surface (`vigiles`), where
|
|
30
|
+
* `recordCheck` and `skip` already live.
|
|
31
|
+
*
|
|
32
|
+
* ๐ด WHY ITS OWN MODULE. `core/test-utils.ts` also carries `makeSpec` and
|
|
33
|
+
* `initGitRepo`, which pull in spec types and `execSync`; the runtime modules
|
|
34
|
+
* that need a temp root must not drag those in. `test-utils` re-exports these two
|
|
35
|
+
* so existing imports keep working, and there is still exactly one definition.
|
|
36
|
+
*
|
|
37
|
+
* โ ๏ธ On Linux `realpathSync` is the identity here, so no behavioural test on this
|
|
38
|
+
* platform can hold the fix in place. What holds it is the regression test beside
|
|
39
|
+
* this file, which builds its own symlink rather than relying on the platform's.
|
|
40
|
+
*/
|
|
41
|
+
const node_fs_1 = require("node:fs");
|
|
42
|
+
const node_path_1 = require("node:path");
|
|
43
|
+
const node_os_1 = require("node:os");
|
|
44
|
+
/**
|
|
45
|
+
* Create a temporary directory and return the path with every symlink resolved.
|
|
46
|
+
*
|
|
47
|
+
* The `realpathSync` wrapper is the whole point: it must stay OUTSIDE
|
|
48
|
+
* `mkdtempSync`, because the directory has to exist before it can be resolved.
|
|
49
|
+
*
|
|
50
|
+
* @param suffix distinguishes roots in a listing; the name is `vigiles-<suffix>-*`
|
|
51
|
+
*/
|
|
52
|
+
function makeTmpDir(suffix = "test") {
|
|
53
|
+
return (0, node_fs_1.realpathSync)((0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), `vigiles-${suffix}-`)));
|
|
54
|
+
}
|
|
55
|
+
/** Remove a root made by {@link makeTmpDir}. Safe on a path that is already gone. */
|
|
56
|
+
function cleanupTmpDir(dir) {
|
|
57
|
+
(0, node_fs_1.rmSync)(dir, { recursive: true, force: true });
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=tmp-root.js.map
|
package/dist/eval.js
CHANGED
|
@@ -84,6 +84,7 @@ const check_count_js_1 = require("./check-count.js");
|
|
|
84
84
|
const coverage_probe_js_1 = require("./coverage-probe.js");
|
|
85
85
|
const tool_intercept_js_1 = require("./tool-intercept.js");
|
|
86
86
|
const tool_stub_js_1 = require("./tool-stub.js");
|
|
87
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
87
88
|
function writeFiles(cwd, files) {
|
|
88
89
|
for (const [p, content] of Object.entries(files)) {
|
|
89
90
|
const full = (0, node_path_1.resolve)(cwd, p);
|
|
@@ -650,7 +651,7 @@ function whichSkillsFired(trace) {
|
|
|
650
651
|
* falls out of one pass over the prompts (Nร cheaper than re-running per pair).
|
|
651
652
|
*/
|
|
652
653
|
async function runSkillSelectionTrial(args) {
|
|
653
|
-
const cwd = (0,
|
|
654
|
+
const cwd = (0, tmp_root_js_1.makeTmpDir)("selection");
|
|
654
655
|
try {
|
|
655
656
|
if (args.fixture)
|
|
656
657
|
writeFiles(cwd, args.fixture);
|
|
@@ -1008,7 +1009,7 @@ function seedEphemeralHome(throwawayHome, realHome, keep = exports.EPHEMERAL_HOM
|
|
|
1008
1009
|
}
|
|
1009
1010
|
/** Execute one trial in a fresh sandbox; returns its metric row + usage. */
|
|
1010
1011
|
async function executeTrial(spec, arm, trialIndex, runner, cfg) {
|
|
1011
|
-
const cwd = (0,
|
|
1012
|
+
const cwd = (0, tmp_root_js_1.makeTmpDir)("eval");
|
|
1012
1013
|
try {
|
|
1013
1014
|
const resolved = (0, plugin_loader_js_1.resolveHarness)({
|
|
1014
1015
|
plugin: arm.plugin,
|
|
@@ -1456,7 +1457,7 @@ function packageSkillsDir(skillsDir, opts = {}) {
|
|
|
1456
1457
|
const abs = (0, node_path_1.resolve)(skillsDir);
|
|
1457
1458
|
if (!(0, node_fs_1.existsSync)(abs))
|
|
1458
1459
|
throw new Error(`skillsDir not found: ${skillsDir} (resolved ${abs})`);
|
|
1459
|
-
const root = (0,
|
|
1460
|
+
const root = (0, tmp_root_js_1.makeTmpDir)("skills");
|
|
1460
1461
|
(0, node_fs_1.mkdirSync)((0, node_path_1.join)(root, ".claude-plugin"), { recursive: true });
|
|
1461
1462
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(root, ".claude-plugin", "plugin.json"), JSON.stringify({ name: opts.name ?? "vigiles-loose-skills", version: "0.0.0" }, null, 2));
|
|
1462
1463
|
const skillsOut = (0, node_path_1.join)(root, "skills");
|
|
@@ -1649,7 +1650,7 @@ function copySkillsInto(src, skillsOut, stub, present) {
|
|
|
1649
1650
|
* dir. Pure (filesystem only).
|
|
1650
1651
|
*/
|
|
1651
1652
|
function packageInstallSet(opts) {
|
|
1652
|
-
const root = (0,
|
|
1653
|
+
const root = (0, tmp_root_js_1.makeTmpDir)("harness");
|
|
1653
1654
|
try {
|
|
1654
1655
|
(0, node_fs_1.mkdirSync)((0, node_path_1.join)(root, ".claude-plugin"), { recursive: true });
|
|
1655
1656
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(root, ".claude-plugin", "plugin.json"), JSON.stringify({ name: opts.name, version: "0.0.0" }, null, 2));
|
|
@@ -1737,7 +1738,7 @@ function resolveTriggerPluginDir(spec) {
|
|
|
1737
1738
|
/** Run one prompt set ร trials through `runner`, aggregating fired counts. */
|
|
1738
1739
|
/** Run one trigger trial in a throwaway cwd (fixture seeded) โ fired 0/1. */
|
|
1739
1740
|
async function runTriggerTrial(prompt, cfg, runner) {
|
|
1740
|
-
const cwd = (0,
|
|
1741
|
+
const cwd = (0, tmp_root_js_1.makeTmpDir)("trigger");
|
|
1741
1742
|
try {
|
|
1742
1743
|
if (cfg.fixture)
|
|
1743
1744
|
writeFiles(cwd, cfg.fixture);
|
package/dist/harness-test.js
CHANGED
|
@@ -43,7 +43,6 @@ exports.runHarness = runHarness;
|
|
|
43
43
|
*/
|
|
44
44
|
const node_child_process_1 = require("node:child_process");
|
|
45
45
|
const node_fs_1 = require("node:fs");
|
|
46
|
-
const node_os_1 = require("node:os");
|
|
47
46
|
const node_path_1 = require("node:path");
|
|
48
47
|
const adapter_conformance_js_1 = require("./adapter-conformance.js");
|
|
49
48
|
const check_count_js_1 = require("./check-count.js");
|
|
@@ -61,6 +60,7 @@ var sandbox_js_2 = require("./sandbox.js");
|
|
|
61
60
|
Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return sandbox_js_2.decideSandbox; } });
|
|
62
61
|
Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return sandbox_js_2.specTrusted; } });
|
|
63
62
|
Object.defineProperty(exports, "sandboxAvailable", { enumerable: true, get: function () { return sandbox_js_2.sandboxAvailable; } });
|
|
63
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
64
64
|
function contentText(content) {
|
|
65
65
|
if (typeof content === "string")
|
|
66
66
|
return content;
|
|
@@ -443,7 +443,7 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
443
443
|
if (decision.action === "sandbox" && !isClaudeCode) {
|
|
444
444
|
throw new Error(`sandbox not supported for ${driver.runtime.name}: confined execution is Claude Code only. Pass sandbox: false to run ${driver.runtime.name} unconfined (you audited the code, or trust the outer container).`);
|
|
445
445
|
}
|
|
446
|
-
const cwd = (0,
|
|
446
|
+
const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
|
|
447
447
|
const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
|
|
448
448
|
plugin: spec.plugin,
|
|
449
449
|
settings: spec.settings,
|
package/dist/run-script.js
CHANGED
|
@@ -33,12 +33,12 @@ exports.runScript = runScript;
|
|
|
33
33
|
*/
|
|
34
34
|
const node_child_process_1 = require("node:child_process");
|
|
35
35
|
const node_fs_1 = require("node:fs");
|
|
36
|
-
const node_os_1 = require("node:os");
|
|
37
36
|
const node_path_1 = require("node:path");
|
|
38
37
|
const egress_js_1 = require("./egress.js");
|
|
39
38
|
const sandbox_js_1 = require("./sandbox.js");
|
|
40
39
|
const check_count_js_1 = require("./check-count.js");
|
|
41
40
|
const coverage_probe_js_1 = require("./coverage-probe.js");
|
|
41
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
42
42
|
/**
|
|
43
43
|
* True when the exit code is the shell's own report that it never reached the
|
|
44
44
|
* program โ so nothing of the harness ran and no surface may be credited.
|
|
@@ -260,7 +260,7 @@ function snapshotTree(dir) {
|
|
|
260
260
|
return out;
|
|
261
261
|
}
|
|
262
262
|
function sandboxedSpawn(command, stdin, opts) {
|
|
263
|
-
const ioDir = (0,
|
|
263
|
+
const ioDir = (0, tmp_root_js_1.makeTmpDir)("hook-sbx");
|
|
264
264
|
const home = (0, node_path_1.join)(ioDir, "home");
|
|
265
265
|
(0, node_fs_1.mkdirSync)(home);
|
|
266
266
|
// The hook's confined writable work dir: the caller's cwd if given, else a
|
|
@@ -354,7 +354,7 @@ function readEgressResult(resultFile) {
|
|
|
354
354
|
: { status: 1, signal: null, stdout: "", stderr: "", counters: "" };
|
|
355
355
|
}
|
|
356
356
|
function egressSpawn(command, stdin, opts) {
|
|
357
|
-
const ioDir = (0,
|
|
357
|
+
const ioDir = (0, tmp_root_js_1.makeTmpDir)("hook-egr");
|
|
358
358
|
const home = (0, node_path_1.join)(ioDir, "home");
|
|
359
359
|
(0, node_fs_1.mkdirSync)(home);
|
|
360
360
|
const work = opts.cwd ?? (0, node_path_1.join)(ioDir, "work");
|
package/dist/sandbox.js
CHANGED
|
@@ -33,9 +33,9 @@ exports.runSandboxed = runSandboxed;
|
|
|
33
33
|
*/
|
|
34
34
|
const node_child_process_1 = require("node:child_process");
|
|
35
35
|
const node_fs_1 = require("node:fs");
|
|
36
|
-
const node_os_1 = require("node:os");
|
|
37
36
|
const node_path_1 = require("node:path");
|
|
38
37
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
38
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
39
39
|
let cachedAvailable;
|
|
40
40
|
/**
|
|
41
41
|
* Whether this environment can ACTUALLY confine untrusted code under bubblewrap.
|
|
@@ -247,7 +247,7 @@ const WRAPPER = [
|
|
|
247
247
|
bwrap-backed integration test (skipped without bwrap), not the unit gate โ
|
|
248
248
|
the pure policy/args/parse helpers above carry the testable logic. */
|
|
249
249
|
function runSandboxed(opts) {
|
|
250
|
-
const ioDir = (0,
|
|
250
|
+
const ioDir = (0, tmp_root_js_1.makeTmpDir)("sbx");
|
|
251
251
|
const home = (0, node_path_1.join)(ioDir, "home");
|
|
252
252
|
(0, node_fs_1.mkdirSync)(home);
|
|
253
253
|
const scriptF = (0, node_path_1.join)(ioDir, "script.json");
|
package/dist/scan-behavioral.js
CHANGED
|
@@ -34,7 +34,6 @@ exports.formatGateReport = formatGateReport;
|
|
|
34
34
|
const node_fs_1 = require("node:fs");
|
|
35
35
|
const foreign_runner_js_1 = require("./core/foreign-runner.js");
|
|
36
36
|
const eval_load_phase_js_1 = require("./core/eval-load-phase.js");
|
|
37
|
-
const node_os_1 = require("node:os");
|
|
38
37
|
const node_path_1 = require("node:path");
|
|
39
38
|
const node_child_process_1 = require("node:child_process");
|
|
40
39
|
const scan_js_1 = require("./scan.js");
|
|
@@ -46,6 +45,7 @@ const harness_test_js_1 = require("./harness-test.js");
|
|
|
46
45
|
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
47
46
|
const eval_js_2 = require("./adapters/codex/eval.js");
|
|
48
47
|
const driver_js_1 = require("./adapters/codex/driver.js");
|
|
48
|
+
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
49
49
|
function buildProbe(dir, harness) {
|
|
50
50
|
if (harness === "codex") {
|
|
51
51
|
return {
|
|
@@ -529,7 +529,7 @@ function gateRubric(gate) {
|
|
|
529
529
|
}
|
|
530
530
|
/** Run ONE attack against the unstubbed plugin โ the agent's output (or errored). */
|
|
531
531
|
async function runGateAttack(dir, job, deps, model) {
|
|
532
|
-
const cwd = (0,
|
|
532
|
+
const cwd = (0, tmp_root_js_1.makeTmpDir)("gate");
|
|
533
533
|
try {
|
|
534
534
|
const out = await deps.driver.runner({
|
|
535
535
|
task: job.attack,
|
package/dist/test.d.ts
CHANGED
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
* distinction in its config rather than in its imports.
|
|
52
52
|
*/
|
|
53
53
|
export { recordCheck } from "./check-count.js";
|
|
54
|
+
export { makeTmpDir, cleanupTmpDir } from "./core/tmp-root.js";
|
|
54
55
|
export { runScript } from "./run-script.js";
|
|
55
56
|
export type { RunScriptOptions, ScriptRunResult } from "./run-script.js";
|
|
56
57
|
export { runHook, parseHookOutput, decideHook, propertyHook, fileToolEvents, egressRoutes, } from "./run-hook.js";
|
package/dist/test.js
CHANGED
|
@@ -66,8 +66,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
66
66
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
67
67
|
};
|
|
68
68
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
69
|
-
exports.
|
|
70
|
-
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = void 0;
|
|
69
|
+
exports.parseSubagents = exports.parseToolCalls = exports.runHarness = exports.runHarnessTest = exports.experimental_formatPluginGuardReport = exports.experimental_verifyPluginGuards = exports.experimental_alternateSpellings = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.experimental_hookState = exports.loadHook = exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.cleanupTmpDir = exports.makeTmpDir = exports.recordCheck = void 0;
|
|
70
|
+
exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = void 0;
|
|
71
71
|
// --- reporting: how much did this script actually do? ---
|
|
72
72
|
// `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
|
|
73
73
|
// prints the same `โ` as one that ran and passed (measured 2026-08-08 on a file
|
|
@@ -76,6 +76,21 @@ exports.experimental_makeDockerRuntime = exports.experimental_dockerRuntime = ex
|
|
|
76
76
|
// visible to the runner too. See check-count.ts.
|
|
77
77
|
var check_count_js_1 = require("./check-count.js");
|
|
78
78
|
Object.defineProperty(exports, "recordCheck", { enumerable: true, get: function () { return check_count_js_1.recordCheck; } });
|
|
79
|
+
// --- fixture roots: a temp directory whose two spellings agree ---
|
|
80
|
+
// ๐ด DO NOT hand-roll `mkdtempSync(join(tmpdir(), โฆ))` in a harness. On macOS
|
|
81
|
+
// `/var` is a symlink to `/private/var`, so that shape hands you a directory with
|
|
82
|
+
// TWO spellings: Node resolves a module's own URL to the realpath but leaves
|
|
83
|
+
// `process.argv[1]` and any path you composed as typed. Every assertion that
|
|
84
|
+
// compares them is then red on macOS and green on Linux, for a reason that
|
|
85
|
+
// belongs to neither the test nor the code under test โ measured three times in
|
|
86
|
+
// one consumer suite (#241, zernie/research-paper-pipeline#9).
|
|
87
|
+
//
|
|
88
|
+
// `makeTmpDir` resolves the root once, at creation. The trap is unreachable from
|
|
89
|
+
// anything built under it, including a symlink the harness creates ITSELF to test
|
|
90
|
+
// symlink handling โ that stays a genuine test, because it is explicit.
|
|
91
|
+
var tmp_root_js_1 = require("./core/tmp-root.js");
|
|
92
|
+
Object.defineProperty(exports, "makeTmpDir", { enumerable: true, get: function () { return tmp_root_js_1.makeTmpDir; } });
|
|
93
|
+
Object.defineProperty(exports, "cleanupTmpDir", { enumerable: true, get: function () { return tmp_root_js_1.cleanupTmpDir; } });
|
|
79
94
|
// --- the process primitives: runScript (any program) + runHook (plus a decision) ---
|
|
80
95
|
// `runScript` runs any program and reports what it DID (exit, both streams,
|
|
81
96
|
// writes, egress). `runHook` is that plus the hook protocol: event to stdin,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "27.
|
|
3
|
+
"version": "27.3.0",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on โ grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|