vigiles 16.1.0 → 16.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/bash-effects.d.ts +57 -0
- package/dist/core/bash-effects.js +147 -0
- package/dist/core/markdown.d.ts +53 -0
- package/dist/core/markdown.js +99 -0
- package/dist/core/skill-resources.js +58 -57
- package/dist/core/source-refs.d.ts +118 -0
- package/dist/core/source-refs.js +206 -0
- package/dist/coverage-artifact.d.ts +56 -2
- package/dist/coverage-artifact.js +126 -8
- package/dist/coverage-evidence.d.ts +33 -0
- package/dist/coverage-evidence.js +62 -3
- package/dist/coverage-probe.d.ts +14 -6
- package/dist/coverage-probe.js +14 -6
- package/dist/plugin-loader.js +22 -28
- package/dist/scan-core.js +41 -14
- package/dist/scan-files.js +13 -19
- package/dist/test-coverage-files.js +10 -4
- package/dist/test-coverage.js +1 -6
- package/package.json +1 -1
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* vigiles — recovering a FILE REFERENCE from PLAIN SOURCE TEXT.
|
|
4
|
+
*
|
|
5
|
+
* The third of three reference grammars, and the only one with no parser behind
|
|
6
|
+
* it. A markdown body is read by markdown-it (`core/markdown.ts` →
|
|
7
|
+
* `markdownRefs`), a hook `command` is read by mvdan-sh (`core/bash-effects.ts`
|
|
8
|
+
* → `commandWords`), and what is left — the body of a hook script, a helper
|
|
9
|
+
* `.js` / `.py` / `.rb` — is scanned for path-shaped character runs, because a
|
|
10
|
+
* general-purpose "find every path this program touches" analysis is not a
|
|
11
|
+
* thing this tool can be.
|
|
12
|
+
*
|
|
13
|
+
* Since that scan IS a character run, the only thing standing between it and a
|
|
14
|
+
* false accusation is where the run may START and STOP. This module owns both
|
|
15
|
+
* boundaries and the extension vocabularies, so no caller can build a pattern
|
|
16
|
+
* without them.
|
|
17
|
+
*
|
|
18
|
+
* ## The two boundaries, and the two live defects that came from omitting them
|
|
19
|
+
*
|
|
20
|
+
* RIGHT — the extension must END the token. `INTRA_REF_EXTS` used to be a bare
|
|
21
|
+
* alternation with no trailing assertion, and `js` sits ahead of `json` in it,
|
|
22
|
+
* so `hooks/hooks.json` matched as `hooks/hooks.js`. Measured 2026-08-17 on
|
|
23
|
+
* `microsoft/power-platform-skills`: a `//` comment naming `hooks/hooks.json`,
|
|
24
|
+
* with that exact file sitting beside it on disk, was reported as
|
|
25
|
+
* "hooks/hooks.js (referenced but MISSING)" — a maintainer told a file they
|
|
26
|
+
* have is missing, under a name they never wrote.
|
|
27
|
+
*
|
|
28
|
+
* LEFT — the surface dir must START the token. `(?:agents|hooks|skills)/` with
|
|
29
|
+
* nothing before it also matches the tail of `claude-agents/`. Measured the
|
|
30
|
+
* same day on `fcakyon/claude-codex-settings`: a
|
|
31
|
+
* `new URL("../../../claude-agents/fable-advisor.md", …)` — a correct,
|
|
32
|
+
* resolving reference — was reported as a broken `agents/fable-advisor.md`.
|
|
33
|
+
* That boundary is a predicate ({@link startsAtSeparator}) rather than a regex
|
|
34
|
+
* lookbehind, because the browser twin (`scan-files.ts`) compiles this into the
|
|
35
|
+
* demo engine, and because the caller already inspects `m.index` for its
|
|
36
|
+
* plugin-rooted test.
|
|
37
|
+
*
|
|
38
|
+
* ## Comments are prose in every language, not only in shell
|
|
39
|
+
*
|
|
40
|
+
* `stripShellComments` had the right idea and too narrow a domain: a full-line
|
|
41
|
+
* `#` in a `.sh` file is prose, and so is a full-line `//` or a JSDoc `*` in a
|
|
42
|
+
* `.js` file. Both remaining corpus false accusations in this detector came out
|
|
43
|
+
* of JSDoc — `* It is deliberately not registered in hooks/hooks.json:` and
|
|
44
|
+
* "`git log --grep=.env -- hooks/x.mjs` is refused …". Neither is a file
|
|
45
|
+
* operation; both were reported as broken references.
|
|
46
|
+
*
|
|
47
|
+
* FULL-LINE ONLY, the same rule the shell version already held. A trailing
|
|
48
|
+
* comment on a real code line is left alone, so a genuine reference sharing a
|
|
49
|
+
* line with code is never dropped. The cost is stated rather than hidden: a
|
|
50
|
+
* path mentioned ONLY in a trailing comment still counts as a reference.
|
|
51
|
+
*/
|
|
52
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
53
|
+
exports.SCRIPT_REF_EXTENSIONS = exports.INTRA_REF_EXTENSIONS = void 0;
|
|
54
|
+
exports.startsAtSeparator = startsAtSeparator;
|
|
55
|
+
exports.intraRefPattern = intraRefPattern;
|
|
56
|
+
exports.scriptWordPattern = scriptWordPattern;
|
|
57
|
+
exports.scriptRefPattern = scriptRefPattern;
|
|
58
|
+
exports.stripFullLineComments = stripFullLineComments;
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
// Boundaries — the two assertions no pattern here may be built without
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
/**
|
|
63
|
+
* The right boundary: the matched extension must be the END of the token.
|
|
64
|
+
* A negative lookahead over the characters an extension is drawn from, so `js`
|
|
65
|
+
* cannot claim the head of `json` / `jsonl` and `py` cannot claim `pyc`. `.` is
|
|
66
|
+
* deliberately NOT in the class, which preserves the prior `\b` behaviour on
|
|
67
|
+
* `bundle.js.map`.
|
|
68
|
+
*/
|
|
69
|
+
const ENDS_TOKEN = "(?![A-Za-z0-9_])";
|
|
70
|
+
/**
|
|
71
|
+
* Characters a path SEGMENT is made of. A match preceded by one of these landed
|
|
72
|
+
* in the middle of a longer name (`claude-agents/`), which is not a reference
|
|
73
|
+
* to the surface dir it happens to end with.
|
|
74
|
+
*/
|
|
75
|
+
const SEGMENT_CHAR = /[A-Za-z0-9._-]/;
|
|
76
|
+
/**
|
|
77
|
+
* Whether a match at `idx` begins at a path boundary rather than inside a
|
|
78
|
+
* longer name. Position 0 counts as a boundary.
|
|
79
|
+
*
|
|
80
|
+
* `/` is a boundary on purpose: `${CLAUDE_PLUGIN_ROOT}/hooks/x.sh` is the
|
|
81
|
+
* standard spelling, and the caller's plugin-rooted test decides whether the
|
|
82
|
+
* segment before that slash roots the path inside the plugin or outside it.
|
|
83
|
+
*/
|
|
84
|
+
function startsAtSeparator(content, idx) {
|
|
85
|
+
if (idx <= 0)
|
|
86
|
+
return true;
|
|
87
|
+
return !SEGMENT_CHAR.test(content[idx - 1] ?? "");
|
|
88
|
+
}
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Extension vocabularies
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
/**
|
|
93
|
+
* Extensions an intra-plugin reference may carry — the file kinds a plugin's
|
|
94
|
+
* own hook / helper source legitimately points at.
|
|
95
|
+
*/
|
|
96
|
+
exports.INTRA_REF_EXTENSIONS = [
|
|
97
|
+
"md",
|
|
98
|
+
"sh",
|
|
99
|
+
"cmd",
|
|
100
|
+
"mjs",
|
|
101
|
+
"cjs",
|
|
102
|
+
"js",
|
|
103
|
+
"ts",
|
|
104
|
+
"py",
|
|
105
|
+
"rb",
|
|
106
|
+
"txt",
|
|
107
|
+
"json",
|
|
108
|
+
];
|
|
109
|
+
/**
|
|
110
|
+
* Extensions a RUNNABLE script carries. Shared by the hook scanner and the
|
|
111
|
+
* coverage twins, which each declared their own copy before.
|
|
112
|
+
*/
|
|
113
|
+
exports.SCRIPT_REF_EXTENSIONS = [
|
|
114
|
+
"sh",
|
|
115
|
+
"mjs",
|
|
116
|
+
"cjs",
|
|
117
|
+
"js",
|
|
118
|
+
"ts",
|
|
119
|
+
"py",
|
|
120
|
+
"rb",
|
|
121
|
+
];
|
|
122
|
+
/**
|
|
123
|
+
* A plugin-relative path under one of `dirs`, carrying a known extension — the
|
|
124
|
+
* pattern for scanning a plugin's own SOURCE TEXT.
|
|
125
|
+
*
|
|
126
|
+
* The left boundary is not baked into the returned regex (see
|
|
127
|
+
* {@link startsAtSeparator}): a caller scanning raw text must apply that
|
|
128
|
+
* predicate at `m.index`, which both the disk detector and its browser twin do
|
|
129
|
+
* inside their plugin-rooted test.
|
|
130
|
+
*/
|
|
131
|
+
function intraRefPattern(dirs) {
|
|
132
|
+
const exts = exports.INTRA_REF_EXTENSIONS.join("|");
|
|
133
|
+
return new RegExp(`(?:${dirs.join("|")})/[A-Za-z0-9._/-]+\\.(?:${exts})${ENDS_TOKEN}`, "g");
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* A script path occupying a WHOLE shell WORD.
|
|
137
|
+
*
|
|
138
|
+
* ⚠️ ANCHORING IS NOT WHAT FIXES THE `node -e` DEFECT, and saying so would be
|
|
139
|
+
* wrong twice over — the payload
|
|
140
|
+
* `import(require(node:url).pathToFileURL(require(node:path).join(root,hooks,always-on.mjs`
|
|
141
|
+
* contains no whitespace and ends in `.mjs`, so it satisfies this pattern
|
|
142
|
+
* perfectly (asserted in source-refs.test.ts, so the claim cannot drift back).
|
|
143
|
+
* That defect is fixed one level up, by `commandWords` refusing to hand the
|
|
144
|
+
* argument of `-e` to anyone.
|
|
145
|
+
*
|
|
146
|
+
* What anchoring buys is narrower and worth stating exactly: the reported name
|
|
147
|
+
* is always a word a shell could hand to `execve`, never a fragment cut out of
|
|
148
|
+
* a longer one. `echo "see hooks/x.sh"` is one word containing a path; before,
|
|
149
|
+
* `hooks/x.sh` was lifted out of it and checked as though the hook ran it.
|
|
150
|
+
*
|
|
151
|
+
* The cost, stated rather than discovered later: a path bundled into a flag
|
|
152
|
+
* (`--require=hooks/x.js`) is no longer seen. Measured across the 32-repo
|
|
153
|
+
* dogfood corpus, that costs zero findings.
|
|
154
|
+
*/
|
|
155
|
+
function scriptWordPattern() {
|
|
156
|
+
const exts = exports.SCRIPT_REF_EXTENSIONS.join("|");
|
|
157
|
+
return new RegExp(`^[^\\s*?]+\\.(?:${exts})$`);
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* A script path appearing anywhere inside a string — for the one caller with no
|
|
161
|
+
* shell parse to hand (the coverage twins scan a serialized settings blob).
|
|
162
|
+
* Carries the right boundary; it cannot carry the left one, and its callers
|
|
163
|
+
* gate every hit on the file existing, so a stray match is dropped rather than
|
|
164
|
+
* reported.
|
|
165
|
+
*/
|
|
166
|
+
function scriptRefPattern() {
|
|
167
|
+
const exts = exports.SCRIPT_REF_EXTENSIONS.join("|");
|
|
168
|
+
return new RegExp("[\\w./$" + "{}@-]+\\." + `(?:${exts})${ENDS_TOKEN}`, "g");
|
|
169
|
+
}
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
// Comments (prose inside an executable source)
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
/** Source kinds whose full-line comments this module knows how to drop. */
|
|
174
|
+
const HASH_COMMENT = /\.(?:sh|bash|zsh|cmd|py|rb|pl)$/i;
|
|
175
|
+
const SLASH_COMMENT = /\.(?:m|c)?[jt]s$/i;
|
|
176
|
+
/**
|
|
177
|
+
* A line that is ENTIRELY a comment in a `//`-style language: a `//` line, a
|
|
178
|
+
* block-comment delimiter, or a JSDoc continuation `*` followed by whitespace
|
|
179
|
+
* or end of line.
|
|
180
|
+
*
|
|
181
|
+
* A bare `*name` does NOT count — `*run() {}` is a generator method, and
|
|
182
|
+
* dropping that line would silently delete a real reference from code.
|
|
183
|
+
*/
|
|
184
|
+
const SLASH_COMMENT_LINE = /^(?:\/\/|\/\*|\*\/|\*(?:\s|$))/;
|
|
185
|
+
/**
|
|
186
|
+
* Drop FULL-LINE comments (including a shebang, which also starts with `#`)
|
|
187
|
+
* from an executable source before it is scanned for path references.
|
|
188
|
+
*
|
|
189
|
+
* A file of unknown kind is returned unchanged: guessing a comment syntax is
|
|
190
|
+
* how a real reference gets deleted, and this detector's contract is that it
|
|
191
|
+
* under-reports rather than accuses.
|
|
192
|
+
*/
|
|
193
|
+
function stripFullLineComments(path, content) {
|
|
194
|
+
const isHash = HASH_COMMENT.test(path);
|
|
195
|
+
const isSlash = SLASH_COMMENT.test(path);
|
|
196
|
+
if (!isHash && !isSlash)
|
|
197
|
+
return content;
|
|
198
|
+
return content
|
|
199
|
+
.split("\n")
|
|
200
|
+
.filter((line) => {
|
|
201
|
+
const t = line.trimStart();
|
|
202
|
+
return isHash ? !t.startsWith("#") : !SLASH_COMMENT_LINE.test(t);
|
|
203
|
+
})
|
|
204
|
+
.join("\n");
|
|
205
|
+
}
|
|
206
|
+
//# sourceMappingURL=source-refs.js.map
|
|
@@ -12,6 +12,26 @@ export declare const COVERAGE_ARTIFACT_FILE = "coverage.json";
|
|
|
12
12
|
* silence "nothing ever measured whether this fires".
|
|
13
13
|
*/
|
|
14
14
|
export type CoverageTierName = "harness" | "eval";
|
|
15
|
+
/**
|
|
16
|
+
* How a RECORD came to name its surface.
|
|
17
|
+
*
|
|
18
|
+
* The three {@link ProbeOrigin} values are what the run REPORTED about itself
|
|
19
|
+
* from inside its own process — a command line it executed, a transcript it got
|
|
20
|
+
* back. `colocated` is the fourth and is different in kind: nothing inside the
|
|
21
|
+
* test said it. The RUNNER observed that the script it just executed is the
|
|
22
|
+
* surface's colocated test, by the same {@link isColocatedTest} rule that is
|
|
23
|
+
* already willing to credit that file for merely existing.
|
|
24
|
+
*
|
|
25
|
+
* 🔴 THE SPLIT IS THE POINT, AND IT IS WHY THIS IS NOT ON {@link ProbeOrigin}.
|
|
26
|
+
* `ProbeOrigin` is the wire vocabulary: `parseCheckReport` reads it out of a
|
|
27
|
+
* scratch file the CHILD writes. Putting `colocated` there would let a harness
|
|
28
|
+
* hand-write `{"how":"colocated","ref":"…"}` and mint execution coverage for a
|
|
29
|
+
* surface by declaring it — which is `vigiles:covers` returning through the back
|
|
30
|
+
* door, the tier this file's header records being deleted for exactly that. Kept
|
|
31
|
+
* off `ProbeOrigin`, a script cannot spell it at all: `toProbe` has no branch
|
|
32
|
+
* that produces it, so the only producer in the program is {@link recordsFrom}.
|
|
33
|
+
*/
|
|
34
|
+
export type RunAttribution = ProbeOrigin | "colocated";
|
|
15
35
|
/** One surface, exercised by one script, at one moment, against one version. */
|
|
16
36
|
export interface CoverageRun {
|
|
17
37
|
readonly kind: SurfaceKind;
|
|
@@ -19,8 +39,8 @@ export interface CoverageRun {
|
|
|
19
39
|
readonly path: string;
|
|
20
40
|
readonly name: string;
|
|
21
41
|
readonly tier: CoverageTierName;
|
|
22
|
-
/** How the run named it — see {@link
|
|
23
|
-
readonly how:
|
|
42
|
+
/** How the run named it — see {@link RunAttribution}. */
|
|
43
|
+
readonly how: RunAttribution;
|
|
24
44
|
/** The script file that did the exercising. */
|
|
25
45
|
readonly by: string;
|
|
26
46
|
/** ISO-8601 timestamp of the run. */
|
|
@@ -192,6 +212,18 @@ export interface ScriptRunRecord {
|
|
|
192
212
|
/** The script file that ran. */
|
|
193
213
|
readonly file: string;
|
|
194
214
|
readonly probes: readonly SurfaceProbe[];
|
|
215
|
+
/**
|
|
216
|
+
* How many checks the run REPORTED, or `undefined` when it reported nothing.
|
|
217
|
+
*
|
|
218
|
+
* The distinction is load-bearing and is not this file's invention — see
|
|
219
|
+
* `statusFor` in run-scripts.ts. `undefined` means the script never imported
|
|
220
|
+
* vigiles and therefore *could not* report; that silence is the legacy branch
|
|
221
|
+
* and is never a verdict. A positive number is the script saying, through the
|
|
222
|
+
* library, that it made an observation. Only the second is evidence, which is
|
|
223
|
+
* why {@link recordsFrom} attributes colocation on `> 0` rather than on
|
|
224
|
+
* "it exited 0".
|
|
225
|
+
*/
|
|
226
|
+
readonly checks?: number;
|
|
195
227
|
}
|
|
196
228
|
/**
|
|
197
229
|
* The runs worth recording, out of a whole `vigiles test` / `vigiles eval`.
|
|
@@ -226,10 +258,26 @@ export interface ScriptRunRecord {
|
|
|
226
258
|
* CLI would destroy a real result. A skip must not write, because it did not
|
|
227
259
|
* finish. Both follow from "a skip is not an execution"; only the direction of
|
|
228
260
|
* the conclusion differs.
|
|
261
|
+
*
|
|
262
|
+
* ## 🔴 A RUN WITH NO PROBES IS NO LONGER DISCARDED
|
|
263
|
+
*
|
|
264
|
+
* The second clause used to be `surfaces.length > 0`: a passing run that named no
|
|
265
|
+
* surface contributed nothing. That threw away the ordinary case. MEASURED on a
|
|
266
|
+
* 52-surface consumer repo, 23 of its 27 covered surfaces were carried by a
|
|
267
|
+
* colocated `*.harness.mjs` that `vigiles test` RUNS on every push — and every one
|
|
268
|
+
* of them reported `colocated` evidence, *"this says the file EXISTS, not that it
|
|
269
|
+
* ran"*, because a harness asserting through `node:assert` reports a CHECK COUNT
|
|
270
|
+
* and no probe. The run happened, the runner watched it happen, and the metric
|
|
271
|
+
* described it as a directory listing.
|
|
272
|
+
*
|
|
273
|
+
* So a passing run that reported at least one CHECK is kept even with no probes:
|
|
274
|
+
* {@link recordsFrom} can attribute it by where the script sits. `checks` travels
|
|
275
|
+
* with it because "exited 0" is not the bar — see {@link ScriptRunRecord.checks}.
|
|
229
276
|
*/
|
|
230
277
|
export declare function runsFromResults(results: readonly {
|
|
231
278
|
readonly file: string;
|
|
232
279
|
readonly status: string;
|
|
280
|
+
readonly checks?: number;
|
|
233
281
|
readonly surfaces?: readonly SurfaceProbe[];
|
|
234
282
|
}[]): ScriptRunRecord[];
|
|
235
283
|
/**
|
|
@@ -238,6 +286,12 @@ export declare function runsFromResults(results: readonly {
|
|
|
238
286
|
* run); a surface it cannot read is skipped — an unhashable record could never
|
|
239
287
|
* be checked for staleness and would therefore be permanent, unfalsifiable
|
|
240
288
|
* coverage.
|
|
289
|
+
*
|
|
290
|
+
* Two sources of attribution, in one pass: the probes a run reported, and — for a
|
|
291
|
+
* run that reported a check — the surface its own script is colocated with
|
|
292
|
+
* ({@link colocatedSurface}). The dedupe key already carries `how`, so a run that
|
|
293
|
+
* BOTH probed a surface and sits beside it records once per attribution and the
|
|
294
|
+
* report can still say which kinds of evidence exist.
|
|
241
295
|
*/
|
|
242
296
|
export declare function recordsFrom(opts: ResolveOptions & {
|
|
243
297
|
readonly runs: readonly ScriptRunRecord[];
|
|
@@ -57,10 +57,26 @@ exports.indexRuns = indexRuns;
|
|
|
57
57
|
const node_crypto_1 = require("node:crypto");
|
|
58
58
|
const node_fs_1 = require("node:fs");
|
|
59
59
|
const node_path_1 = require("node:path");
|
|
60
|
+
const coverage_evidence_js_1 = require("./coverage-evidence.js");
|
|
60
61
|
/** Bumped when the record shape changes in a non-additive way. */
|
|
61
62
|
exports.COVERAGE_ARTIFACT_VERSION = 1;
|
|
62
63
|
/** The artifact filename under `.vigiles/`. */
|
|
63
64
|
exports.COVERAGE_ARTIFACT_FILE = "coverage.json";
|
|
65
|
+
/**
|
|
66
|
+
* Every {@link RunAttribution}, as data — the artifact's validator reads it, so
|
|
67
|
+
* the accepted set and the type cannot drift apart into a record shape the
|
|
68
|
+
* program can produce and the reader silently discards. The `satisfies` is the
|
|
69
|
+
* lock: drop a member and this stops compiling.
|
|
70
|
+
*/
|
|
71
|
+
const ATTRIBUTIONS = new Set([
|
|
72
|
+
"command",
|
|
73
|
+
"fired",
|
|
74
|
+
"dispatched",
|
|
75
|
+
"colocated",
|
|
76
|
+
]);
|
|
77
|
+
function isAttribution(value) {
|
|
78
|
+
return typeof value === "string" && ATTRIBUTIONS.has(value);
|
|
79
|
+
}
|
|
64
80
|
/** Content hash of a surface file — the staleness key. */
|
|
65
81
|
function surfaceSha(content) {
|
|
66
82
|
return (0, node_crypto_1.createHash)("sha256").update(content).digest("hex").slice(0, 16);
|
|
@@ -347,11 +363,83 @@ function only(matches) {
|
|
|
347
363
|
* CLI would destroy a real result. A skip must not write, because it did not
|
|
348
364
|
* finish. Both follow from "a skip is not an execution"; only the direction of
|
|
349
365
|
* the conclusion differs.
|
|
366
|
+
*
|
|
367
|
+
* ## 🔴 A RUN WITH NO PROBES IS NO LONGER DISCARDED
|
|
368
|
+
*
|
|
369
|
+
* The second clause used to be `surfaces.length > 0`: a passing run that named no
|
|
370
|
+
* surface contributed nothing. That threw away the ordinary case. MEASURED on a
|
|
371
|
+
* 52-surface consumer repo, 23 of its 27 covered surfaces were carried by a
|
|
372
|
+
* colocated `*.harness.mjs` that `vigiles test` RUNS on every push — and every one
|
|
373
|
+
* of them reported `colocated` evidence, *"this says the file EXISTS, not that it
|
|
374
|
+
* ran"*, because a harness asserting through `node:assert` reports a CHECK COUNT
|
|
375
|
+
* and no probe. The run happened, the runner watched it happen, and the metric
|
|
376
|
+
* described it as a directory listing.
|
|
377
|
+
*
|
|
378
|
+
* So a passing run that reported at least one CHECK is kept even with no probes:
|
|
379
|
+
* {@link recordsFrom} can attribute it by where the script sits. `checks` travels
|
|
380
|
+
* with it because "exited 0" is not the bar — see {@link ScriptRunRecord.checks}.
|
|
350
381
|
*/
|
|
351
382
|
function runsFromResults(results) {
|
|
352
383
|
return results
|
|
353
|
-
.filter((r) => r.status === "pass" &&
|
|
354
|
-
|
|
384
|
+
.filter((r) => r.status === "pass" &&
|
|
385
|
+
((r.surfaces?.length ?? 0) > 0 || (r.checks ?? 0) > 0))
|
|
386
|
+
.map((r) => ({
|
|
387
|
+
file: r.file,
|
|
388
|
+
probes: r.surfaces ?? [],
|
|
389
|
+
...(r.checks === undefined ? {} : { checks: r.checks }),
|
|
390
|
+
}));
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* The surface a RUN's own script is the colocated test of, or `null`.
|
|
394
|
+
*
|
|
395
|
+
* This is the fourth attribution ({@link RunAttribution}), and the only one the
|
|
396
|
+
* script does not report about itself. The runner knows two facts the script
|
|
397
|
+
* cannot: which file it just executed, and which surfaces this repo has. Put
|
|
398
|
+
* together by {@link isColocatedTest} — the SAME predicate that already grants
|
|
399
|
+
* `colocated` evidence for that file merely existing — they say which surface the
|
|
400
|
+
* run was about.
|
|
401
|
+
*
|
|
402
|
+
* ## The bar is a REPORTED CHECK, not exit zero
|
|
403
|
+
*
|
|
404
|
+
* 🔴 WITHOUT THAT, THIS WOULD BE THE ORIGINAL DEFECT WEARING THE STRONGER LABEL.
|
|
405
|
+
* Measured 2026-08-17 on a two-skill fixture: `touch
|
|
406
|
+
* .claude/skills/argument-arc/argument-arc.harness.mjs` — a ZERO-BYTE file —
|
|
407
|
+
* drops the untested count by one, and `vigiles test` then RUNS it and prints
|
|
408
|
+
* `✓ … passed`. An empty script exits 0. Attributing on "it ran and exited 0"
|
|
409
|
+
* would promote that file from `colocated` ("the file EXISTS, not that it ran")
|
|
410
|
+
* to "MEASURED BY A RUN", which is strictly worse than the hole being closed:
|
|
411
|
+
* the same emptiness, now wearing execution's name.
|
|
412
|
+
*
|
|
413
|
+
* A zero-byte file cannot report a check — it never imports vigiles — so its
|
|
414
|
+
* `checks` is `undefined` and it earns nothing here, falling back to colocation
|
|
415
|
+
* exactly as before. This is `statusFor`'s distinction reused, not a new one:
|
|
416
|
+
* silence is the legacy branch, `0` is `vacuous` (never a `pass`, so it never
|
|
417
|
+
* reaches this function), and a positive count is the script saying through the
|
|
418
|
+
* library that it observed something.
|
|
419
|
+
*
|
|
420
|
+
* ⚠️ WHAT THIS STILL CANNOT SEE, stated rather than implied: a harness that
|
|
421
|
+
* asserts entirely on its own and never imports vigiles reports nothing, so it is
|
|
422
|
+
* indistinguishable here from the empty file. It keeps its colocated credit and
|
|
423
|
+
* gains no execution credit. That is deliberate and it is `check-count.ts`'s
|
|
424
|
+
* standing decision — force-loading the counter into every child was considered
|
|
425
|
+
* and rejected there, because it would report `0` for a blameless hand-rolled
|
|
426
|
+
* harness. The cost is a missed upgrade; the alternative was a false one.
|
|
427
|
+
*
|
|
428
|
+
* 🔴 AMBIGUITY RESOLVES TO NOTHING, the same rule {@link only} applies to probes.
|
|
429
|
+
* Two surfaces in one directory can both claim a script when one name prefixes
|
|
430
|
+
* the other (`foo` and `foo.bar` both match `foo.bar.harness.mjs`, since
|
|
431
|
+
* colocation asks only that the basename START with `<name>.`). Exactly one of
|
|
432
|
+
* them is what the test is about, and nothing here can say which — so crediting
|
|
433
|
+
* both would INVENT a record, which is the failure the rest of this file spends
|
|
434
|
+
* its length refusing.
|
|
435
|
+
*/
|
|
436
|
+
function colocatedSurface(run, surfaces, alreadyProbed) {
|
|
437
|
+
if ((run.checks ?? 0) <= 0)
|
|
438
|
+
return null;
|
|
439
|
+
const matches = surfaces.filter((s) => s.path !== run.file &&
|
|
440
|
+
!alreadyProbed.has(s.path) &&
|
|
441
|
+
(0, coverage_evidence_js_1.isColocatedTest)(s, run.file));
|
|
442
|
+
return matches.length === 1 ? (matches[0] ?? null) : null;
|
|
355
443
|
}
|
|
356
444
|
/**
|
|
357
445
|
* Turn resolved probes into records. `readSurface` supplies the current content
|
|
@@ -359,15 +447,45 @@ function runsFromResults(results) {
|
|
|
359
447
|
* run); a surface it cannot read is skipped — an unhashable record could never
|
|
360
448
|
* be checked for staleness and would therefore be permanent, unfalsifiable
|
|
361
449
|
* coverage.
|
|
450
|
+
*
|
|
451
|
+
* Two sources of attribution, in one pass: the probes a run reported, and — for a
|
|
452
|
+
* run that reported a check — the surface its own script is colocated with
|
|
453
|
+
* ({@link colocatedSurface}). The dedupe key already carries `how`, so a run that
|
|
454
|
+
* BOTH probed a surface and sits beside it records once per attribution and the
|
|
455
|
+
* report can still say which kinds of evidence exist.
|
|
362
456
|
*/
|
|
363
457
|
function recordsFrom(opts) {
|
|
364
458
|
const out = [];
|
|
365
459
|
const seen = new Set();
|
|
366
|
-
const pairs =
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
460
|
+
const pairs = [];
|
|
461
|
+
for (const run of opts.runs) {
|
|
462
|
+
const probed = new Set();
|
|
463
|
+
for (const probe of run.probes)
|
|
464
|
+
for (const surface of resolveProbe(probe, opts.surfaces, opts)) {
|
|
465
|
+
pairs.push({ run, probe, surface });
|
|
466
|
+
probed.add(surface.path);
|
|
467
|
+
}
|
|
468
|
+
// 🔴 COLOCATION IS THE FALLBACK ATTRIBUTION, NOT AN ADDITIONAL ONE, and
|
|
469
|
+
// skipping what this run already named is what keeps it from DOWNGRADING the
|
|
470
|
+
// record. MEASURED 2026-08-17 on a 52-surface repo the first time this tier
|
|
471
|
+
// ran: `.claude/hooks/paper-lint.mjs` went from `command` — the harness
|
|
472
|
+
// literally executed that path — to `colocated`, and three `fired` skill
|
|
473
|
+
// activations went the same way. The cause is one key: `mergeRuns` dedupes on
|
|
474
|
+
// (surface, tier, script) and does NOT include `how`, so a second attribution
|
|
475
|
+
// of the same pair does not sit beside the first, it REPLACES it — and with
|
|
476
|
+
// both records stamped the same `at`, `held.at <= run.at` hands the win to
|
|
477
|
+
// whichever was pushed last.
|
|
478
|
+
//
|
|
479
|
+
// The ordering is the same one this file already applies one level up: a
|
|
480
|
+
// direct observation of the surface being exercised outranks an inference
|
|
481
|
+
// from where a file sits. Emitting both and teaching the merge to keep the
|
|
482
|
+
// stronger was the alternative; it costs a wider key, a second record per
|
|
483
|
+
// pair, and a ranking function — to store a fact that adds nothing, since
|
|
484
|
+
// both resolve to the same `executed` evidence.
|
|
485
|
+
const beside = colocatedSurface(run, opts.surfaces, probed);
|
|
486
|
+
if (beside)
|
|
487
|
+
pairs.push({ run, probe: { how: "colocated" }, surface: beside });
|
|
488
|
+
}
|
|
371
489
|
for (const { run, probe, surface } of pairs) {
|
|
372
490
|
const key = `${surface.path}\u0000${run.file}\u0000${probe.how}`;
|
|
373
491
|
if (seen.has(key))
|
|
@@ -492,7 +610,7 @@ function isCoverageRun(value) {
|
|
|
492
610
|
typeof r.path === "string" &&
|
|
493
611
|
typeof r.name === "string" &&
|
|
494
612
|
(r.tier === "harness" || r.tier === "eval") &&
|
|
495
|
-
(r.how
|
|
613
|
+
isAttribution(r.how) &&
|
|
496
614
|
typeof r.by === "string" &&
|
|
497
615
|
typeof r.at === "string" &&
|
|
498
616
|
typeof r.sha === "string");
|
|
@@ -27,6 +27,39 @@ export interface CoverableSurface {
|
|
|
27
27
|
export interface PreparedTest {
|
|
28
28
|
readonly path: string;
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* Is `testPath` the colocated test OF this surface — NAMED after it, SITTING
|
|
32
|
+
* BESIDE it?
|
|
33
|
+
*
|
|
34
|
+
* 🔴 THIS USED TO BE WRITTEN TWICE, and the second copy said so in its own
|
|
35
|
+
* comment: `test-coverage-files.ts` carried a function whose docstring was
|
|
36
|
+
* *"Mirror of test-coverage.ts `isColocated`"*. A mirror is a promise that two
|
|
37
|
+
* bodies stay equal, kept by whoever remembers — and this file already documents
|
|
38
|
+
* three separate occasions where a change landed in one report builder and not
|
|
39
|
+
* the other ({@link declaredSurfaceName}, {@link hookScriptRefs}). The rule now
|
|
40
|
+
* has ONE body and three callers: the disk detector, the browser twin, and the
|
|
41
|
+
* runner that turns an executed script into an execution record
|
|
42
|
+
* (`coverage-artifact.ts`). There is nothing left to mirror wrong.
|
|
43
|
+
*
|
|
44
|
+
* That third caller is why this became shared rather than merely deduplicated.
|
|
45
|
+
* Colocation and execution were answering the same question — *which surface is
|
|
46
|
+
* this test about?* — with two different pieces of code, so a script vigiles had
|
|
47
|
+
* just RUN could not be attributed by the very rule that was already willing to
|
|
48
|
+
* credit it for existing.
|
|
49
|
+
*
|
|
50
|
+
* SEPARATORS ARE NORMALISED FIRST, then POSIX semantics apply. The disk detector
|
|
51
|
+
* feeds paths from `globSync`, which yields `\` on Windows; the browser twin
|
|
52
|
+
* feeds file-map keys, which are always `/`. Folding `\` to `/` makes one body
|
|
53
|
+
* correct for both instead of asking each caller to bring its own path module —
|
|
54
|
+
* which is exactly the seam the mirror lived in.
|
|
55
|
+
*
|
|
56
|
+
* ⚠️ THE COST, STATED: a POSIX file whose NAME literally contains a backslash is
|
|
57
|
+
* now read as a directory boundary. That is not reachable for either input here —
|
|
58
|
+
* a skill directory or a test filename with a `\` in it does not survive the glob
|
|
59
|
+
* patterns that discover it in the first place — and it is the same trade
|
|
60
|
+
* `hookScriptRefs` already makes one function up.
|
|
61
|
+
*/
|
|
62
|
+
export declare function isColocatedTest(surface: Pick<CoverableSurface, "name" | "path">, testPath: string): boolean;
|
|
30
63
|
/** Wrap a discovered path. Kept as a function so both twins share one shape. */
|
|
31
64
|
export declare function prepareTest(path: string): PreparedTest;
|
|
32
65
|
/** @param filename a BASENAME — callers strip the directory with their own path
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isColocatedTest = isColocatedTest;
|
|
3
4
|
exports.prepareTest = prepareTest;
|
|
4
5
|
exports.isEvalScript = isEvalScript;
|
|
5
6
|
exports.hookScriptRefs = hookScriptRefs;
|
|
@@ -91,13 +92,71 @@ exports.declaredSurfaceName = declaredSurfaceName;
|
|
|
91
92
|
* impossible there and its count is 0 — see `test-coverage-files.ts`.
|
|
92
93
|
*/
|
|
93
94
|
const frontmatter_read_js_1 = require("./core/frontmatter-read.js");
|
|
95
|
+
const posix_path_js_1 = require("./posix-path.js");
|
|
96
|
+
const source_refs_js_1 = require("./core/source-refs.js");
|
|
97
|
+
/**
|
|
98
|
+
* Is `testPath` the colocated test OF this surface — NAMED after it, SITTING
|
|
99
|
+
* BESIDE it?
|
|
100
|
+
*
|
|
101
|
+
* 🔴 THIS USED TO BE WRITTEN TWICE, and the second copy said so in its own
|
|
102
|
+
* comment: `test-coverage-files.ts` carried a function whose docstring was
|
|
103
|
+
* *"Mirror of test-coverage.ts `isColocated`"*. A mirror is a promise that two
|
|
104
|
+
* bodies stay equal, kept by whoever remembers — and this file already documents
|
|
105
|
+
* three separate occasions where a change landed in one report builder and not
|
|
106
|
+
* the other ({@link declaredSurfaceName}, {@link hookScriptRefs}). The rule now
|
|
107
|
+
* has ONE body and three callers: the disk detector, the browser twin, and the
|
|
108
|
+
* runner that turns an executed script into an execution record
|
|
109
|
+
* (`coverage-artifact.ts`). There is nothing left to mirror wrong.
|
|
110
|
+
*
|
|
111
|
+
* That third caller is why this became shared rather than merely deduplicated.
|
|
112
|
+
* Colocation and execution were answering the same question — *which surface is
|
|
113
|
+
* this test about?* — with two different pieces of code, so a script vigiles had
|
|
114
|
+
* just RUN could not be attributed by the very rule that was already willing to
|
|
115
|
+
* credit it for existing.
|
|
116
|
+
*
|
|
117
|
+
* SEPARATORS ARE NORMALISED FIRST, then POSIX semantics apply. The disk detector
|
|
118
|
+
* feeds paths from `globSync`, which yields `\` on Windows; the browser twin
|
|
119
|
+
* feeds file-map keys, which are always `/`. Folding `\` to `/` makes one body
|
|
120
|
+
* correct for both instead of asking each caller to bring its own path module —
|
|
121
|
+
* which is exactly the seam the mirror lived in.
|
|
122
|
+
*
|
|
123
|
+
* ⚠️ THE COST, STATED: a POSIX file whose NAME literally contains a backslash is
|
|
124
|
+
* now read as a directory boundary. That is not reachable for either input here —
|
|
125
|
+
* a skill directory or a test filename with a `\` in it does not survive the glob
|
|
126
|
+
* patterns that discover it in the first place — and it is the same trade
|
|
127
|
+
* `hookScriptRefs` already makes one function up.
|
|
128
|
+
*/
|
|
129
|
+
function isColocatedTest(surface, testPath) {
|
|
130
|
+
const test = posixly(testPath);
|
|
131
|
+
const home = posixly(surface.path);
|
|
132
|
+
if (!(0, posix_path_js_1.basename)(test).startsWith(`${surface.name}.`))
|
|
133
|
+
return false;
|
|
134
|
+
// A root `SKILL.md` (single-skill-dir target) lives at ".", and discovery
|
|
135
|
+
// returns top-level files without a "./" prefix — so `dirname` is "." on both
|
|
136
|
+
// sides and the comparison holds without a special case.
|
|
137
|
+
return (0, posix_path_js_1.dirname)(test) === (0, posix_path_js_1.dirname)(home);
|
|
138
|
+
}
|
|
139
|
+
function posixly(p) {
|
|
140
|
+
return p.replaceAll("\\", "/");
|
|
141
|
+
}
|
|
94
142
|
/** Wrap a discovered path. Kept as a function so both twins share one shape. */
|
|
95
143
|
function prepareTest(path) {
|
|
96
144
|
return { path };
|
|
97
145
|
}
|
|
98
|
-
/**
|
|
99
|
-
*
|
|
100
|
-
|
|
146
|
+
/**
|
|
147
|
+
* A path that looks like a script, inside a `command` string. Both twins used to
|
|
148
|
+
* declare this separately; the extension vocabulary and the trailing boundary
|
|
149
|
+
* now live in `core/source-refs.ts`, shared with the hook scanner, so the three
|
|
150
|
+
* cannot drift and none can omit the boundary (without it, `hooks.json` matched
|
|
151
|
+
* as `hooks.js`).
|
|
152
|
+
*
|
|
153
|
+
* ⚠️ This one scans a SERIALIZED settings blob rather than a shell parse, so it
|
|
154
|
+
* cannot tell an operand from inline program text the way `commandWords` can.
|
|
155
|
+
* It stays a regex because every hit is gated on the file EXISTING before it
|
|
156
|
+
* becomes a surface — a stray match is dropped, never reported — so the failure
|
|
157
|
+
* mode here is under-counting, not accusation.
|
|
158
|
+
*/
|
|
159
|
+
const SCRIPT_RE = (0, source_refs_js_1.scriptRefPattern)();
|
|
101
160
|
/**
|
|
102
161
|
* Is this filename one the PAID real-model runner would actually run?
|
|
103
162
|
*
|
package/dist/coverage-probe.d.ts
CHANGED
|
@@ -50,12 +50,20 @@
|
|
|
50
50
|
* file is named by anyone. That API contract is the feature — it is how you
|
|
51
51
|
* test a hook you did not write, verbatim as its plugin ships it.
|
|
52
52
|
*
|
|
53
|
-
* - **The in-process tier CAN
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
53
|
+
* - **The in-process tier CAN — and since 2026-08-13 it DOES.** ⚠️ This bullet
|
|
54
|
+
* used to read *"`harness-assert.ts` / `load-hook.ts` contain ZERO
|
|
55
|
+
* `recordSurfaceProbe` calls — the tier where attribution needs no parsing at
|
|
56
|
+
* all is the tier that attributes nothing"*, and that is no longer true:
|
|
57
|
+
* `runCountedHookProgram` in harness-assert.ts records a `command` probe for a
|
|
58
|
+
* hook it is about to evaluate, when `loadHook` knew the file. Verified by
|
|
59
|
+
* count, not by memory — `harness-assert.ts` has the call, `load-hook.ts`
|
|
60
|
+
* still does not, and deliberately so (loading a hook is not running it; see
|
|
61
|
+
* that function's note). Left corrected rather than deleted because a header
|
|
62
|
+
* that advertises a gap somebody already closed sends the next reader to build
|
|
63
|
+
* a second recorder.
|
|
64
|
+
*
|
|
65
|
+
* It did not retire this parser, as predicted: it ADDS a source, and
|
|
66
|
+
* `runHook(command)` is still public.
|
|
59
67
|
*
|
|
60
68
|
* So parsing stays, and the lesson taken instead is about DIRECTION. The scan was
|
|
61
69
|
* a deny-list — skip what we recognise, attribute what is left — so every gap in
|
package/dist/coverage-probe.js
CHANGED
|
@@ -56,12 +56,20 @@ exports.probeTrace = probeTrace;
|
|
|
56
56
|
* file is named by anyone. That API contract is the feature — it is how you
|
|
57
57
|
* test a hook you did not write, verbatim as its plugin ships it.
|
|
58
58
|
*
|
|
59
|
-
* - **The in-process tier CAN
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
59
|
+
* - **The in-process tier CAN — and since 2026-08-13 it DOES.** ⚠️ This bullet
|
|
60
|
+
* used to read *"`harness-assert.ts` / `load-hook.ts` contain ZERO
|
|
61
|
+
* `recordSurfaceProbe` calls — the tier where attribution needs no parsing at
|
|
62
|
+
* all is the tier that attributes nothing"*, and that is no longer true:
|
|
63
|
+
* `runCountedHookProgram` in harness-assert.ts records a `command` probe for a
|
|
64
|
+
* hook it is about to evaluate, when `loadHook` knew the file. Verified by
|
|
65
|
+
* count, not by memory — `harness-assert.ts` has the call, `load-hook.ts`
|
|
66
|
+
* still does not, and deliberately so (loading a hook is not running it; see
|
|
67
|
+
* that function's note). Left corrected rather than deleted because a header
|
|
68
|
+
* that advertises a gap somebody already closed sends the next reader to build
|
|
69
|
+
* a second recorder.
|
|
70
|
+
*
|
|
71
|
+
* It did not retire this parser, as predicted: it ADDS a source, and
|
|
72
|
+
* `runHook(command)` is still public.
|
|
65
73
|
*
|
|
66
74
|
* So parsing stays, and the lesson taken instead is about DIRECTION. The scan was
|
|
67
75
|
* a deny-list — skip what we recognise, attribute what is left — so every gap in
|