vigiles 33.1.0 → 33.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-conformance.js +14 -10
- package/dist/adapter.d.ts +2 -0
- package/dist/adapters/claude-code/layout.js +3 -0
- package/dist/adapters/claude-code/output-style.d.ts +3 -0
- package/dist/adapters/claude-code/output-style.js +81 -0
- package/dist/adapters/claude-code/replies.d.ts +14 -0
- package/dist/adapters/claude-code/replies.js +78 -0
- package/dist/audit-report.d.ts +5 -0
- package/dist/audit-report.js +29 -20
- package/dist/audit-report.template.html +25 -25
- package/dist/audit-score.js +2 -2
- package/dist/check.d.ts +20 -0
- package/dist/check.js +87 -2
- package/dist/cli-main.js +25 -13
- package/dist/core/config-schema.d.ts +9 -0
- package/dist/core/config-schema.js +1 -0
- package/dist/core/eval-driver.d.ts +10 -0
- package/dist/core/frontmatter-read.d.ts +31 -0
- package/dist/core/frontmatter-read.js +4 -2
- package/dist/core/harness-driver.d.ts +2 -0
- package/dist/core/layout.d.ts +7 -0
- package/dist/core/output-style.d.ts +84 -0
- package/dist/core/output-style.js +90 -0
- package/dist/core/rule-meta.d.ts +1 -1
- package/dist/core/rule-meta.js +8 -0
- package/dist/core/types.d.ts +2 -0
- package/dist/coverage-evidence.d.ts +42 -0
- package/dist/coverage-evidence.js +39 -0
- package/dist/eval.d.ts +26 -3
- package/dist/eval.js +54 -5
- package/dist/harness-test.d.ts +30 -1
- package/dist/harness-test.js +65 -15
- package/dist/optimize.js +1 -5
- package/dist/output-style-arms.d.ts +27 -0
- package/dist/output-style-arms.js +46 -0
- package/dist/scan-files.js +1 -1
- package/dist/scan.d.ts +6 -0
- package/dist/scan.js +14 -4
- package/dist/score-core.d.ts +3 -0
- package/dist/score-core.js +5 -1
- package/dist/setup-plan.d.ts +1 -1
- package/dist/setup-plan.js +1 -0
- package/dist/test-coverage-files.d.ts +3 -1
- package/dist/test-coverage-files.js +20 -6
- package/dist/test-coverage.d.ts +18 -1
- package/dist/test-coverage.js +64 -14
- package/dist/test.d.ts +3 -1
- package/dist/test.js +8 -2
- package/dist/vigilesrc.schema.json +134 -0
- package/package.json +1 -1
package/dist/audit-score.js
CHANGED
|
@@ -450,12 +450,12 @@ function evaluated(r, firingMeasured) {
|
|
|
450
450
|
score: null,
|
|
451
451
|
weight: 1,
|
|
452
452
|
advisory: true,
|
|
453
|
-
findings: ["no surface
|
|
453
|
+
findings: ["no surface a real model could be measured on"],
|
|
454
454
|
};
|
|
455
455
|
}
|
|
456
456
|
const unevaluated = r.unevaluated ?? evaluable;
|
|
457
457
|
const evalCovered = evaluable - unevaluated;
|
|
458
|
-
const gap = `${String(unevaluated)} ${pluralizeLabel(unevaluated, "surface(s)
|
|
458
|
+
const gap = `${String(unevaluated)} ${pluralizeLabel(unevaluated, "surface(s) never measured with a real model")}`;
|
|
459
459
|
// Nothing on disk measures firing AND nothing ran this session → the question
|
|
460
460
|
// was never asked. Say that, and name the command — do NOT score it a 0.
|
|
461
461
|
if (evalCovered === 0 && !firingMeasured) {
|
package/dist/check.d.ts
CHANGED
|
@@ -137,6 +137,26 @@ export declare function onlyTools(allowed: readonly string[]): Check<Trace>;
|
|
|
137
137
|
export declare function skill(id: string): Check<Trace>;
|
|
138
138
|
/** The agent's final output contains a substring / matches a RegExp. */
|
|
139
139
|
export declare function output(matcher: string | RegExp): Check<Trace>;
|
|
140
|
+
/**
|
|
141
|
+
* EVERY reply of the run contains a substring / matches a RegExp — for a rule
|
|
142
|
+
* about each reply ("end with a status block"), where `output` sees only the
|
|
143
|
+
* last one. A run with no replies fails: there is nothing the rule held for.
|
|
144
|
+
*
|
|
145
|
+
* @experimental The matcher may grow a function form, and a check may come to
|
|
146
|
+
* be attached to a spec rule (vigiles#323).
|
|
147
|
+
*/
|
|
148
|
+
export declare function experimental_eachReply(matcher: string | Readonly<RegExp>): Readonly<Check<Trace>>;
|
|
149
|
+
/**
|
|
150
|
+
* How many replies of the run match, within bounds — `{ max: 1 }` is "at most
|
|
151
|
+
* one status block for one user message", which a Stop hook that makes the
|
|
152
|
+
* agent reply again can break while `output` still shows one.
|
|
153
|
+
*
|
|
154
|
+
* @experimental See {@link experimental_eachReply}.
|
|
155
|
+
*/
|
|
156
|
+
export declare function experimental_replyCount(matcher: string | Readonly<RegExp>, opts: {
|
|
157
|
+
readonly min?: number;
|
|
158
|
+
readonly max?: number;
|
|
159
|
+
}): Readonly<Check<Trace>>;
|
|
140
160
|
/** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
|
|
141
161
|
export declare function hookFired(event: string): Check<Trace>;
|
|
142
162
|
/**
|
package/dist/check.js
CHANGED
|
@@ -8,6 +8,8 @@ exports.notTool = notTool;
|
|
|
8
8
|
exports.onlyTools = onlyTools;
|
|
9
9
|
exports.skill = skill;
|
|
10
10
|
exports.output = output;
|
|
11
|
+
exports.experimental_eachReply = experimental_eachReply;
|
|
12
|
+
exports.experimental_replyCount = experimental_replyCount;
|
|
11
13
|
exports.hookFired = hookFired;
|
|
12
14
|
exports.received = received;
|
|
13
15
|
exports.turns = turns;
|
|
@@ -75,6 +77,29 @@ function truncate(s, n = 120) {
|
|
|
75
77
|
const flat = s.replace(/\s+/g, " ").trim();
|
|
76
78
|
return flat.length > n ? `${flat.slice(0, n)}…` : flat;
|
|
77
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Does `text` match? A RegExp is copied first: with `g` / `y`, `test` advances
|
|
82
|
+
* `lastIndex` on the object it is called on, so reusing the caller's object
|
|
83
|
+
* would miss on its second reply or its second trial.
|
|
84
|
+
*/
|
|
85
|
+
const matches = (matcher, text) => typeof matcher === "string"
|
|
86
|
+
? text.includes(matcher)
|
|
87
|
+
: new RegExp(matcher).test(text);
|
|
88
|
+
/** A matcher as the report prints it: the string, or the RegExp literal. */
|
|
89
|
+
const show = (matcher) => typeof matcher === "string" ? matcher : `/${matcher.source}/${matcher.flags}`;
|
|
90
|
+
/** Why a reply check cannot run: the trace carries no replies to read. */
|
|
91
|
+
const NO_REPLIES = "the run has no `replies` (the harness does not tell replies apart, or the run was not streamed — pass `transcript: true`)";
|
|
92
|
+
/** Why a `{ min, max }` bound cannot mean anything, or null when it can. */
|
|
93
|
+
function boundsProblem(opts) {
|
|
94
|
+
const given = [opts.min, opts.max].filter((n) => n !== undefined);
|
|
95
|
+
if (given.length === 0)
|
|
96
|
+
return "give `min`, `max` or both";
|
|
97
|
+
if (given.some((n) => !Number.isInteger(n) || n < 0))
|
|
98
|
+
return "bounds are whole numbers, 0 or more";
|
|
99
|
+
if (opts.min !== undefined && opts.max !== undefined && opts.min > opts.max)
|
|
100
|
+
return `min ${String(opts.min)} is above max ${String(opts.max)}`;
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
78
103
|
// ---------------------------------------------------------------------------
|
|
79
104
|
// Trace checks (an agent run — runHarness / runEval)
|
|
80
105
|
// ---------------------------------------------------------------------------
|
|
@@ -248,7 +273,7 @@ function output(matcher) {
|
|
|
248
273
|
return {
|
|
249
274
|
kind: "output",
|
|
250
275
|
eval: (t) => {
|
|
251
|
-
const pass =
|
|
276
|
+
const pass = matches(matcher, t.output);
|
|
252
277
|
return pass
|
|
253
278
|
? ok(`output matched ${String(matcher)}`)
|
|
254
279
|
: no(`expected output to ${isRe ? "match" : "contain"} ${String(matcher)}; got "${truncate(t.output) || "(empty)"}"`);
|
|
@@ -256,6 +281,66 @@ function output(matcher) {
|
|
|
256
281
|
toJSON: () => ({ kind: "output", matcher: String(matcher), regex: isRe }),
|
|
257
282
|
};
|
|
258
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* EVERY reply of the run contains a substring / matches a RegExp — for a rule
|
|
286
|
+
* about each reply ("end with a status block"), where `output` sees only the
|
|
287
|
+
* last one. A run with no replies fails: there is nothing the rule held for.
|
|
288
|
+
*
|
|
289
|
+
* @experimental The matcher may grow a function form, and a check may come to
|
|
290
|
+
* be attached to a spec rule (vigiles#323).
|
|
291
|
+
*/
|
|
292
|
+
function experimental_eachReply(matcher) {
|
|
293
|
+
return {
|
|
294
|
+
kind: "eachReply",
|
|
295
|
+
eval: (t) => {
|
|
296
|
+
if (t.replies === undefined)
|
|
297
|
+
return no(NO_REPLIES);
|
|
298
|
+
if (t.replies.length === 0)
|
|
299
|
+
return no("the run wrote no replies");
|
|
300
|
+
const miss = t.replies.findIndex((r) => !matches(matcher, r));
|
|
301
|
+
return miss === -1
|
|
302
|
+
? ok(`all ${String(t.replies.length)} replies matched ${show(matcher)}`)
|
|
303
|
+
: no(`reply ${String(miss + 1)} of ${String(t.replies.length)} did not match ${show(matcher)}: "${truncate(t.replies[miss] ?? "") || "(empty)"}"`);
|
|
304
|
+
},
|
|
305
|
+
toJSON: () => ({
|
|
306
|
+
kind: "eachReply",
|
|
307
|
+
matcher: show(matcher),
|
|
308
|
+
regex: typeof matcher !== "string",
|
|
309
|
+
}),
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* How many replies of the run match, within bounds — `{ max: 1 }` is "at most
|
|
314
|
+
* one status block for one user message", which a Stop hook that makes the
|
|
315
|
+
* agent reply again can break while `output` still shows one.
|
|
316
|
+
*
|
|
317
|
+
* @experimental See {@link experimental_eachReply}.
|
|
318
|
+
*/
|
|
319
|
+
function experimental_replyCount(matcher, opts) {
|
|
320
|
+
const problem = boundsProblem(opts);
|
|
321
|
+
if (problem !== null)
|
|
322
|
+
throw new Error(`replyCount: ${problem}`);
|
|
323
|
+
return {
|
|
324
|
+
kind: "replyCount",
|
|
325
|
+
eval: (t) => {
|
|
326
|
+
if (t.replies === undefined)
|
|
327
|
+
return no(NO_REPLIES);
|
|
328
|
+
const n = t.replies.filter((r) => matches(matcher, r)).length;
|
|
329
|
+
const pass = (opts.min === undefined || n >= opts.min) &&
|
|
330
|
+
(opts.max === undefined || n <= opts.max);
|
|
331
|
+
const bound = `min ${String(opts.min ?? "-")}, max ${String(opts.max ?? "-")}`;
|
|
332
|
+
const says = `${String(n)} of ${String(t.replies.length)} replies matched ${show(matcher)} (${bound})`;
|
|
333
|
+
return pass ? ok(says) : no(`expected ${bound}; ${says}`);
|
|
334
|
+
},
|
|
335
|
+
toJSON: () => ({
|
|
336
|
+
kind: "replyCount",
|
|
337
|
+
matcher: show(matcher),
|
|
338
|
+
regex: typeof matcher !== "string",
|
|
339
|
+
min: opts.min,
|
|
340
|
+
max: opts.max,
|
|
341
|
+
}),
|
|
342
|
+
};
|
|
343
|
+
}
|
|
259
344
|
/** A hook fired for this event (e.g. `"PreToolUse"`, `"Stop"`). */
|
|
260
345
|
function hookFired(event) {
|
|
261
346
|
return {
|
|
@@ -284,7 +369,7 @@ function received(matcher) {
|
|
|
284
369
|
const text = t.modelRequests
|
|
285
370
|
.map((r) => `${r.system} ${r.messages.map((m) => m.text).join(" ")}`)
|
|
286
371
|
.join(" ");
|
|
287
|
-
const pass =
|
|
372
|
+
const pass = matches(matcher, text);
|
|
288
373
|
return pass
|
|
289
374
|
? ok(`the model received ${String(matcher)}`)
|
|
290
375
|
: no(`expected the model to receive ${String(matcher)} (a slash-command expansion or injected context); ${t.modelRequests.length === 0 ? "no requests captured (eval tier captures none — use runHarness)" : `got "${truncate(text)}"`}`);
|
package/dist/cli-main.js
CHANGED
|
@@ -3595,6 +3595,18 @@ function harnessLayoutFor(root, config, flag) {
|
|
|
3595
3595
|
return adapter_registry_js_1.defaultAdapter.layout;
|
|
3596
3596
|
}
|
|
3597
3597
|
}
|
|
3598
|
+
/**
|
|
3599
|
+
* The rule that gates each surface kind, one row per kind. `satisfies` makes a
|
|
3600
|
+
* missing row a compile error: the nested ternary this replaced ended in the
|
|
3601
|
+
* hook rule, so a kind it did not name was gated by `untested-hook` and still
|
|
3602
|
+
* compiled.
|
|
3603
|
+
*/
|
|
3604
|
+
const UNTESTED_RULE = {
|
|
3605
|
+
skill: "untested-skill",
|
|
3606
|
+
agent: "untested-subagent",
|
|
3607
|
+
hook: "untested-hook",
|
|
3608
|
+
"output-style": "untested-output-style",
|
|
3609
|
+
};
|
|
3598
3610
|
/**
|
|
3599
3611
|
* The `untested-*` rules AS THE DETECTOR TAKES THEM: which kinds this repo
|
|
3600
3612
|
* enabled, and the discovery options merged from whichever of the three rules
|
|
@@ -3611,21 +3623,21 @@ function harnessLayoutFor(root, config, flag) {
|
|
|
3611
3623
|
*/
|
|
3612
3624
|
function untestedRules(config) {
|
|
3613
3625
|
const rules = config?.rules;
|
|
3614
|
-
const
|
|
3615
|
-
const
|
|
3616
|
-
const
|
|
3617
|
-
|
|
3618
|
-
...(0, types_js_1.ruleOptions)(rules?.[
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
};
|
|
3626
|
+
const severity = (kind) => (0, types_js_1.ruleSeverity)(rules?.[UNTESTED_RULE[kind]]);
|
|
3627
|
+
const names = Object.values(UNTESTED_RULE);
|
|
3628
|
+
const opts = names.reduce((acc, rule) => ({
|
|
3629
|
+
...acc,
|
|
3630
|
+
...(0, types_js_1.ruleOptions)(rules?.[rule]),
|
|
3631
|
+
}), {});
|
|
3632
|
+
const on = (kind) => severity(kind) !== false;
|
|
3622
3633
|
return {
|
|
3623
|
-
severity
|
|
3624
|
-
anyEnabled:
|
|
3634
|
+
severity,
|
|
3635
|
+
anyEnabled: names.some((rule) => (0, types_js_1.ruleSeverity)(rules?.[rule]) !== false),
|
|
3625
3636
|
options: {
|
|
3626
|
-
skills:
|
|
3627
|
-
agents:
|
|
3628
|
-
hooks:
|
|
3637
|
+
skills: on("skill"),
|
|
3638
|
+
agents: on("agent"),
|
|
3639
|
+
hooks: on("hook"),
|
|
3640
|
+
outputStyles: on("output-style"),
|
|
3629
3641
|
include: opts.include,
|
|
3630
3642
|
exclude: opts.exclude,
|
|
3631
3643
|
// Without this the `testExtension` documented on TestCoverageOptions was a
|
|
@@ -109,6 +109,15 @@ export declare const vigilesConfigSchema: z.ZodObject<{
|
|
|
109
109
|
exclude: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>, z.ZodTransform<readonly string[], string | string[]>>>;
|
|
110
110
|
testExtension: z.ZodOptional<z.ZodString>;
|
|
111
111
|
}, z.core.$strict>], null>]>>;
|
|
112
|
+
"untested-output-style": z.ZodDefault<z.ZodUnion<readonly [z.ZodPipe<z.ZodUnion<readonly [z.ZodLiteral<"warn">, z.ZodLiteral<"error">, z.ZodLiteral<false>, z.ZodLiteral<"off">, z.ZodLiteral<0>, z.ZodLiteral<1>, z.ZodLiteral<2>, z.ZodLiteral<true>]>, z.ZodTransform<false | "warn" | "error", boolean | 0 | 1 | 2 | "warn" | "error" | "off">>, z.ZodTuple<[z.ZodLiteral<"warn">, z.ZodObject<{
|
|
113
|
+
include: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>, z.ZodTransform<readonly string[], string | string[]>>>;
|
|
114
|
+
exclude: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>, z.ZodTransform<readonly string[], string | string[]>>>;
|
|
115
|
+
testExtension: z.ZodOptional<z.ZodString>;
|
|
116
|
+
}, z.core.$strict>], null>, z.ZodTuple<[z.ZodLiteral<"error">, z.ZodObject<{
|
|
117
|
+
include: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>, z.ZodTransform<readonly string[], string | string[]>>>;
|
|
118
|
+
exclude: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>, z.ZodTransform<readonly string[], string | string[]>>>;
|
|
119
|
+
testExtension: z.ZodOptional<z.ZodString>;
|
|
120
|
+
}, z.core.$strict>], null>]>>;
|
|
112
121
|
"unmarked-refs": z.ZodDefault<z.ZodPipe<z.ZodUnion<readonly [z.ZodLiteral<"warn">, z.ZodLiteral<"error">, z.ZodLiteral<false>, z.ZodLiteral<"off">, z.ZodLiteral<0>, z.ZodLiteral<1>, z.ZodLiteral<2>, z.ZodLiteral<true>]>, z.ZodTransform<false | "warn" | "error", boolean | 0 | 1 | 2 | "warn" | "error" | "off">>>;
|
|
113
122
|
"subagent-tool-contract": z.ZodDefault<z.ZodPipe<z.ZodUnion<readonly [z.ZodLiteral<"warn">, z.ZodLiteral<"error">, z.ZodLiteral<false>, z.ZodLiteral<"off">, z.ZodLiteral<0>, z.ZodLiteral<1>, z.ZodLiteral<2>, z.ZodLiteral<true>]>, z.ZodTransform<false | "warn" | "error", boolean | 0 | 1 | 2 | "warn" | "error" | "off">>>;
|
|
114
123
|
"hook-events": z.ZodDefault<z.ZodPipe<z.ZodUnion<readonly [z.ZodLiteral<"warn">, z.ZodLiteral<"error">, z.ZodLiteral<false>, z.ZodLiteral<"off">, z.ZodLiteral<0>, z.ZodLiteral<1>, z.ZodLiteral<2>, z.ZodLiteral<true>]>, z.ZodTransform<false | "warn" | "error", boolean | 0 | 1 | 2 | "warn" | "error" | "off">>>;
|
|
@@ -141,6 +141,7 @@ const rulesSchema = zod_1.z
|
|
|
141
141
|
"untested-skill": withOptions(testCoverageConfig).default("warn"),
|
|
142
142
|
"untested-subagent": withOptions(testCoverageConfig).default("warn"),
|
|
143
143
|
"untested-hook": withOptions(testCoverageConfig).default("warn"),
|
|
144
|
+
"untested-output-style": withOptions(testCoverageConfig).default("warn"),
|
|
144
145
|
"unmarked-refs": severitySchema.default("warn"),
|
|
145
146
|
"subagent-tool-contract": severitySchema.default("warn"),
|
|
146
147
|
"hook-events": severitySchema.default("warn"),
|
|
@@ -71,6 +71,14 @@ export interface Trace {
|
|
|
71
71
|
readonly hooks: readonly HookFire[];
|
|
72
72
|
/** The agent's final answer text (the terminal `result` event), or "". */
|
|
73
73
|
readonly output: string;
|
|
74
|
+
/**
|
|
75
|
+
* Every reply the agent ended a turn with, in order — the `output` is only
|
|
76
|
+
* the last. A run a Stop hook continued has two, and anything the first one
|
|
77
|
+
* printed (a status footer) shows only here. Text the agent writes before a
|
|
78
|
+
* tool call is narration, not a reply. Absent when the harness's run output
|
|
79
|
+
* does not tell replies apart, or the run was not streamed (`transcript: true`).
|
|
80
|
+
*/
|
|
81
|
+
readonly replies?: readonly string[];
|
|
74
82
|
/**
|
|
75
83
|
* The requests the model received, captured by the scripted mock — each with
|
|
76
84
|
* its `system` prompt and `messages`, flattened to text. Lets a test assert
|
|
@@ -153,6 +161,8 @@ export interface ParsedModelRun {
|
|
|
153
161
|
readonly toolCalls: ToolCall[];
|
|
154
162
|
readonly hooks: HookFire[];
|
|
155
163
|
readonly subagents: SubagentTrace[];
|
|
164
|
+
/** Every reply, in order; absent when the harness cannot tell them apart. */
|
|
165
|
+
readonly replies?: readonly string[];
|
|
156
166
|
readonly usage: EvalUsage;
|
|
157
167
|
}
|
|
158
168
|
export type ModelOutputParser = (out: RunOut) => ParsedModelRun;
|
|
@@ -1,3 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lenient frontmatter reader — ONE reader for the SKILL.md / subagent `---` block,
|
|
3
|
+
* shared by `scan` and the PreToolUse rail (`agent-runtime`). It is deliberately
|
|
4
|
+
* fault-tolerant: it audits arbitrary third-party files, so it must never throw
|
|
5
|
+
* and must salvage the few scalar/list fields it needs even from a block that
|
|
6
|
+
* isn't valid YAML.
|
|
7
|
+
*
|
|
8
|
+
* The strategy is "real parser, with a safety net": try `js-yaml` (so block
|
|
9
|
+
* scalars, quoted/multi-line values, and flow arrays parse correctly for free);
|
|
10
|
+
* if the block isn't valid YAML, fall back to a regex salvage of the requested
|
|
11
|
+
* field and record `malformed: true` (the signal the `frontmatter-valid` rule
|
|
12
|
+
* reports). A single bad line therefore never blanks out a whole file's metadata.
|
|
13
|
+
*
|
|
14
|
+
* This replaces three divergent hand-parsers (the old `readField` in scan.ts and
|
|
15
|
+
* the regex parse in agent-runtime.ts); `core/frontmatter.ts` is a DIFFERENT
|
|
16
|
+
* concern (the Level-1 `vigiles:` rule block) and is untouched.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* 🔴 `js-yaml` IS REQUIRED LAZILY, AND THAT IS A MEASUREMENT, NOT A STYLE. The
|
|
20
|
+
* same wrapper is in `core/settings-codec.ts` and `core/markdown.ts` with the
|
|
21
|
+
* reason recorded: a top-level import puts the parser into the module graph of
|
|
22
|
+
* every HOOK DECISION, because the hook runtime reaches the adapter registry
|
|
23
|
+
* and from there a layout. This module became reachable that way on 2026-09-21
|
|
24
|
+
* through `PluginLayout.instructionChain` (a rule's `paths:` frontmatter is
|
|
25
|
+
* what decides whether it loads at launch) — measured by
|
|
26
|
+
* `src/hook-runtime-graph.test.ts`: 37 modules became 71 with an eager import.
|
|
27
|
+
* `tsc` lowers this to CommonJS, so the `require` does not run until a
|
|
28
|
+
* frontmatter block is actually parsed, and {@link frontmatterBody}, which needs
|
|
29
|
+
* no YAML at all, never triggers it.
|
|
30
|
+
*/
|
|
31
|
+
export declare const yaml: () => typeof import("js-yaml");
|
|
1
32
|
export interface FrontmatterRead {
|
|
2
33
|
/** Parsed mapping when the block is valid YAML, else null (malformed or scalar). */
|
|
3
34
|
readonly data: Record<string, unknown> | null;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.yaml = void 0;
|
|
3
4
|
exports.frontmatterBody = frontmatterBody;
|
|
4
5
|
exports.readFrontmatter = readFrontmatter;
|
|
5
6
|
exports.frontmatterScalar = frontmatterScalar;
|
|
@@ -35,6 +36,7 @@ exports.frontmatterList = frontmatterList;
|
|
|
35
36
|
* no YAML at all, never triggers it.
|
|
36
37
|
*/
|
|
37
38
|
const yaml = () => require("js-yaml");
|
|
39
|
+
exports.yaml = yaml;
|
|
38
40
|
// Frontmatter is the very first thing in the file. Anchoring at the start — not
|
|
39
41
|
// `(?:^|\n)` — means a `---` horizontal rule in the BODY is never mistaken for
|
|
40
42
|
// frontmatter (which matters for the malformed-YAML verdict). A leading BOM is
|
|
@@ -67,7 +69,7 @@ function readFrontmatter(markdown) {
|
|
|
67
69
|
return { data: null, block: null, malformed: false };
|
|
68
70
|
const block = m[1];
|
|
69
71
|
try {
|
|
70
|
-
const parsed = yaml().load(block);
|
|
72
|
+
const parsed = (0, exports.yaml)().load(block);
|
|
71
73
|
if (parsed !== null &&
|
|
72
74
|
typeof parsed === "object" &&
|
|
73
75
|
!Array.isArray(parsed)) {
|
|
@@ -82,7 +84,7 @@ function readFrontmatter(markdown) {
|
|
|
82
84
|
return { data: null, block, malformed: false };
|
|
83
85
|
}
|
|
84
86
|
catch (e) {
|
|
85
|
-
if (e instanceof yaml().YAMLException)
|
|
87
|
+
if (e instanceof (0, exports.yaml)().YAMLException)
|
|
86
88
|
return { data: null, block, malformed: true };
|
|
87
89
|
throw e;
|
|
88
90
|
}
|
|
@@ -130,6 +130,8 @@ export interface ParsedRun {
|
|
|
130
130
|
readonly hooks: readonly HookFire[];
|
|
131
131
|
/** The agent's final answer text, or "". */
|
|
132
132
|
readonly output: string;
|
|
133
|
+
/** Every reply the agent ended a turn with; absent when the output is not a stream. */
|
|
134
|
+
readonly replies?: readonly string[];
|
|
133
135
|
}
|
|
134
136
|
/**
|
|
135
137
|
* What a harness brings so `runHarnessTest` can drive it. Implemented in each
|
package/dist/core/layout.d.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* Paths are repo-relative (POSIX-style, `join`-friendly). The Claude Code
|
|
11
11
|
* implementation is `claudeCodeLayout` in `src/adapters/claude-code/layout.ts`.
|
|
12
12
|
*/
|
|
13
|
+
import type { OutputStyleRules } from "./output-style.js";
|
|
13
14
|
import type { InstructionChain } from "./instruction-chain.js";
|
|
14
15
|
import type { SettingsCodec } from "./settings-codec.js";
|
|
15
16
|
/**
|
|
@@ -127,6 +128,12 @@ export interface PluginLayout {
|
|
|
127
128
|
* docblock there for the count behind that.
|
|
128
129
|
*/
|
|
129
130
|
readonly rulesDir?: string;
|
|
131
|
+
/**
|
|
132
|
+
* Where this harness keeps output styles and how it switches one on (absent =
|
|
133
|
+
* this harness has none). Not a {@link SurfaceKind}: like a rules file, a
|
|
134
|
+
* style is read into the session, not invoked. See `output-style.ts`.
|
|
135
|
+
*/
|
|
136
|
+
readonly outputStyles?: OutputStyleRules;
|
|
130
137
|
/**
|
|
131
138
|
* The WORD a per-machine settings sibling inserts before the extension, e.g.
|
|
132
139
|
* `local` → `.claude/settings.local.json` (absent = this harness has no
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Output styles: files that set how the agent talks for a whole session,
|
|
3
|
+
* chosen by a name in the harness settings.
|
|
4
|
+
*
|
|
5
|
+
* The core knows only that idea. Where the files live, how a name is read from
|
|
6
|
+
* one, which setting selects it and how a run shows that it arrived are each
|
|
7
|
+
* harness's own rules, given by its layout as `outputStyles`. A layout without
|
|
8
|
+
* them is a harness that has no output styles.
|
|
9
|
+
*/
|
|
10
|
+
import type { ModelRequest } from "./harness-driver.js";
|
|
11
|
+
import type { PluginLayout } from "./layout.js";
|
|
12
|
+
/** One style, read by its harness's rules. */
|
|
13
|
+
export interface OutputStyle {
|
|
14
|
+
/**
|
|
15
|
+
* Relative to the directory the caller scanned (a bundle), `/`-separated,
|
|
16
|
+
* exactly as the caller's file map keyed it. A plain string by the same
|
|
17
|
+
* contract as every scan-core path; `Bundle.scanned` in `frame.ts` gives it
|
|
18
|
+
* a frame. Not the `BundlePath` brand: that type is internal and must not
|
|
19
|
+
* reach this public interface.
|
|
20
|
+
*/
|
|
21
|
+
readonly path: string;
|
|
22
|
+
/** The name a setting must spell, or null when the harness cannot tell. */
|
|
23
|
+
readonly name: string | null;
|
|
24
|
+
/** The text the harness delivers, frontmatter removed. */
|
|
25
|
+
readonly body: string;
|
|
26
|
+
}
|
|
27
|
+
/** How one harness keeps output styles and switches one on. */
|
|
28
|
+
export interface OutputStyleRules {
|
|
29
|
+
/** The folder of style files, e.g. `output-styles`. */
|
|
30
|
+
readonly dir: string;
|
|
31
|
+
/** Whether a path inside that folder is a file the harness loads. */
|
|
32
|
+
readonly isStyleFile: (pathInDir: string) => boolean;
|
|
33
|
+
/** Reads one style file as this harness does. */
|
|
34
|
+
readonly read: (path: string, text: string) => OutputStyle;
|
|
35
|
+
/** The settings that switch the named style on. */
|
|
36
|
+
readonly select: (name: string) => Readonly<Record<string, unknown>>;
|
|
37
|
+
/**
|
|
38
|
+
* The settings that switch every style off, so a run without the style does
|
|
39
|
+
* not pick up one the user turned on for all their projects.
|
|
40
|
+
*/
|
|
41
|
+
readonly selectNone: Readonly<Record<string, unknown>>;
|
|
42
|
+
/** Whether one model request carries the style. */
|
|
43
|
+
readonly reached: (style: OutputStyle, request: ModelRequest) => boolean;
|
|
44
|
+
}
|
|
45
|
+
/** What a repository holds, or that its harness has no such thing. */
|
|
46
|
+
export type OutputStyleScan = {
|
|
47
|
+
readonly kind: "not-supported";
|
|
48
|
+
} | {
|
|
49
|
+
readonly kind: "found";
|
|
50
|
+
readonly styles: readonly OutputStyle[];
|
|
51
|
+
};
|
|
52
|
+
type StyleLayout = Pick<PluginLayout, "outputStyles" | "userSurfaceRoot">;
|
|
53
|
+
/** The folders a harness reads styles from: a plugin's root and the user surface root. */
|
|
54
|
+
export declare function outputStyleHomes(layout: StyleLayout): readonly string[];
|
|
55
|
+
/** Every style among `files` (repository path → text). Pure: the caller reads the disk. */
|
|
56
|
+
export declare function findOutputStyles(layout: StyleLayout, files: ReadonlyMap<string, string>): OutputStyleScan;
|
|
57
|
+
/** A run with one style switched on, or why it cannot be set up. */
|
|
58
|
+
export type StyleRunPlan = {
|
|
59
|
+
readonly kind: "refused";
|
|
60
|
+
readonly reason: string;
|
|
61
|
+
} | {
|
|
62
|
+
readonly kind: "planned";
|
|
63
|
+
/** The style as the harness will read it, at its place in the work dir. */
|
|
64
|
+
readonly style: OutputStyle;
|
|
65
|
+
readonly files: Readonly<Record<string, string>>;
|
|
66
|
+
readonly settings: Readonly<Record<string, unknown>>;
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Put one style file into a test's fixture and switch it on BY THE NAME THE
|
|
70
|
+
* HARNESS READS from it — never a name the caller types. Pure: the caller reads
|
|
71
|
+
* the file. Refuses instead of guessing when the harness has no styles, would
|
|
72
|
+
* not load such a file, cannot tell its name, or the fixture already decides.
|
|
73
|
+
*/
|
|
74
|
+
export declare function planStyleRun(layout: StyleLayout, source: {
|
|
75
|
+
readonly path: string;
|
|
76
|
+
readonly text: string;
|
|
77
|
+
}, fixture: {
|
|
78
|
+
readonly files: Readonly<Record<string, string>>;
|
|
79
|
+
readonly settings: unknown;
|
|
80
|
+
}): StyleRunPlan;
|
|
81
|
+
/** Whether any request of a run carried the style, by its harness's own rule. */
|
|
82
|
+
export declare function styleReached(rules: OutputStyleRules, style: OutputStyle, requests: readonly ModelRequest[]): boolean;
|
|
83
|
+
export {};
|
|
84
|
+
//# sourceMappingURL=output-style.d.ts.map
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.outputStyleHomes = outputStyleHomes;
|
|
4
|
+
exports.findOutputStyles = findOutputStyles;
|
|
5
|
+
exports.planStyleRun = planStyleRun;
|
|
6
|
+
exports.styleReached = styleReached;
|
|
7
|
+
/** The folders a harness reads styles from: a plugin's root and the user surface root. */
|
|
8
|
+
function outputStyleHomes(layout) {
|
|
9
|
+
const dir = layout.outputStyles?.dir;
|
|
10
|
+
if (dir === undefined)
|
|
11
|
+
return [];
|
|
12
|
+
const root = layout.userSurfaceRoot;
|
|
13
|
+
return root === undefined || root === "" ? [dir] : [dir, `${root}/${dir}`];
|
|
14
|
+
}
|
|
15
|
+
const pathInside = (home, path) => path.startsWith(`${home}/`) ? path.slice(home.length + 1) : null;
|
|
16
|
+
/** Every style among `files` (repository path → text). Pure: the caller reads the disk. */
|
|
17
|
+
function findOutputStyles(layout, files) {
|
|
18
|
+
const rules = layout.outputStyles;
|
|
19
|
+
if (rules === undefined)
|
|
20
|
+
return { kind: "not-supported" };
|
|
21
|
+
const homes = outputStyleHomes(layout);
|
|
22
|
+
const isStyle = (path) => homes.some((home) => {
|
|
23
|
+
const rest = pathInside(home, path);
|
|
24
|
+
return rest !== null && rules.isStyleFile(rest);
|
|
25
|
+
});
|
|
26
|
+
const styles = [...files]
|
|
27
|
+
.filter(([path]) => isStyle(path))
|
|
28
|
+
.map(([path, text]) => rules.read(path, text));
|
|
29
|
+
return {
|
|
30
|
+
kind: "found",
|
|
31
|
+
styles: [...styles].sort((a, b) => a.path.localeCompare(b.path)),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
const isPlainObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
35
|
+
/** Where the style goes and what it is called there, or why it cannot go. */
|
|
36
|
+
function placeStyle(layout, source) {
|
|
37
|
+
const rules = layout.outputStyles;
|
|
38
|
+
const home = outputStyleHomes(layout).at(-1);
|
|
39
|
+
if (rules === undefined || home === undefined)
|
|
40
|
+
return "this harness has no output styles";
|
|
41
|
+
// Either separator: a Windows path from `path.resolve` uses backslashes.
|
|
42
|
+
const file = source.path.split(/[\\/]/).at(-1) ?? source.path;
|
|
43
|
+
if (!rules.isStyleFile(file))
|
|
44
|
+
return `this harness would not load "${file}" as an output style`;
|
|
45
|
+
const style = rules.read(`${home}/${file}`, source.text);
|
|
46
|
+
return style.name === null
|
|
47
|
+
? `cannot tell which name selects "${source.path}"`
|
|
48
|
+
: style;
|
|
49
|
+
}
|
|
50
|
+
/** The fixture's settings with the selection added, or why they clash. */
|
|
51
|
+
function selectIn(settings, selection) {
|
|
52
|
+
const base = settings ?? {};
|
|
53
|
+
if (!isPlainObject(base))
|
|
54
|
+
return "the fixture's settings are not an object";
|
|
55
|
+
const clash = Object.keys(selection).find((key) => key in base);
|
|
56
|
+
return clash === undefined
|
|
57
|
+
? { ...base, ...selection }
|
|
58
|
+
: `the fixture's settings already set "${clash}"; the style option sets it`;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Put one style file into a test's fixture and switch it on BY THE NAME THE
|
|
62
|
+
* HARNESS READS from it — never a name the caller types. Pure: the caller reads
|
|
63
|
+
* the file. Refuses instead of guessing when the harness has no styles, would
|
|
64
|
+
* not load such a file, cannot tell its name, or the fixture already decides.
|
|
65
|
+
*/
|
|
66
|
+
function planStyleRun(layout, source, fixture) {
|
|
67
|
+
const style = placeStyle(layout, source);
|
|
68
|
+
if (typeof style === "string")
|
|
69
|
+
return { kind: "refused", reason: style };
|
|
70
|
+
if (style.path in fixture.files)
|
|
71
|
+
return {
|
|
72
|
+
kind: "refused",
|
|
73
|
+
reason: `the fixture already has a file at ${style.path}`,
|
|
74
|
+
};
|
|
75
|
+
const selection = layout.outputStyles?.select(style.name ?? "") ?? {};
|
|
76
|
+
const settings = selectIn(fixture.settings, selection);
|
|
77
|
+
return typeof settings === "string"
|
|
78
|
+
? { kind: "refused", reason: settings }
|
|
79
|
+
: {
|
|
80
|
+
kind: "planned",
|
|
81
|
+
style,
|
|
82
|
+
files: { ...fixture.files, [style.path]: source.text },
|
|
83
|
+
settings,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** Whether any request of a run carried the style, by its harness's own rule. */
|
|
87
|
+
function styleReached(rules, style, requests) {
|
|
88
|
+
return requests.some((request) => rules.reached(style, request));
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=output-style.js.map
|
package/dist/core/rule-meta.d.ts
CHANGED
|
@@ -37,7 +37,7 @@ import type { RulesConfig } from "./types.js";
|
|
|
37
37
|
*/
|
|
38
38
|
export type RuleBucket = "structural-closed" | "external-decidable" | "heuristic-behavioral";
|
|
39
39
|
/** The artifact surface a rule reads (coarse; a rule may span several). */
|
|
40
|
-
export type RuleSurface = "instruction" | "skill" | "subagent" | "hook" | "mcp" | "plugin" | "docs";
|
|
40
|
+
export type RuleSurface = "instruction" | "skill" | "subagent" | "hook" | "output-style" | "mcp" | "plugin" | "docs";
|
|
41
41
|
/** Where a rule sits by default — `"off"` is the normalized form of `false`. */
|
|
42
42
|
export type RuleDefaultSeverity = "error" | "warn" | "off";
|
|
43
43
|
/** Every named rule: the `RulesConfig` keys plus the built-in `orphan-docs`. */
|
package/dist/core/rule-meta.js
CHANGED
|
@@ -69,6 +69,14 @@ exports.RULE_META = {
|
|
|
69
69
|
summary: "A file-backed hook script ships with a test or eval.",
|
|
70
70
|
detector: "findUntestedSurfaces",
|
|
71
71
|
},
|
|
72
|
+
"untested-output-style": {
|
|
73
|
+
id: "untested-output-style",
|
|
74
|
+
bucket: "external-decidable",
|
|
75
|
+
surface: ["output-style"],
|
|
76
|
+
defaultSeverity: "warn",
|
|
77
|
+
summary: "An output style ships with a test or eval.",
|
|
78
|
+
detector: "findUntestedSurfaces",
|
|
79
|
+
},
|
|
72
80
|
// --- Reference marking ----------------------------------------------------
|
|
73
81
|
"unmarked-refs": {
|
|
74
82
|
id: "unmarked-refs",
|
package/dist/core/types.d.ts
CHANGED
|
@@ -167,6 +167,8 @@ export interface RulesConfig {
|
|
|
167
167
|
"untested-subagent"?: RuleWithOptions<TestCoverageConfig>;
|
|
168
168
|
/** Flag a hook script that ships with no test or eval. Default: "warn". */
|
|
169
169
|
"untested-hook"?: RuleWithOptions<TestCoverageConfig>;
|
|
170
|
+
/** Flag an output style that ships with no test or eval. Default: "warn". */
|
|
171
|
+
"untested-output-style"?: RuleWithOptions<TestCoverageConfig>;
|
|
170
172
|
/**
|
|
171
173
|
* Nudge (or block) when an instruction file has code-shaped references that
|
|
172
174
|
* aren't expressed as vigiles marks (so the lint can't verify them), or a
|
|
@@ -197,4 +197,46 @@ export declare function formatEvidence(counts: EvidenceCounts): string;
|
|
|
197
197
|
* (`basename` of the audited dir on disk, the repo name in the browser).
|
|
198
198
|
*/
|
|
199
199
|
export declare function declaredSurfaceName(content: string): string | null;
|
|
200
|
+
/**
|
|
201
|
+
* One output style as a coverage surface — shared by the disk detector and the
|
|
202
|
+
* browser twin, so the two name and exempt a style the same way.
|
|
203
|
+
*
|
|
204
|
+
* Named by FILE (`status-block.md` → `status-block`), not by the name a
|
|
205
|
+
* harness selects it by: the surface name is what a colocated test is spelled
|
|
206
|
+
* after, and a declared name may hold spaces or colons no file name should.
|
|
207
|
+
*/
|
|
208
|
+
export declare function outputStyleSurface(path: string, content: string): {
|
|
209
|
+
readonly kind: "output-style";
|
|
210
|
+
readonly path: string;
|
|
211
|
+
readonly name: string;
|
|
212
|
+
readonly tokens: readonly string[];
|
|
213
|
+
readonly ignored: boolean;
|
|
214
|
+
};
|
|
215
|
+
/**
|
|
216
|
+
* How many output styles a scan found, or that the harness has none. Shared by
|
|
217
|
+
* the disk and browser scan engines, so the inventory reads the one discovery
|
|
218
|
+
* pass both already ran. Exempt styles (`vigiles:ignore-test`) are counted: the
|
|
219
|
+
* marker waives the test, not the style, and a style-only repository with an
|
|
220
|
+
* exempt style is not an empty machine.
|
|
221
|
+
*/
|
|
222
|
+
export declare function outputStyleCount(layouts: readonly {
|
|
223
|
+
readonly outputStyles?: unknown;
|
|
224
|
+
}[], tier: {
|
|
225
|
+
readonly covered: readonly {
|
|
226
|
+
readonly kind: string;
|
|
227
|
+
}[];
|
|
228
|
+
readonly untested: readonly {
|
|
229
|
+
readonly kind: string;
|
|
230
|
+
}[];
|
|
231
|
+
}, exempt: readonly {
|
|
232
|
+
readonly kind: string;
|
|
233
|
+
}[]): number | "not-supported";
|
|
234
|
+
/** The inventory counts both scan engines report the same way. */
|
|
235
|
+
export declare function inventoryCounts(files: Readonly<Record<string, string>>, isCommand: (path: string) => boolean, layouts: Parameters<typeof outputStyleCount>[0], coverage: {
|
|
236
|
+
readonly harness: Parameters<typeof outputStyleCount>[1];
|
|
237
|
+
readonly exemptSurfaces: Parameters<typeof outputStyleCount>[2];
|
|
238
|
+
}): {
|
|
239
|
+
readonly commands: number;
|
|
240
|
+
readonly outputStyles: number | "not-supported";
|
|
241
|
+
};
|
|
200
242
|
//# sourceMappingURL=coverage-evidence.d.ts.map
|