vigiles 33.1.0 → 33.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +3 -2
  2. package/dist/adapter-conformance.js +14 -10
  3. package/dist/adapter.d.ts +3 -1
  4. package/dist/adapters/claude-code/layout.js +3 -0
  5. package/dist/adapters/claude-code/output-style.d.ts +3 -0
  6. package/dist/adapters/claude-code/output-style.js +81 -0
  7. package/dist/adapters/claude-code/replies.d.ts +14 -0
  8. package/dist/adapters/claude-code/replies.js +78 -0
  9. package/dist/adapters/claude-code/runtime.js +70 -0
  10. package/dist/adapters/claude-code/skill-reachability.d.ts +42 -21
  11. package/dist/adapters/claude-code/skill-reachability.js +91 -28
  12. package/dist/audit-report.d.ts +5 -0
  13. package/dist/audit-report.js +29 -20
  14. package/dist/audit-report.template.html +25 -25
  15. package/dist/audit-score.js +2 -2
  16. package/dist/check.d.ts +20 -0
  17. package/dist/check.js +87 -2
  18. package/dist/cli-main.js +249 -30
  19. package/dist/core/adapter.d.ts +7 -0
  20. package/dist/core/config-schema.d.ts +9 -0
  21. package/dist/core/config-schema.js +1 -0
  22. package/dist/core/eval-driver.d.ts +10 -0
  23. package/dist/core/frontmatter-read.d.ts +31 -0
  24. package/dist/core/frontmatter-read.js +4 -2
  25. package/dist/core/harness-driver.d.ts +2 -0
  26. package/dist/core/install-reader.js +15 -0
  27. package/dist/core/layout.d.ts +17 -0
  28. package/dist/core/layout.js +17 -0
  29. package/dist/core/output-style.d.ts +84 -0
  30. package/dist/core/output-style.js +90 -0
  31. package/dist/core/rule-meta.d.ts +1 -1
  32. package/dist/core/rule-meta.js +8 -0
  33. package/dist/core/run-env.d.ts +50 -0
  34. package/dist/core/run-env.js +51 -0
  35. package/dist/core/runtime.d.ts +44 -0
  36. package/dist/core/skill-link-target.d.ts +20 -0
  37. package/dist/core/skill-link-target.js +33 -0
  38. package/dist/core/types.d.ts +2 -0
  39. package/dist/coverage-evidence.d.ts +42 -0
  40. package/dist/coverage-evidence.js +39 -0
  41. package/dist/eval.d.ts +65 -42
  42. package/dist/eval.js +116 -120
  43. package/dist/harness-test.d.ts +65 -1
  44. package/dist/harness-test.js +114 -22
  45. package/dist/optimize.js +1 -5
  46. package/dist/output-style-arms.d.ts +27 -0
  47. package/dist/output-style-arms.js +51 -0
  48. package/dist/plugin-declaration.d.ts +6 -6
  49. package/dist/plugin-declaration.js +6 -6
  50. package/dist/scan-files.js +1 -1
  51. package/dist/scan.d.ts +6 -0
  52. package/dist/scan.js +14 -4
  53. package/dist/score-core.d.ts +3 -0
  54. package/dist/score-core.js +5 -1
  55. package/dist/setup-plan.d.ts +50 -10
  56. package/dist/setup-plan.js +103 -68
  57. package/dist/skill-links.d.ts +183 -0
  58. package/dist/skill-links.js +255 -0
  59. package/dist/test-coverage-files.d.ts +3 -1
  60. package/dist/test-coverage-files.js +20 -6
  61. package/dist/test-coverage.d.ts +18 -1
  62. package/dist/test-coverage.js +101 -15
  63. package/dist/test.d.ts +3 -1
  64. package/dist/test.js +8 -2
  65. package/dist/vigilesrc.schema.json +134 -0
  66. package/package.json +1 -1
@@ -2,10 +2,15 @@ import type { HarnessAdapter } from "./core/adapter.js";
2
2
  import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn } from "./core/harness-driver.js";
3
3
  import type { Trace, SubagentTrace } from "./core/eval-driver.js";
4
4
  export type { Trace, SubagentTrace } from "./core/eval-driver.js";
5
+ import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
6
+ /** Re-exported for the eval tier's parser, which reads the same stream. */
7
+ export { parseReplies, reachedModel };
8
+ import type { HarnessRuntime } from "./core/runtime.js";
5
9
  import { type SandboxMode } from "./sandbox.js";
6
10
  export { scriptModel } from "./mock-model.js";
7
11
  export type { ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, } from "./core/harness-driver.js";
8
- export { loadPlugin, resolveHarness, } from "./adapters/claude-code/plugin-loader.js";
12
+ import { loadPlugin, resolveHarness } from "./adapters/claude-code/plugin-loader.js";
13
+ export { loadPlugin, resolveHarness };
9
14
  export { decideSandbox, specTrusted, sandboxAvailable, type SandboxMode, } from "./sandbox.js";
10
15
  export interface HarnessTestSpec {
11
16
  /** Fixture files to write in a fresh temp working dir (path → contents). */
@@ -28,6 +33,21 @@ export interface HarnessTestSpec {
28
33
  * `plugin` still layer on top. Resolved to an absolute path.
29
34
  */
30
35
  readonly pluginDir?: string;
36
+ /**
37
+ * Path to an OUTPUT STYLE file to switch on for this run. The style is
38
+ * written where the harness keeps styles, and selected by the name the
39
+ * harness reads from the file — never by a name you type, because a setting
40
+ * that misses the name loads no style and raises no error.
41
+ *
42
+ * The run is then checked: if no model request carried the style,
43
+ * `runHarnessTest` THROWS instead of returning a trace, so a style that never
44
+ * loaded cannot pass a test. Refused up front on a harness without output
45
+ * styles, or when the fixture's files or settings already decide.
46
+ *
47
+ * What it proves is delivery. A scripted model does not read the style, so
48
+ * nothing in this tier shows that a model FOLLOWS it.
49
+ */
50
+ readonly outputStyle?: string;
31
51
  /** The scripted model turns the agent will take. */
32
52
  readonly model: readonly ModelTurn[];
33
53
  /** The user prompt. Default: "go". */
@@ -107,6 +127,31 @@ export declare function parseHooks(stdout: string): HookFire[];
107
127
  export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[];
108
128
  /** Parse the `claude` stdout/stream into the unified trace fields. */
109
129
  export declare function parseClaudeRun(stdout: string): ParsedRun;
130
+ /**
131
+ * Where a harness-tier run's agent gets HOME and the rest of its environment.
132
+ *
133
+ * - `inherit` — the caller's env (real HOME), minus the harness's declared
134
+ * session identity. The default of `runHarnessTest`: a test that reads the
135
+ * machine's config keeps doing so, but no run executes AS the caller's live
136
+ * session.
137
+ * - `throwaway` — a fresh HOME/TMPDIR at `dir`, the OS essentials, and nothing
138
+ * else from the caller: no harness auth (the scripted mock needs none, and a
139
+ * config-dir variable would point the run back at the real config), no
140
+ * identity, no secrets. The same scrubbed shape as the eval tier's
141
+ * `ephemeralEnv` (`scrubbedRunEnv`), without the auth.
142
+ */
143
+ export type HarnessRunHome = {
144
+ readonly home: "inherit";
145
+ } | {
146
+ readonly home: "throwaway";
147
+ readonly dir: string;
148
+ };
149
+ /**
150
+ * The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
151
+ * `base`, with the runtime's mock wiring on top. Pure — the testable seam of
152
+ * the un-coverable spawn.
153
+ */
154
+ export declare function harnessSpawnEnv(runtime: HarnessRuntime, wiredEnv: Readonly<Record<string, string>>, home: HarnessRunHome, base?: NodeJS.ProcessEnv): Readonly<Record<string, string>>;
110
155
  /** Whether the agent CLI is available — harness tests need it. */
111
156
  export declare function claudeAvailable(): boolean;
112
157
  /**
@@ -138,6 +183,15 @@ export interface RunHarnessTestOptions {
138
183
  * only — requesting confinement for another harness throws.
139
184
  */
140
185
  export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise<HarnessTestResult>;
186
+ /**
187
+ * {@link runHarnessTest} with the run's HOME chosen by the caller — see
188
+ * {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
189
+ * a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
190
+ * run must not depend on, or act as, the machine it happens to run on. The
191
+ * confined (bubblewrap) path ignores `home`: it already runs with its own HOME
192
+ * and a cleared env.
193
+ */
194
+ export declare function runHarnessTestIn(spec: HarnessTestSpec, opts: RunHarnessTestOptions, home: HarnessRunHome["home"]): Promise<HarnessTestResult>;
141
195
  /**
142
196
  * `runHarness` — the harness-scope entry of the revamped API (Phase 2 of
143
197
  * `research/testing-api-design.md`). The harness has two execution scopes, `hook`
@@ -156,4 +210,14 @@ export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessT
156
210
  export declare function runHarness(spec: HarnessTestSpec, opts?: RunHarnessTestOptions & {
157
211
  model?: "mock" | "real";
158
212
  }): Promise<HarnessTestResult>;
213
+ /**
214
+ * The files and settings a run with `outputStyle` writes — the style at its
215
+ * place, selected by the name the harness reads from it. The same fixture
216
+ * `runHarnessTest({ outputStyle })` runs, so an eval arm built from it gets
217
+ * exactly what that run proved reaches the model.
218
+ */
219
+ export declare function outputStyleFixture(outputStyle: string, adapter?: HarnessAdapter): {
220
+ readonly files: Readonly<Record<string, string>>;
221
+ readonly settings: unknown;
222
+ };
159
223
  //# sourceMappingURL=harness-test.d.ts.map
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.claudeCodeDriver = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = void 0;
3
+ exports.claudeCodeDriver = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.resolveHarness = exports.loadPlugin = exports.scriptModel = exports.reachedModel = exports.parseReplies = void 0;
4
4
  exports.parseToolCalls = parseToolCalls;
5
5
  exports.parseSubagents = parseSubagents;
6
6
  exports.parseResultEvent = parseResultEvent;
@@ -8,9 +8,12 @@ exports.parseOutput = parseOutput;
8
8
  exports.parseHooks = parseHooks;
9
9
  exports.buildClaudeArgs = buildClaudeArgs;
10
10
  exports.parseClaudeRun = parseClaudeRun;
11
+ exports.harnessSpawnEnv = harnessSpawnEnv;
11
12
  exports.claudeAvailable = claudeAvailable;
12
13
  exports.runHarnessTest = runHarnessTest;
14
+ exports.runHarnessTestIn = runHarnessTestIn;
13
15
  exports.runHarness = runHarness;
16
+ exports.outputStyleFixture = outputStyleFixture;
14
17
  /**
15
18
  * vigiles — deterministic Claude Code harness testing.
16
19
  *
@@ -49,13 +52,18 @@ const check_count_js_1 = require("./check-count.js");
49
52
  const coverage_probe_js_1 = require("./coverage-probe.js");
50
53
  const runtime_js_1 = require("./adapters/claude-code/runtime.js");
51
54
  const mock_model_js_1 = require("./mock-model.js");
52
- const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
55
+ const replies_js_1 = require("./adapters/claude-code/replies.js");
56
+ Object.defineProperty(exports, "parseReplies", { enumerable: true, get: function () { return replies_js_1.parseReplies; } });
57
+ Object.defineProperty(exports, "reachedModel", { enumerable: true, get: function () { return replies_js_1.reachedModel; } });
58
+ const adapter_registry_js_1 = require("./adapter-registry.js");
59
+ const output_style_js_1 = require("./core/output-style.js");
60
+ const run_env_js_1 = require("./core/run-env.js");
53
61
  const sandbox_js_1 = require("./sandbox.js");
54
62
  var mock_model_js_2 = require("./mock-model.js");
55
63
  Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
56
- var plugin_loader_js_2 = require("./adapters/claude-code/plugin-loader.js");
57
- Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_2.loadPlugin; } });
58
- Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_2.resolveHarness; } });
64
+ const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
65
+ Object.defineProperty(exports, "loadPlugin", { enumerable: true, get: function () { return plugin_loader_js_1.loadPlugin; } });
66
+ Object.defineProperty(exports, "resolveHarness", { enumerable: true, get: function () { return plugin_loader_js_1.resolveHarness; } });
59
67
  var sandbox_js_2 = require("./sandbox.js");
60
68
  Object.defineProperty(exports, "decideSandbox", { enumerable: true, get: function () { return sandbox_js_2.decideSandbox; } });
61
69
  Object.defineProperty(exports, "specTrusted", { enumerable: true, get: function () { return sandbox_js_2.specTrusted; } });
@@ -285,8 +293,9 @@ function buildClaudeArgs(spec, hasSettings) {
285
293
  ? ["--plugin-dir", (0, node_path_1.resolve)(spec.pluginDir)]
286
294
  : []),
287
295
  ...(hasSettings ? ["--settings", "settings.json"] : []),
288
- "--allowedTools",
289
- ...tools,
296
+ // A permission allowlist: an empty one approves nothing, so the flag is
297
+ // left out (a bare `--allowedTools` exits before any model turn).
298
+ ...(tools.length === 0 ? [] : ["--allowedTools", ...tools]),
290
299
  ];
291
300
  }
292
301
  /** Build the `claude` argv from the driver context — wraps `buildClaudeArgs`. */
@@ -305,8 +314,22 @@ function parseClaudeRun(stdout) {
305
314
  toolCalls: parseToolCalls(stdout),
306
315
  hooks: parseHooks(stdout),
307
316
  output: parseOutput(stdout),
317
+ replies: (0, replies_js_1.parseReplies)(stdout),
308
318
  };
309
319
  }
320
+ /**
321
+ * The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
322
+ * `base`, with the runtime's mock wiring on top. Pure — the testable seam of
323
+ * the un-coverable spawn.
324
+ */
325
+ function harnessSpawnEnv(runtime, wiredEnv, home, base = process.env) {
326
+ switch (home.home) {
327
+ case "inherit":
328
+ return { ...(0, run_env_js_1.withoutSessionIdentity)(base, runtime.runEnv), ...wiredEnv };
329
+ case "throwaway":
330
+ return { ...(0, run_env_js_1.scrubbedRunEnv)(base, { home: home.dir }), ...wiredEnv };
331
+ }
332
+ }
310
333
  /* v8 ignore start -- spawns the real claude CLI + filesystem; exercised by the
311
334
  claude-backed suite, excluded from the deterministic coverage gate (the parse
312
335
  helpers above carry the testable logic). */
@@ -338,13 +361,24 @@ function writeFixture(cwd, files, settings) {
338
361
  (0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), json);
339
362
  }
340
363
  }
364
+ /**
365
+ * The {@link HarnessRunHome} for a run in `cwd`. A throwaway HOME lives under
366
+ * the run's own temp dir, as the eval tier's does, so `cleanup()` removes it
367
+ * with everything else.
368
+ */
369
+ function runHomeIn(cwd, home) {
370
+ return home === "throwaway"
371
+ ? { home, dir: (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(cwd, "home-")) }
372
+ : { home };
373
+ }
341
374
  /**
342
375
  * Spawn the agent binary against the mock. The mock-wiring *args* are already in
343
376
  * `args` (the driver placed `wireMock(url).args` at the correct argv position
344
- * via `ctx.mockArgs`); here we only layer `wireMock(url).env` over the caller's
345
- * env. Driver-agnostic at the transport seam.
377
+ * via `ctx.mockArgs`); here the env is {@link harnessSpawnEnv} of the caller's,
378
+ * with `wireMock(url).env` on top. Driver-agnostic at the transport seam.
346
379
  */
347
- function spawnAgent(runtime, args, cwd, baseUrl, timeoutMs) {
380
+ function spawnAgent(runtime, args, run) {
381
+ const { cwd, baseUrl, timeoutMs, home } = run;
348
382
  const wired = runtime.wireMock(baseUrl);
349
383
  return new Promise((resolvePromise) => {
350
384
  // vigiles:free-tier — the deterministic harness tier. `runtime.wireMock`
@@ -354,9 +388,8 @@ function spawnAgent(runtime, args, cwd, baseUrl, timeoutMs) {
354
388
  const child = (0, node_child_process_1.spawn)(runtime.agentBinary, [...args], {
355
389
  cwd,
356
390
  // Any key works — the mock ignores auth. wireMock supplies the overlay
357
- // env (base-URL var + dummy key for CC; the dummy key for Codex); layer it
358
- // over the caller's env.
359
- env: { ...process.env, ...wired.env },
391
+ // env (base-URL var + dummy key for CC; the dummy key for Codex).
392
+ env: harnessSpawnEnv(runtime, wired.env, home),
360
393
  stdio: ["ignore", "pipe", "pipe"],
361
394
  });
362
395
  let stdout = "";
@@ -399,6 +432,7 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
399
432
  toolCalls: parsed.toolCalls,
400
433
  hooks: parsed.hooks,
401
434
  output: parsed.output,
435
+ ...(parsed.replies === undefined ? {} : { replies: parsed.replies }),
402
436
  modelRequests,
403
437
  subagents: parseSubagents(out.stdout),
404
438
  file: (p) => {
@@ -423,6 +457,17 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
423
457
  * only — requesting confinement for another harness throws.
424
458
  */
425
459
  async function runHarnessTest(spec, opts = {}) {
460
+ return runHarnessTestIn(spec, opts, "inherit");
461
+ }
462
+ /**
463
+ * {@link runHarnessTest} with the run's HOME chosen by the caller — see
464
+ * {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
465
+ * a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
466
+ * run must not depend on, or act as, the machine it happens to run on. The
467
+ * confined (bubblewrap) path ignores `home`: it already runs with its own HOME
468
+ * and a cleared env.
469
+ */
470
+ async function runHarnessTestIn(spec, opts, home) {
426
471
  // Tell the CLI runner this script exercised the harness, so a file that runs
427
472
  // NOTHING can be told apart from one that ran and passed. See check-count.ts.
428
473
  (0, check_count_js_1.recordCheck)();
@@ -443,12 +488,8 @@ async function runHarnessTest(spec, opts = {}) {
443
488
  if (decision.action === "sandbox" && !isClaudeCode) {
444
489
  throw new Error(`sandbox not supported for ${driver.runtime.name}: confined execution is Claude Code only. Pass sandbox: false to run ${driver.runtime.name} unconfined (you audited the code, or trust the outer container).`);
445
490
  }
491
+ const { files, settings, check } = fixtureFor(spec, opts.adapter);
446
492
  const cwd = (0, tmp_root_js_1.makeTmpDir)("harness");
447
- const { files, settings } = (0, plugin_loader_js_1.resolveHarness)({
448
- plugin: spec.plugin,
449
- settings: spec.settings,
450
- files: spec.files,
451
- });
452
493
  writeFixture(cwd, files, settings);
453
494
  const tools = spec.allowedTools ?? ["Read", "Edit", "Write", "Bash"];
454
495
  const timeoutMs = spec.timeoutMs ?? 60000;
@@ -481,14 +522,19 @@ async function runHarnessTest(spec, opts = {}) {
481
522
  const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(count, sideChannelCount);
482
523
  if (unconsumed !== undefined)
483
524
  console.error(unconsumed);
484
- return makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests);
525
+ return check(makeResult(cwd, out, parseClaudeRun(out.stdout), count, out.requests));
485
526
  }
486
527
  // Direct path: mock runs in this process; the agent reaches it over localhost.
487
528
  // Start the mock first so its URL feeds the driver's mock-wiring args.
488
529
  const mock = await driver.startMock(spec.model);
489
530
  try {
490
531
  const args = buildArgs(driver.runtime.wireMock(mock.url).args);
491
- const out = await spawnAgent(driver.runtime, args, cwd, mock.url, timeoutMs);
532
+ const out = await spawnAgent(driver.runtime, args, {
533
+ cwd,
534
+ baseUrl: mock.url,
535
+ timeoutMs,
536
+ home: runHomeIn(cwd, home),
537
+ });
492
538
  // A script that was never consumed is otherwise invisible — the run just
493
539
  // looks empty. `scriptUnconsumedWarning` names the one shape that produces
494
540
  // it (every request arriving without tool declarations, so nothing looked
@@ -496,9 +542,9 @@ async function runHarnessTest(spec, opts = {}) {
496
542
  const unconsumed = (0, mock_model_js_1.scriptUnconsumedWarning)(mock.count, mock.sideChannelCount ?? 0);
497
543
  if (unconsumed !== undefined)
498
544
  console.error(unconsumed);
499
- return makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
545
+ return check(makeResult(cwd, out, driver.parseRun(out.stdout), mock.count, [
500
546
  ...mock.requests,
501
- ]);
547
+ ]));
502
548
  }
503
549
  finally {
504
550
  await mock.close();
@@ -527,6 +573,52 @@ async function runHarness(spec, opts = {}) {
527
573
  }
528
574
  return runHarnessTest(spec, opts);
529
575
  }
576
+ /**
577
+ * The fixture to write, and the check its result must pass. Without
578
+ * `outputStyle` the check returns the result as it is.
579
+ */
580
+ function fixtureFor(spec, adapter) {
581
+ // One layout for both phases: the plugin is loaded the way the selected
582
+ // harness lays it out, the same layout the style is then planned against.
583
+ const { layout } = adapter ?? adapter_registry_js_1.defaultAdapter;
584
+ const resolved = (0, plugin_loader_js_1.resolveHarness)({ plugin: spec.plugin, settings: spec.settings, files: spec.files }, layout);
585
+ if (spec.outputStyle === undefined)
586
+ return { ...resolved, check: (result) => result };
587
+ const rules = layout.outputStyles;
588
+ if (rules === undefined)
589
+ throw new Error("outputStyle: this harness has no output styles");
590
+ const plan = (0, output_style_js_1.planStyleRun)(layout, { path: spec.outputStyle, text: readStyleFile(spec.outputStyle) }, resolved);
591
+ if (plan.kind === "refused")
592
+ throw new Error(`outputStyle: ${plan.reason}`);
593
+ return {
594
+ files: plan.files,
595
+ settings: plan.settings,
596
+ check: (result) => {
597
+ if ((0, output_style_js_1.styleReached)(rules, plan.style, result.modelRequests))
598
+ return result;
599
+ throw new Error(`outputStyle: "${plan.style.name ?? ""}" never reached the model — ` +
600
+ `no request carried it, so the harness did not load it. ` +
601
+ `Work dir kept for inspection: ${result.cwd}`);
602
+ },
603
+ };
604
+ }
605
+ /**
606
+ * The files and settings a run with `outputStyle` writes — the style at its
607
+ * place, selected by the name the harness reads from it. The same fixture
608
+ * `runHarnessTest({ outputStyle })` runs, so an eval arm built from it gets
609
+ * exactly what that run proved reaches the model.
610
+ */
611
+ function outputStyleFixture(outputStyle, adapter) {
612
+ const { files, settings } = fixtureFor({ outputStyle, model: [] }, adapter);
613
+ return { files, settings };
614
+ }
615
+ /** The style file's text, or a clear error naming the path. */
616
+ function readStyleFile(path) {
617
+ const abs = (0, node_path_1.resolve)(path);
618
+ if (!(0, node_fs_1.existsSync)(abs))
619
+ throw new Error(`outputStyle: no such file: ${path}`);
620
+ return (0, node_fs_1.readFileSync)(abs, "utf-8");
621
+ }
530
622
  /** Pull the pillar-2 driver off an adapter, asserting it supports testing. */
531
623
  async function requireDriver(adapter) {
532
624
  (0, adapter_conformance_js_1.assertHarnessTestable)(adapter);
package/dist/optimize.js CHANGED
@@ -36,10 +36,6 @@ const score_explainer_js_1 = require("./score-explainer.js");
36
36
  function actionFor(e) {
37
37
  return e.symptom === "wrong-skill-fires" ? "differentiate" : "fix";
38
38
  }
39
- function isEmptyMachine(r) {
40
- const surfaces = r.skills.length + r.agents.length + r.hooks.length + r.commands;
41
- return surfaces === 0 && !r.mcp;
42
- }
43
39
  /**
44
40
  * Turn a scan report into a prioritized optimization plan: the structural-health
45
41
  * score + a typed recommendation per deterministic finding (`likely` dead-ends
@@ -60,7 +56,7 @@ function optimize(report) {
60
56
  score,
61
57
  grade: (0, score_core_js_1.gradeFor)(score),
62
58
  recommendations,
63
- empty: isEmptyMachine(report),
59
+ empty: (0, score_core_js_1.isEmptyMachine)(report),
64
60
  };
65
61
  }
66
62
  const ACTION_LABEL = {
@@ -0,0 +1,27 @@
1
+ import type { HarnessAdapter } from "./core/adapter.js";
2
+ import type { EvalArm } from "./eval.js";
3
+ /** The style on and the style off, ready for `paid_measureArms({ arms })`. */
4
+ export interface OutputStyleArms {
5
+ /** The style file at its place in the work dir, selected in the settings. */
6
+ readonly with: EvalArm;
7
+ /** Every style switched off, including one the user set for all projects. */
8
+ readonly without: EvalArm;
9
+ }
10
+ /**
11
+ * Build the with/without arms for the style at `path` (a file in the shape the
12
+ * harness loads, e.g. a Claude Code `.md` style), for the harness the paid tier
13
+ * drives. Spends nothing: the one run it makes uses the scripted model. Throws,
14
+ * before any eval, when the harness has no output styles, cannot tell the
15
+ * style's name, or the style does not reach the model.
16
+ *
17
+ * @experimental One real-model run behind it; how arms are built may change
18
+ * once a style can be authored as a compiled spec with its checks attached.
19
+ */
20
+ export declare function experimental_outputStyleArms(path: string): Promise<OutputStyleArms>;
21
+ /**
22
+ * {@link experimental_outputStyleArms} on a given adapter. Not public: the paid
23
+ * tier drives only the default harness, so arms built for another one would be
24
+ * run by a harness they were not checked on.
25
+ */
26
+ export declare function outputStyleArmsWith(path: string, adapter: HarnessAdapter): Promise<OutputStyleArms>;
27
+ //# sourceMappingURL=output-style-arms.d.ts.map
@@ -0,0 +1,51 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.experimental_outputStyleArms = experimental_outputStyleArms;
4
+ exports.outputStyleArmsWith = outputStyleArmsWith;
5
+ /**
6
+ * The two arms of a paid output-style eval — the style on, the style off — built
7
+ * from the style FILE, after one free run has shown the style reaches the model.
8
+ *
9
+ * Three silent failures this closes. A hand-typed arm name that misses the
10
+ * style's name loads no style and raises no error, so the "with" arm is the
11
+ * "without" arm and the eval compares a run with itself. A bare "without" arm
12
+ * still loads a style the user switched on for every project, which is the same
13
+ * comparison from the other side. And the paid tier cannot check delivery at
14
+ * all: it drives the real API, so no request is captured. The free run here
15
+ * uses the scripted model, where requests ARE captured, with the same fixture
16
+ * the paid arm gets — so a style that would not load stops the eval before the
17
+ * first billed trial. That free run uses a throwaway HOME and none of the
18
+ * caller's environment beyond the OS essentials.
19
+ */
20
+ const adapter_registry_js_1 = require("./adapter-registry.js");
21
+ const harness_test_js_1 = require("./harness-test.js");
22
+ /**
23
+ * Build the with/without arms for the style at `path` (a file in the shape the
24
+ * harness loads, e.g. a Claude Code `.md` style), for the harness the paid tier
25
+ * drives. Spends nothing: the one run it makes uses the scripted model. Throws,
26
+ * before any eval, when the harness has no output styles, cannot tell the
27
+ * style's name, or the style does not reach the model.
28
+ *
29
+ * @experimental One real-model run behind it; how arms are built may change
30
+ * once a style can be authored as a compiled spec with its checks attached.
31
+ */
32
+ async function experimental_outputStyleArms(path) {
33
+ return outputStyleArmsWith(path, adapter_registry_js_1.defaultAdapter);
34
+ }
35
+ /**
36
+ * {@link experimental_outputStyleArms} on a given adapter. Not public: the paid
37
+ * tier drives only the default harness, so arms built for another one would be
38
+ * run by a harness they were not checked on.
39
+ */
40
+ async function outputStyleArmsWith(path, adapter) {
41
+ // A throwaway HOME: this run happens on every import of an eval file (even
42
+ // `vigiles eval --check`), so it must not read the machine's user config or
43
+ // run as the session that imported it. What it proves — the style reaches the
44
+ // model — is a fact about the fixture, not about this machine.
45
+ const preflight = await (0, harness_test_js_1.runHarnessTestIn)({ outputStyle: path, prompt: "hi", model: [{ text: "ok" }] }, { adapter }, "throwaway");
46
+ preflight.cleanup();
47
+ const { files, settings } = (0, harness_test_js_1.outputStyleFixture)(path, adapter);
48
+ const none = adapter.layout.outputStyles?.selectNone ?? {};
49
+ return { with: { files, settings }, without: { settings: none } };
50
+ }
51
+ //# sourceMappingURL=output-style-arms.js.map
@@ -17,12 +17,12 @@
17
17
  * boundary is deliberate: plugins "can execute arbitrary code on your machine
18
18
  * with your user privileges", so a repo is not allowed to install one for you.
19
19
  *
20
- * The declaration buys exactly one thing, and it is worth having: a collaborator
21
- * who clones the repo and never runs `vigiles init` currently gets **silence** —
22
- * the package is in `node_modules`, its six skills are unreachable, and nothing
23
- * says so. With the declaration, Claude Code tells them the project wants this
24
- * plugin and prints the command that installs it. **Silent absence becomes a
25
- * prompt.** It does not become a working install.
20
+ * The declaration buys exactly one thing: a collaborator who clones the repo is
21
+ * told the project wants this plugin, with the command that installs it.
22
+ * **Silent absence becomes a prompt.** It does not become a working install. What
23
+ * the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
24
+ * any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
25
+ * the links travel with the clone.
26
26
  *
27
27
  * Nothing is vendored: what lands in the repo is a *reference* (a marketplace
28
28
  * source plus an enabled flag). The plugin content still lives in the global
@@ -18,12 +18,12 @@
18
18
  * boundary is deliberate: plugins "can execute arbitrary code on your machine
19
19
  * with your user privileges", so a repo is not allowed to install one for you.
20
20
  *
21
- * The declaration buys exactly one thing, and it is worth having: a collaborator
22
- * who clones the repo and never runs `vigiles init` currently gets **silence** —
23
- * the package is in `node_modules`, its six skills are unreachable, and nothing
24
- * says so. With the declaration, Claude Code tells them the project wants this
25
- * plugin and prints the command that installs it. **Silent absence becomes a
26
- * prompt.** It does not become a working install.
21
+ * The declaration buys exactly one thing: a collaborator who clones the repo is
22
+ * told the project wants this plugin, with the command that installs it.
23
+ * **Silent absence becomes a prompt.** It does not become a working install. What
24
+ * the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
25
+ * any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
26
+ * the links travel with the clone.
27
27
  *
28
28
  * Nothing is vendored: what lands in the repo is a *reference* (a marketplace
29
29
  * source plus an enabled flag). The plugin content still lives in the global
@@ -539,7 +539,7 @@ function scanFiles(files, layout = layout_js_1.claudeCodeLayout, dialect = diale
539
539
  hooks,
540
540
  inlineHooks: inline,
541
541
  manualHookCount: manual,
542
- commands: Object.keys(loaded.files).filter(cls.isCommand).length,
542
+ ...(0, coverage_evidence_js_1.inventoryCounts)(loaded.files, cls.isCommand, [lay], coverage),
543
543
  // A declared server set counts even when the loader emitted no warning —
544
544
  // otherwise a plugin whose servers come from the Agent Plugins `mcp.json`
545
545
  // reports "MCP servers: no" while the report lists an MCP finding.
package/dist/scan.d.ts CHANGED
@@ -211,6 +211,12 @@ export interface ScanReport {
211
211
  */
212
212
  readonly manualHookCount: number;
213
213
  readonly commands: number;
214
+ /**
215
+ * Output styles found, exempt ones aside, or `"not-supported"` when the
216
+ * harness has none — the inventory side of `untested-output-style`. Optional
217
+ * so a hand-built report predating the field still type-checks.
218
+ */
219
+ readonly outputStyles?: number | "not-supported";
214
220
  readonly mcp: boolean;
215
221
  /**
216
222
  * Intra-plugin file references (hook scripts, skill bodies) pointing at files
package/dist/scan.js CHANGED
@@ -258,6 +258,7 @@ function scanPlugin(dir, layout, dialect, opts = {}) {
258
258
  const coverage = (0, test_coverage_js_1.findUntestedSurfaces)({
259
259
  basePath: dir,
260
260
  layout: lay,
261
+ harnessLayouts: declared.map((h) => h.layout),
261
262
  // The SAME `.vigilesrc.json#exclude`. Untested-surface discovery is a
262
263
  // second walk over the same trees, so leaving it out would have excluded a
263
264
  // skill from the GRADE while still naming it in "Untested surfaces: 1" — a
@@ -276,7 +277,7 @@ function scanPlugin(dir, layout, dialect, opts = {}) {
276
277
  hooks,
277
278
  inlineHooks: inline,
278
279
  manualHookCount: manual,
279
- commands: Object.keys(loaded.files).filter(cls.isCommand).length,
280
+ ...(0, coverage_evidence_js_1.inventoryCounts)(loaded.files, cls.isCommand, declared.map((h) => h.layout), coverage),
280
281
  // A declared server set counts even when the loader emitted no warning —
281
282
  // otherwise a plugin whose servers come from the Agent Plugins `mcp.json`
282
283
  // reports "MCP servers: no" while the report lists an MCP finding.
@@ -655,6 +656,16 @@ function instructionWeightLines(w) {
655
656
  ...tail,
656
657
  ];
657
658
  }
659
+ /** The commands and output-style lines of a scan's facts; a harness without styles says so. */
660
+ function inventoryFacts(r) {
661
+ const commands = r.commands > 0 ? [`Commands: ${String(r.commands)}`] : [];
662
+ if (r.outputStyles === "not-supported")
663
+ return [...commands, "Output styles: n/a (this harness has none)"];
664
+ const styles = r.outputStyles ?? 0;
665
+ return styles > 0
666
+ ? [...commands, `Output styles: ${String(styles)}`]
667
+ : commands;
668
+ }
658
669
  function formatScanReport(r) {
659
670
  const out = [`Scan: ${r.dir}`, ""];
660
671
  if (r.instructions) {
@@ -715,15 +726,14 @@ function formatScanReport(r) {
715
726
  out.push(...section("Ineffective hook guards (false confidence)", r.hookBlockFindings.map((h) => ` ✗ [${h.event}] ${h.scriptPath ?? "(inline)"}: ${h.message}`)));
716
727
  out.push(...section("Hook matchers that don't fire as written", r.hookMatcherFindings.map((m) => ` ✗ ${m.message}`)));
717
728
  const facts = [];
718
- if (r.commands > 0)
719
- facts.push(`Commands: ${String(r.commands)}`);
729
+ facts.push(...inventoryFacts(r));
720
730
  facts.push(`MCP servers: ${r.mcp ? "yes" : "no"}`);
721
731
  facts.push(`Untested surfaces: ${String(r.untested)}`);
722
732
  // The two tiers, named separately — a surface with a deterministic harness and
723
733
  // no eval is NOT the same position as one with neither. Shown only when the
724
734
  // producer supplied the split (a hand-built report may not have).
725
735
  if (r.untestedHarness !== undefined && r.unevaluated !== undefined) {
726
- facts.push(` no harness: ${String(r.untestedHarness)} · firing never measured: ${String(r.unevaluated)}`);
736
+ facts.push(` no harness: ${String(r.untestedHarness)} · never measured with a real model: ${String(r.unevaluated)}`);
727
737
  }
728
738
  // …and HOW the rest passed. Without this the count is unfalsifiable from the
729
739
  // outside: "28 covered" and "28 names that appear in some file" print the same.
@@ -180,6 +180,9 @@ export declare function reportDeductions(r: ScanReport): Deduction[];
180
180
  * surface of any kind", because it reads as "this plugin is broken" — a
181
181
  * different statement from "this plugin has no skills or subagents".
182
182
  *
183
+ * An output style counts too: a repo can ship only a style, and "nothing to
184
+ * audit" for it would hide the untested-output-style finding.
185
+ *
183
186
  * Hooks count in BOTH forms: `hooks[]` is the script-backed ones, `inlineHooks`
184
187
  * the shell one-liners that carry no script file to path-check. The second is
185
188
  * still a gate that runs.
@@ -332,6 +332,9 @@ function reportDeductions(r) {
332
332
  * surface of any kind", because it reads as "this plugin is broken" — a
333
333
  * different statement from "this plugin has no skills or subagents".
334
334
  *
335
+ * An output style counts too: a repo can ship only a style, and "nothing to
336
+ * audit" for it would hide the untested-output-style finding.
337
+ *
335
338
  * Hooks count in BOTH forms: `hooks[]` is the script-backed ones, `inlineHooks`
336
339
  * the shell one-liners that carry no script file to path-check. The second is
337
340
  * still a gate that runs.
@@ -341,7 +344,8 @@ function isEmptyMachine(r) {
341
344
  r.agents.length +
342
345
  r.hooks.length +
343
346
  r.inlineHooks +
344
- r.commands;
347
+ r.commands +
348
+ (typeof r.outputStyles === "number" ? r.outputStyles : 0);
345
349
  return surfaces === 0 && !r.mcp;
346
350
  }
347
351
  /**