vigiles 14.13.5 → 15.0.1

package/README.md

@@ -130,13 +130,14 @@ It's free and open-source, runs entirely on your machine, and never bills per to
   <img src="vigiles-audit.png" width="760" alt="vigiles audit report for my-plugin: a verdict header reading 'Two one-line fixes away from a B.' next to a C (77/100) grade, a five-category strip (Truthfulness, Triggering, Structure, Safety, Tested), ranked fix cards with '+N pts' impact badges, broken-reference findings — and, lower down and badged experimental, a 'Your rules → enforced' preview mapping a prose rule the config silently turns off" />
 </p>
 
-**Like Google's Lighthouse, but for your agent harness.** One command grades it A–F across five categories, leads with a plain-English verdict — _"two one-line fixes away from a B"_ — and ranks every fix by the points it buys back:
+**Like Google's Lighthouse, but for your agent harness.** One command grades it A–F across six categories, leads with a plain-English verdict — _"two one-line fixes away from a B"_ — and ranks every fix by the points it buys back:
 
 - **Truthfulness** — do the references resolve?
 - **Triggering** — do skills fire, without colliding?
 - **Structure** — are tool contracts and configs valid?
 - **Safety** — any way for the agent to leak your data?
-- **Tested** — does the harness ship tests?
+- **Tested** — does the harness ship deterministic tests?
+- **Evaluated** — has anything measured whether your skills actually _fire_? (Distinct from `0`: if nothing asked, it says **not measured**.)
 
 And it closes the loop from prose to enforcement: **your rules → enforced** maps each rule you wrote to the lint rule that actually enforces it — already on, one config line away, or silently turned off (below).
 
@@ -208,8 +209,10 @@ Or run it yourself:
 
 ```bash
 npx vigiles init   # sets up the typed spec for structural rules (non-destructive — eject reverses), adds CI,
-                   # installs vigiles's skills + hooks as a Claude Code plugin (in
-                   # ~/.claude/, not your repo). On Codex, skills install globally too.
+                   # installs vigiles's skills + hooks as a Claude Code plugin. The plugin
+                   # CONTENT goes to ~/.claude/, never your repo; init also commits a two-key
+                   # reference to it in .claude/settings.json so teammates get prompted to
+                   # install it rather than silently missing it. Codex: skills install globally.
 ```
 
 **Already have a harness, or a non-JS repo?** `npx vigiles init --ci-only` sets up just the CI integrity gate — nothing installed, zero conflict. **[When to use gate vs full →](docs/agent-setup.md#non-interactive-setup-agents--ci)**
@@ -230,6 +233,7 @@ The **hooks** keep it honest in-loop — nudging the agent to tag a linter-rule
 - **Both lint and test** by default; scope with `--lint` / `--test`.
 - **Already have a CLAUDE.md / AGENTS.md, skills, or subagents? `audit` and `lint` read them as-is** — nothing is moved or rewritten. For the structural rules that want a typed spec, `init` sets one up **non-destructively** (`eject` undoes it).
 - Adds `vigiles` to `devDependencies`; installs the Claude Code plugin (skills + hooks) via the marketplace — globally, never vendored.
+- Declares that plugin in your repo's `.claude/settings.json` (`extraKnownMarketplaces` + `enabledPlugins`, merged into whatever is already there). This is a **reference, not content** — nothing is vendored, and it does **not** install the plugin for a teammate: an external-source plugin declared project-level [doesn't load until each person installs it](https://code.claude.com/docs/en/discover-plugins#configure-team-marketplaces). What it buys is that Claude Code **prompts** them with the install command, instead of a fresh clone silently having the npm package and none of its skills.
 - Wires CI as a `zernie/vigiles@v1` workflow (needs only read + PR-comment permissions) that posts a sticky PR comment + a `valid` output.
 
 Targets Claude Code and Codex out of the box, or [your own harness](docs/authoring-an-adapter.md). Prefer to write tests yourself? JS **or** TS (`*.harness.{mjs,ts}`) — run with `npx vigiles test`.

package/dist/adapters/claude-code/run-scripts.d.ts

@@ -1,8 +1,21 @@
-export type ScriptStatus = "pass" | "skip" | "fail";
+/**
+ * The outcome of running one script.
+ *
+ * `"vacuous"` — the script exited 0 and reported that it made ZERO checks. It
+ * neither passed nor failed: nothing was verified. See {@link statusFor}.
+ */
+export type ScriptStatus = "pass" | "skip" | "fail" | "vacuous";
 export interface ScriptRunResult {
     readonly file: string;
     readonly code: number;
     readonly status: ScriptStatus;
+    /**
+     * How many checks the script reported making, or `undefined` when it reported
+     * nothing at all — a script that never imports `vigiles/testing` has no way to
+     * report, and that silence is NOT a claim about it. `0` is a claim: the script
+     * loaded the library and used none of it.
+     */
+    readonly checks?: number;
 }
 /**
  * Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
@@ -11,6 +24,32 @@ export interface ScriptRunResult {
  * skip never fails the run. Scripts call `skip()` (vigiles/testing) to emit it.
  */
 export declare const SKIP_EXIT_CODE = 77;
+/**
+ * Classify one script's run from its exit code and its reported check count.
+ *
+ * 🔴 THE FOURTH STATE, AND WHY. Exit codes answer "did it fail?", never "did it
+ * do anything?". Measured 2026-08-08: a file whose whole body is
+ * `export default { "never runs": () => assert.equal(1, 2) }` imports fine,
+ * exits 0, and printed `✓ … 1 passed` — a false assertion, never called,
+ * reported as a pass. A consumer repo hit exactly that and now hand-copies a
+ * warning into every new harness header, because the runner could not enforce
+ * it: eight harnesses resting on a comment.
+ *
+ * So a run that ends clean having recorded ZERO checks is `"vacuous"` — its own
+ * visible state, not folded into `passed`, the same way a skip is not.
+ *
+ * NOT A FAILURE, deliberately. Harnesses in the wild predate the counter, and a
+ * tool that turned CI red on the release that taught it a new word would be
+ * punishing people for upgrading. It is loud and it is not fatal.
+ *
+ * AND SILENCE IS NOT ZERO. `checks === undefined` means the script never
+ * reported — it may not import `vigiles/testing` at all — so it stays a plain
+ * `pass`, exactly as before. Only a script that loaded the library and used
+ * none of it says zero. This is the `undefined`-vs-`[]` distinction
+ * `assertNoWrite` already draws: "nobody looked" must not read as "nothing
+ * happened".
+ */
+export declare function statusFor(code: number, checks: number | undefined): ScriptStatus;
 /** Filename extensions accepted for harness/eval scripts (JS and TS). */
 export declare const SCRIPT_EXTS: readonly ["mjs", "cjs", "js", "mts", "cts", "ts"];
 /** Glob suffix matching every accepted script extension, e.g. `harness`. */
@@ -41,10 +80,20 @@ export declare function discoverScripts(patterns: readonly string[], defaultGlob
 /**
  * Run each script as `node <file>`, inheriting stdio so the script's own report
  * streams to the console. `env` is merged over `process.env` for every child
- * (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes.
+ * (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
+ *
+ * Each child is handed its OWN scratch path in `VIGILES_CHECK_COUNT_ENV`, which
+ * `vigiles/testing` writes its check count to on exit — the channel that makes
+ * "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
+ * has to be a file: stdio is inherited so the script's report streams live,
+ * which leaves no stream to parse.
  */
 export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv): ScriptRunResult[];
-/** Whether any script FAILED (a skip is not a failure). */
+/**
+ * Whether any script FAILED. Neither a skip nor a vacuous run counts: the first
+ * declined to run, the second ran and verified nothing, and neither is evidence
+ * that anything is broken. Both are visible in the summary instead.
+ */
 export declare function anyFailed(results: readonly ScriptRunResult[]): boolean;
 /**
  * What a `test`/`eval` invocation should do about actually RUNNING the discovered
@@ -87,7 +136,8 @@ export interface RunScriptsEnv {
  * CLI.
  */
 export declare function decideRunScripts(o: RunScriptsEnv): RunScriptsDecision;
-/** One line per file + an explicit pass/skip/fail tally. Skips are SHOWN, never
- * folded into "passed" — a `⊘ SKIPPED` is loud, not a silent green. */
+/** One line per file + an explicit pass/skip/vacuous/fail tally. Skips and
+ * vacuous runs are SHOWN, never folded into "passed" — a `⊘ SKIPPED` is loud,
+ * not a silent green, and so is a file that verified nothing. */
 export declare function formatScriptSummary(results: readonly ScriptRunResult[]): string;
 //# sourceMappingURL=run-scripts.d.ts.map

package/dist/adapters/claude-code/run-scripts.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SCRIPT_EXTS = exports.SKIP_EXIT_CODE = void 0;
+exports.statusFor = statusFor;
 exports.scriptGlob = scriptGlob;
 exports.interpreterArgs = interpreterArgs;
 exports.detectNodeCaps = detectNodeCaps;
@@ -24,7 +25,9 @@ exports.formatScriptSummary = formatScriptSummary;
 const node_child_process_1 = require("node:child_process");
 const node_path_1 = require("node:path");
 const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
 const glob_1 = require("glob");
+const check_count_js_1 = require("../../check-count.js");
 /**
  * Exit code a harness/eval script uses to report itself SKIPPED (e.g. the
  * deterministic tier when `claude` isn't installed) — the autotools convention.
@@ -32,12 +35,37 @@ const glob_1 = require("glob");
  * skip never fails the run. Scripts call `skip()` (vigiles/testing) to emit it.
  */
 exports.SKIP_EXIT_CODE = 77;
-function statusForCode(code) {
-    if (code === 0)
-        return "pass";
+/**
+ * Classify one script's run from its exit code and its reported check count.
+ *
+ * 🔴 THE FOURTH STATE, AND WHY. Exit codes answer "did it fail?", never "did it
+ * do anything?". Measured 2026-08-08: a file whose whole body is
+ * `export default { "never runs": () => assert.equal(1, 2) }` imports fine,
+ * exits 0, and printed `✓ … 1 passed` — a false assertion, never called,
+ * reported as a pass. A consumer repo hit exactly that and now hand-copies a
+ * warning into every new harness header, because the runner could not enforce
+ * it: eight harnesses resting on a comment.
+ *
+ * So a run that ends clean having recorded ZERO checks is `"vacuous"` — its own
+ * visible state, not folded into `passed`, the same way a skip is not.
+ *
+ * NOT A FAILURE, deliberately. Harnesses in the wild predate the counter, and a
+ * tool that turned CI red on the release that taught it a new word would be
+ * punishing people for upgrading. It is loud and it is not fatal.
+ *
+ * AND SILENCE IS NOT ZERO. `checks === undefined` means the script never
+ * reported — it may not import `vigiles/testing` at all — so it stays a plain
+ * `pass`, exactly as before. Only a script that loaded the library and used
+ * none of it says zero. This is the `undefined`-vs-`[]` distinction
+ * `assertNoWrite` already draws: "nobody looked" must not read as "nothing
+ * happened".
+ */
+function statusFor(code, checks) {
     if (code === exports.SKIP_EXIT_CODE)
         return "skip";
-    return "fail";
+    if (code !== 0)
+        return "fail";
+    return checks === 0 ? "vacuous" : "pass";
 }
 /** Filename extensions accepted for harness/eval scripts (JS and TS). */
 exports.SCRIPT_EXTS = ["mjs", "cjs", "js", "mts", "cts", "ts"];
@@ -104,35 +132,73 @@ function discoverScripts(patterns, defaultGlob, cwd) {
     }
     return [...found].sort();
 }
+/**
+ * The count a script left behind, or `undefined` if it left none (it never
+ * imported `vigiles/testing`, or died before its exit handler). Anything that
+ * isn't a non-negative integer is treated as no report — a corrupt scratch file
+ * must not invent a verdict.
+ */
+function readCheckCount(path) {
+    if (!(0, node_fs_1.existsSync)(path))
+        return undefined;
+    let raw;
+    try {
+        raw = (0, node_fs_1.readFileSync)(path, "utf8").trim();
+    }
+    catch {
+        return undefined;
+    }
+    if (!/^\d+$/.test(raw))
+        return undefined;
+    return Number(raw);
+}
 /**
  * Run each script as `node <file>`, inheriting stdio so the script's own report
  * streams to the console. `env` is merged over `process.env` for every child
- * (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes.
+ * (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts.
+ *
+ * Each child is handed its OWN scratch path in `VIGILES_CHECK_COUNT_ENV`, which
+ * `vigiles/testing` writes its check count to on exit — the channel that makes
+ * "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It
+ * has to be a file: stdio is inherited so the script's report streams live,
+ * which leaves no stream to parse.
  */
 function runScripts(files, cwd, env = {}) {
     const caps = detectNodeCaps(cwd);
     const results = [];
-    for (const file of files) {
-        let argv;
-        try {
-            argv = interpreterArgs(file, caps);
-        }
-        catch (e) {
-            console.error(`✗ ${file}: ${e.message}`);
-            results.push({ file, code: 1, status: "fail" });
-            continue;
-        }
-        const res = (0, node_child_process_1.spawnSync)("node", argv, {
-            cwd,
-            stdio: "inherit",
-            env: { ...process.env, ...env },
+    const countDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "vigiles-checks-"));
+    try {
+        files.forEach((file, i) => {
+            let argv;
+            try {
+                argv = interpreterArgs(file, caps);
+            }
+            catch (e) {
+                console.error(`✗ ${file}: ${e.message}`);
+                results.push({ file, code: 1, status: "fail" });
+                return;
+            }
+            const countFile = (0, node_path_1.join)(countDir, `${String(i)}.count`);
+            const res = (0, node_child_process_1.spawnSync)("node", argv, {
+                cwd,
+                stdio: "inherit",
+                env: { ...process.env, ...env, [check_count_js_1.CHECK_COUNT_ENV]: countFile },
+            });
+            const code = res.status ?? 1;
+            const checks = readCheckCount(countFile);
+            results.push({ file, code, status: statusFor(code, checks), checks });
         });
-        const code = res.status ?? 1;
-        results.push({ file, code, status: statusForCode(code) });
+    }
+    finally {
+        (0, node_fs_1.rmSync)(countDir, { recursive: true, force: true });
     }
     return results;
 }
-/** Whether any script FAILED (a skip is not a failure). */
+/**
+ * Whether any script FAILED. Neither a skip nor a vacuous run counts: the first
+ * declined to run, the second ran and verified nothing, and neither is evidence
+ * that anything is broken. Both are visible in the summary instead.
+ */
 function anyFailed(results) {
     return results.some((r) => r.status === "fail");
 }
@@ -165,24 +231,39 @@ const MARK = {
     pass: "✓",
     skip: "⊘",
     fail: "✗",
+    vacuous: "∅",
 };
-/** One line per file + an explicit pass/skip/fail tally. Skips are SHOWN, never
- * folded into "passed" — a `⊘ SKIPPED` is loud, not a silent green. */
+/** One line per file + an explicit pass/skip/vacuous/fail tally. Skips and
+ * vacuous runs are SHOWN, never folded into "passed" — a `⊘ SKIPPED` is loud,
+ * not a silent green, and so is a file that verified nothing. */
 function formatScriptSummary(results) {
     const lines = results.map((r) => {
         if (r.status === "skip")
             return `  ⊘ ${r.file} — SKIPPED`;
         if (r.status === "fail")
             return `  ✗ ${r.file} (exit ${String(r.code)})`;
+        if (r.status === "vacuous") {
+            return `  ${MARK.vacuous} ${r.file} — 0 CHECKS (it ran clean and verified nothing)`;
+        }
         return `  ${MARK.pass} ${r.file}`;
     });
     const n = (s) => results.filter((r) => r.status === s).length;
     const parts = [`${String(n("pass"))} passed`];
     if (n("skip") > 0)
         parts.push(`${String(n("skip"))} skipped`);
+    if (n("vacuous") > 0)
+        parts.push(`${String(n("vacuous"))} with 0 checks`);
     if (n("fail") > 0)
         parts.push(`${String(n("fail"))} failed`);
     lines.push(`\n${parts.join(", ")}.`);
+    // Name the remedy where it's read, once — the usual cause is a file that
+    // DEFINES tests and never calls them, and the usual second cause is a harness
+    // asserting some other way, which the runner cannot see.
+    if (n("vacuous") > 0) {
+        lines.push(`  ∅ = the file loaded vigiles/testing and used none of it. Either nothing ran ` +
+            `(an exported test object nobody calls), or it asserts another way — in which ` +
+            `case call recordCheck() from vigiles/testing so those count.`);
+    }
     return lines.join("\n");
 }
 //# sourceMappingURL=run-scripts.js.map

package/dist/audit-report.d.ts

@@ -11,7 +11,7 @@
  * timestamp is attached by the CLI at write time, never by this pure builder, so
  * the embedded-in-HTML form stays deterministic).
  */
-import { type AuditScore } from "./audit-score.js";
+import { type AuditScore, type AuditScoreOptions } from "./audit-score.js";
 import { type Recommendation } from "./optimize.js";
 import { type Verdict } from "./audit-verdict.js";
 import type { LedgerSummary } from "./observe.js";
@@ -20,6 +20,7 @@ import type { ScanReport, MarketplaceInfo } from "./scan.js";
 import type { PluginScore } from "./score-core.js";
 import type { RuleInventoryItem } from "./rule-inventory.js";
 import type { RuleRouting } from "./rule-routing.js";
+import type { EvidenceCounts } from "./coverage-evidence.js";
 /**
  * The current schema version. Bump only on a BREAKING change to the shape.
  * v2 (2026-07-15): the rule-map `mechanism` enum value for an unrouted rule
@@ -55,7 +56,26 @@ export interface AuditInventory {
     readonly hooks: number;
     readonly commands: number;
     readonly mcp: boolean;
+    /** Surfaces covered by NEITHER tier — the union count (unchanged). */
     readonly untested: number;
+    /**
+     * The two tiers, carried SEPARATELY so a consumer can tell "has deterministic
+     * coverage, no evals" from "has neither" — a distinction the single `untested`
+     * count erased. `untestedHarness` is free-and-every-push work; `unevaluated` is
+     * paid real-model work. Additive/optional — schema version unchanged.
+     */
+    readonly untestedHarness?: number;
+    readonly unevaluated?: number;
+    /**
+     * HOW the covered surfaces were decided to be covered — `declared` (an explicit
+     * `vigiles:covers` marker), `colocated` (a test placed at the surface), or
+     * `mention` (the surface's path/namespace appears in a test's code). Carried in
+     * the product boundary because a coverage count without its derivation is not
+     * auditable: a repo whose coverage is entirely `mention` looks, in a bare
+     * number, exactly like one with real tests. Additive/optional — schema version
+     * unchanged.
+     */
+    readonly coverageEvidence?: EvidenceCounts;
 }
 /**
  * A surface (skill / subagent / instruction file) that EXISTS but doesn't yet
@@ -86,7 +106,7 @@ export interface Adoptable {
  */
 export interface AuditReport {
     readonly meta: AuditReportMeta;
-    /** The five deterministic category rings + the weighted overall + grade. */
+    /** The six category rings + the weighted overall + grade. */
     readonly score: AuditScore;
     /**
      * The one-line verdict + per-recommendation `pointsIfFixed`, both derived by
@@ -147,7 +167,7 @@ export interface AuditReport {
      */
     readonly ruleRouting?: RuleRouting;
 }
-export interface BuildAuditReportOptions {
+export interface BuildAuditReportOptions extends AuditScoreOptions {
     readonly harness: string;
     readonly vigilesVersion: string;
     /** The flight-recorder summary from the local ledger (omit when empty). */

package/dist/audit-report.js

@@ -53,7 +53,7 @@ function buildAdoptable(surfaces) {
  */
 function buildAuditReport(report, opts) {
     const adoptable = buildAdoptable(opts.adoptableSurfaces);
-    const score = (0, audit_score_js_1.auditScore)(report);
+    const score = (0, audit_score_js_1.auditScore)(report, { firingMeasured: opts.firingMeasured });
     const recommendations = (0, optimize_js_1.optimize)(report).recommendations;
     const verdict = (0, audit_verdict_js_1.computeVerdict)({ report, score, recommendations });
     return {
@@ -78,6 +78,15 @@ function buildAuditReport(report, opts) {
             commands: report.commands,
             mcp: report.mcp,
             untested: report.untested,
+            ...(report.untestedHarness !== undefined
+                ? { untestedHarness: report.untestedHarness }
+                : {}),
+            ...(report.unevaluated !== undefined
+                ? { unevaluated: report.unevaluated }
+                : {}),
+            ...(report.coverageEvidence
+                ? { coverageEvidence: report.coverageEvidence }
+                : {}),
         },
         ...(report.danglingRefs.length
             ? { brokenReferences: report.danglingRefs }