vigiles 16.1.2 → 17.0.0

package/dist/test-coverage.js

@@ -67,6 +67,7 @@ exports.suggestedTestPath = suggestedTestPath;
 exports.coverageEvidenceCounts = coverageEvidenceCounts;
 exports.skillTestNudge = skillTestNudge;
 exports.evalTierQuestion = evalTierQuestion;
+exports.coverageCaveats = coverageCaveats;
 exports.formatUntestedReport = formatUntestedReport;
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
@@ -431,6 +432,7 @@ function findUntestedSurfaces(options = {}) {
         legacyCoversFiles: tests
             .map((t) => t.path)
             .filter((path) => read((0, node_path_1.join)(basePath, path)).includes(LEGACY_COVERS)),
+        retiredTestNames: retiredTestNamesFor(basePath, union.untested),
         decisions: union.decisions,
         harness: tierOf(considered, split.harness, runIndex, "harness"),
         evals: tierOf(considered, split.evals, runIndex, "eval"),
@@ -662,6 +664,93 @@ function staleRunNote(report) {
             `coverage. Re-run \`vigiles test\` / \`vigiles eval\` to refresh it.`,
     ];
 }
+/** Names a default vitest/jest run collects — the suffixes vigiles will not use. */
+const FOREIGN_RUNNER_SUFFIX = /\.(test|spec)\.(ts|mts|cts|js|mjs|cjs)$/;
+/**
+ * The would-be colocated tests that only their NAME disqualifies.
+ *
+ * Same two questions colocation asks — is it named after the surface, is it
+ * sitting beside it — with the third answer inverted: the suffix is one a
+ * default vitest/jest run collects, which is precisely why it left
+ * {@link DEFAULT_TEST_GLOBS}. Reading the directory (rather than globbing the
+ * repo again) keeps the cost at one `readdir` per untested surface and cannot
+ * reach a file that is not beside one.
+ */
+function retiredTestNamesFor(basePath, untested) {
+    const out = [];
+    for (const s of untested) {
+        const dir = (0, node_path_1.dirname)(s.path);
+        let entries;
+        try {
+            entries = (0, node_fs_1.readdirSync)((0, node_path_1.join)(basePath, dir));
+        }
+        catch {
+            continue; // a surface whose directory vanished mid-scan is not our finding
+        }
+        for (const entry of entries.sort()) {
+            if (!entry.startsWith(`${s.name}.`))
+                continue;
+            if (!FOREIGN_RUNNER_SUFFIX.test(entry))
+                continue;
+            out.push({
+                path: dir === "." ? entry : `${dir}/${entry}`,
+                surface: s.path,
+            });
+        }
+    }
+    return out;
+}
+/**
+ * One line for the author staring at a test they already wrote.
+ *
+ * The count says untested; the directory listing says `foo.test.mjs`. Without
+ * this the two never meet: the finding prints the SURFACE path and suggests a
+ * file to add, so the reader's most likely conclusion is that the tool cannot
+ * see their test — which is true, and the reason is a suffix nobody mentioned.
+ */
+function retiredTestNameNote(report) {
+    const found = report.retiredTestNames ?? [];
+    if (found.length === 0)
+        return [];
+    const shown = found
+        .slice(0, 3)
+        .map((f) => f.path)
+        .join(", ");
+    const more = found.length > 3 ? ` (+${String(found.length - 3)} more)` : "";
+    return [
+        `  ${String(found.length)} file(s) sit beside an untested surface and are named ` +
+            `after it, but carry a \`.test.\`/\`.spec.\` suffix (${shown}${more}) — the name a ` +
+            `default vitest/jest run collects, which is why it stopped counting in 15.x. A ` +
+            `harness test drives an agent, and under a foreign runner vigiles refuses to ` +
+            `spawn one, so that run FAILS on it rather than testing anything. Rename to ` +
+            `\`<surface>.harness.<ext>\` and it counts here without being swept up there.`,
+    ];
+}
+/**
+ * The QUALIFIERS on a coverage number — every caveat that says "this count is
+ * not quite what it looks like", in one list, built once.
+ *
+ * 🔴 THIS EXISTS BECAUSE A CAVEAT COULD BE PRINTED BY ONE COMMAND AND NOT THE
+ * OTHER, AND WAS. Both notes below were added to `formatUntestedReport` (the
+ * `lint` renderer) and neither reached `audit`, which assembles its own fact
+ * block from the same {@link UntestedReport}. Measured 2026-08-18 on a fixture
+ * whose only harness carried the retired `vigiles:covers` marker: `lint` named
+ * the file, `audit` printed `Untested surfaces: 0` and nothing else. The
+ * migration note existed, was unit-tested, and was invisible to anyone whose
+ * habit is `audit` — which is the whole point of a note that explains a silent
+ * migration.
+ *
+ * Collecting them here is the subtraction: a caveat is no longer something a
+ * renderer can choose to carry. Adding a third one reaches both callers or
+ * neither, and "neither" is a compile error rather than a quiet omission.
+ */
+function coverageCaveats(report) {
+    return [
+        ...staleRunNote(report),
+        ...legacyCoversNote(report),
+        ...retiredTestNameNote(report),
+    ];
+}
 function formatUntestedReport(report) {
     // Every coverage number is printed WITH its provenance. "28 covered" and
     // "28 covered, all of it a name appearing in a file" are different facts, and
@@ -669,7 +758,7 @@ function formatUntestedReport(report) {
     const provenance = (0, coverage_evidence_js_1.formatEvidence)(coverageEvidenceCounts(report));
     if (report.untested.length === 0) {
         const tail = report.exempt > 0 ? ` (${String(report.exempt)} exempt)` : "";
-        const legacy = [...staleRunNote(report), ...legacyCoversNote(report)];
+        const legacy = coverageCaveats(report);
         const ok = `✓ all ${String(report.total)} surface(s) have a test or eval${tail}` +
             (provenance ? `\n  ${provenance}` : "") +
             (legacy.length ? `\n${legacy.join("\n")}` : "");
@@ -700,8 +789,7 @@ function formatUntestedReport(report) {
     // What the surfaces that DID pass are resting on.
     if (provenance)
         lines.push(`  ${provenance}`);
-    lines.push(...staleRunNote(report));
-    lines.push(...legacyCoversNote(report));
+    lines.push(...coverageCaveats(report));
     // Already testing these another way (a promptfoo suite, a home-grown evals
     // file)? Point `testGlobs` at it so it counts toward coverage (issue #113) —
     // and put the file NEXT TO the surface, which is the only placement that

package/dist/test.d.ts

@@ -70,6 +70,8 @@ export { skillContract } from "./skill-contract.js";
 export type { SkillContract, SkillContractOptions } from "./skill-contract.js";
 export { compareContainment, formatContainment, } from "./trigger-containment.js";
 export type { ContainmentInput, ContainmentVerdict, } from "./trigger-containment.js";
+export { defineEval } from "./eval-define.js";
+export type { EvalDefinition, EvalDefinitionInput, EvalHooks, EvalKind, EvalMeasurements, EvalReports, SelectionMatrixSpec, } from "./eval-define.js";
 export { assertRates, assertPromptDiversity, checkPromptDiversity, checkReportToJUnit, formatCheckReport, formatEvalReport, formatTriggerRateReport, parseClaudeRun, stubSkillBody, } from "./eval.js";
 export type { EvalArm, EvalDriver, EvalSpec, EvalReport, EvalUsage, MeasureSpec, ArmsMeasureSpec, ArmReport, ArmUsage, ArmsCheckReport, CheckRate, CheckReport, MetricStat, Metrics, ModelOutputParser, ParsedModelRun, PromptDiversityIssue, PromptTriggerStat, RunContext, RunOut, SelectionTrialResult, TriggerRateReport, TriggerRateSpec, AgentRunArgs, AgentRunner, } from "./eval.js";
 //# sourceMappingURL=test.d.ts.map

package/dist/test.js

@@ -67,7 +67,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.sandboxAvailable = exports.specTrusted = exports.decideSandbox = exports.parseHooks = exports.parseOutput = exports.parseResultEvent = exports.parseSubagents = exports.parseToolCalls = exports.runHarness = exports.runHarnessTest = exports.formatGuardrailReport = exports.assertBlocksDisasters = exports.unblockedDisasters = exports.verifyGuardrail = exports.DISASTER_CATALOG = exports.cacheTokens = exports.outputTokens = exports.inputTokens = exports.tokens = exports.latency = exports.cost = exports.mcp = exports.allowed = exports.blocked = exports.subagent = exports.didNotWrite = exports.wrote = exports.turns = exports.received = exports.hookFired = exports.output = exports.skill = exports.onlyTools = exports.notTool = exports.toolWith = exports.tool = exports.assertChecks = exports.evalChecks = exports.loadHook = exports.egressRoutes = exports.fileToolEvents = exports.propertyHook = exports.decideHook = exports.parseHookOutput = exports.runHook = exports.runScript = exports.recordCheck = void 0;
-exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.formatContainment = exports.compareContainment = exports.skillContract = void 0;
+exports.stubSkillBody = exports.parseClaudeRun = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.defineEval = exports.formatContainment = exports.compareContainment = exports.skillContract = void 0;
 // --- reporting: how much did this script actually do? ---
 // `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
 // prints the same `✓` as one that ran and passed (measured 2026-08-08 on a file
@@ -175,6 +175,13 @@ Object.defineProperty(exports, "skillContract", { enumerable: true, get: functio
 var trigger_containment_js_1 = require("./trigger-containment.js");
 Object.defineProperty(exports, "compareContainment", { enumerable: true, get: function () { return trigger_containment_js_1.compareContainment; } });
 Object.defineProperty(exports, "formatContainment", { enumerable: true, get: function () { return trigger_containment_js_1.formatContainment; } });
+// --- declaring an eval (free: a description cannot spend) ---
+// `defineEval` is on the FREE barrel and not on `vigiles/eval`, and that is the
+// point rather than an oversight: after this shape an eval file needs nothing
+// that bills. It describes the run; `vigiles eval` performs it. The paid runners
+// moved out of an eval author's vocabulary entirely.
+var eval_define_js_1 = require("./eval-define.js");
+Object.defineProperty(exports, "defineEval", { enumerable: true, get: function () { return eval_define_js_1.defineEval; } });
 // --- free analysis OVER eval results (the coupling from cost #1) ---
 // These read a report that `vigiles/eval` produced. They spend nothing, so they
 // live here and carry no `paid_` prefix; their argument types are defined over

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "16.1.2",
+  "version": "17.0.0",
   "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
   "keywords": [
     "claude-code",

package/skills/test-harness/SKILL.md

@@ -207,38 +207,47 @@ score its output directly against a rubric. No on/off baseline — this is the
 "is it any good?" oracle (what promptfoo/DeepEval lead with), and the right
 default when there's nothing to compare against:
 
+An eval file **describes** its eval — it must never run one at the top level,
+because importing such a file spends real money. Write `<name>.eval.mjs`:
+
 ```ts
-import { paid_measure, paid_judged } from "vigiles/eval"; // `paid_` = these call a model
-import { skill, assertRates } from "vigiles";
-
-const report = await paid_measure({
-  pluginDir: "./",
-  task: "…a task the skill should handle…",
-  checks: [
-    skill("my-plugin:my-skill"), // it fired
-    paid_judged("the answer correctly does X and avoids Y"), // …and the output is good
-  ],
-  trials: 6,
+import { defineEval, skill, assertRates } from "vigiles";
+import { paid_judged } from "vigiles/eval"; // a Check whose default judge bills
+
+export default defineEval({
+  measure: {
+    pluginDir: "./",
+    task: "…a task the skill should handle…",
+    checks: [
+      skill("my-plugin:my-skill"), // it fired
+      paid_judged("the answer correctly does X and avoids Y"), // …and the output is good
+    ],
+    trials: 6,
+  },
+  assert: (report) => assertRates(report, { min: 0.8 }), // each check ≥ 80% of trials
 });
-assertRates(report, { min: 0.8 }); // each check passes ≥ 80% of trials
 ```
 
+Run it with `npx vigiles eval <file>` — never `node <file>`, which refuses.
+
 **Eval — relative (`paid_runEval` + `assertSignificant`)** — when the question is
 _lift over no-skill_ (regression, or proving a change isn't noise): A/B the
 change on vs off and gate on significance, not eyeballing:
 
 ```ts
-import { paid_runEval } from "vigiles/eval"; // `paid_` = a real model runs
-import { assertSignificant } from "vigiles";
-
-const report = await paid_runEval({
-  arms: { off: {}, on: { pluginDir: "./" } },
-  task: "…a task the harness change should affect…",
-  measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
-  trials: 6,
-  cache: "readwrite",
+import { defineEval, assertSignificant } from "vigiles";
+
+export default defineEval({
+  runEval: {
+    arms: { off: {}, on: { pluginDir: "./" } },
+    task: "…a task the harness change should affect…",
+    measure: (ctx) => ({ ok: /* a bare predicate over the trace */ true }),
+    trials: 6,
+    cache: "readwrite",
+  },
+  assert: (report) =>
+    assertSignificant(report, { baseline: "off", arm: "on", metric: "ok" }),
 });
-assertSignificant(report, { baseline: "off", arm: "on", metric: "ok" });
 ```
 
 ### Never hand-roll the runner — it silently eats stderr