vigiles 4.1.0 → 5.0.0

package/dist/tool-stub.d.ts

@@ -0,0 +1,35 @@
+/**
+ * A single fake binary: shadow the tool named `name` on PATH so that any
+ * invocation prints `stdout` (default `""`), writes `stderr` (default none) to
+ * fd 2, and exits with `exitCode` (default 0). Argv is ignored in the MVP.
+ */
+export interface ToolStub {
+    /** The binary name to shadow on PATH (e.g. `"gh"`, `"psql"`). */
+    readonly name: string;
+    /** Canned stdout the fake prints. Default `""`. */
+    readonly stdout?: string;
+    /** Canned stderr the fake writes to fd 2. Default: none. */
+    readonly stderr?: string;
+    /** Exit code the fake returns. Default `0`. */
+    readonly exitCode?: number;
+}
+/**
+ * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
+ * the decoded bytes with no added trailing newline and no backslash/`-n`
+ * surprises, so the fixture round-trips byte-for-byte.
+ */
+export declare function renderToolStub(stub: ToolStub): string;
+/**
+ * Write one executable POSIX shell stub per entry into `binDir` (which must
+ * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
+ * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
+ */
+export declare function writeToolStubs(binDir: string, stubs: readonly ToolStub[]): void;
+/**
+ * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
+ * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
+ * over the real binaries, and removes it when done (it lives under `parentDir`,
+ * so a `parentDir` cleanup also clears it).
+ */
+export declare function stubBinDir(stubs: readonly ToolStub[], parentDir: string): string;
+//# sourceMappingURL=tool-stub.d.ts.map

package/dist/tool-stub.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderToolStub = renderToolStub;
+exports.writeToolStubs = writeToolStubs;
+exports.stubBinDir = stubBinDir;
+/**
+ * vigiles — **tool stubs on PATH** (rung R2 of the eval coverage model).
+ *
+ * A skill/hook/agent often calls a CLI tool (`psql`, `redis-cli`, `gh`, `git`,
+ * `z3`, …) and then works with the RESULT. To test that downstream logic without
+ * a live service, you SHADOW the real binary on PATH with a fake that emits a
+ * **recorded / author-provided canned result** — a standard PATH-shim +
+ * VCR-style record/replay technique, not a novel invention. See
+ * `research/eval-coverage-and-isolation.md` (the three-rung model).
+ *
+ * This module is the REPLAY half: write a fake executable per tool that prints a
+ * canned stdout/stderr and exits a canned code. A record-from-real-tool half (run
+ * the real binary once at a known version, capture its output as the fixture) is
+ * a documented follow-on — NOT implemented here.
+ *
+ * CRITICAL: the canned outputs are author/recorded fixtures, **never**
+ * model-synthesized — a synthesized `gh`/`git` output looks plausible but
+ * diverges from the real tool/version, producing false confidence (a green test
+ * against a fiction).
+ *
+ * MVP scope: one canned result per binary; argv is IGNORED (every invocation of
+ * the stub returns the same result). Argv-matching (different output per
+ * sub-command / flags) is a follow-on.
+ */
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+/**
+ * Encode a string as a single-line base64 literal safe to embed verbatim inside a
+ * POSIX shell script.
+ *
+ * We base64-encode-and-decode the canned content (rather than interpolating it
+ * raw, or `cat`ing a sibling data file) because base64's alphabet is a strict
+ * subset of `[A-Za-z0-9+/=]` — it can never contain a quote, `$`, backtick,
+ * newline, `;`, or any other shell metacharacter, so the embedded literal is
+ * injection-proof regardless of the fixture's bytes. The script decodes it back
+ * with `base64 -d` at run time, so the original bytes round-trip exactly. This is
+ * simpler than a sibling data file (one self-contained script, nothing else to
+ * write/clean up) and strictly safer than quoting.
+ */
+function b64(s) {
+    return Buffer.from(s, "utf-8").toString("base64");
+}
+/**
+ * Render the POSIX shell script for one stub. `printf '%s'` (not `echo`) prints
+ * the decoded bytes with no added trailing newline and no backslash/`-n`
+ * surprises, so the fixture round-trips byte-for-byte.
+ */
+function renderToolStub(stub) {
+    const lines = ["#!/bin/sh"];
+    // stdout: decode the base64 literal straight to stdout. We pipe `base64 -d`'s
+    // output directly (NOT through `printf '%s' "$(...)"`) because command
+    // substitution strips trailing newlines — piping preserves the bytes exactly.
+    if (stub.stdout !== undefined && stub.stdout !== "") {
+        lines.push(`printf '%s' '${b64(stub.stdout)}' | base64 -d`);
+    }
+    // stderr: same, redirected to fd 2.
+    if (stub.stderr !== undefined && stub.stderr !== "") {
+        lines.push(`printf '%s' '${b64(stub.stderr)}' | base64 -d >&2`);
+    }
+    lines.push(`exit ${String(stub.exitCode ?? 0)}`);
+    return lines.join("\n") + "\n";
+}
+/**
+ * Write one executable POSIX shell stub per entry into `binDir` (which must
+ * already exist). Each file is named exactly `stub.name` and `chmod 0o755` so it
+ * is directly executable once `binDir` is on PATH. Pure-ish — fs only, no spawn.
+ */
+function writeToolStubs(binDir, stubs) {
+    for (const stub of stubs) {
+        const file = (0, node_path_1.join)(binDir, stub.name);
+        (0, node_fs_1.writeFileSync)(file, renderToolStub(stub), "utf-8");
+        (0, node_fs_1.chmodSync)(file, 0o755);
+    }
+}
+/**
+ * Convenience: mkdtemp a fresh bin dir under `parentDir`, write `stubs` into it,
+ * and return its absolute path. Caller PREPENDS this dir to PATH so the fakes win
+ * over the real binaries, and removes it when done (it lives under `parentDir`,
+ * so a `parentDir` cleanup also clears it).
+ */
+function stubBinDir(stubs, parentDir) {
+    (0, node_fs_1.mkdirSync)(parentDir, { recursive: true });
+    const binDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(parentDir, "vigiles-stub-bin-"));
+    writeToolStubs(binDir, stubs);
+    return binDir;
+}
+//# sourceMappingURL=tool-stub.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "4.1.0",
+  "version": "5.0.0",
   "description": "Compile .spec.ts files to instruction files (CLAUDE.md, AGENTS.md) with linter cross-referencing",
   "bin": {
     "vigiles": "dist/cli.js"
@@ -9,31 +9,20 @@
   "types": "./dist/core/spec.d.ts",
   "exports": {
     ".": "./dist/core/spec.js",
+    "./spec": "./dist/core/spec.js",
+    "./linting": "./dist/linting.js",
+    "./testing": "./dist/testing.js",
     "./unit": "./dist/unit.js",
     "./integration": "./dist/integration.js",
     "./e2e": "./dist/e2e.js",
-    "./linting": "./dist/linting.js",
-    "./testing": "./dist/testing.js",
     "./claude-code": "./dist/claude-code.js",
     "./codex": "./dist/codex.js",
     "./adapter": "./dist/adapter.js",
-    "./spec": "./dist/core/spec.js",
-    "./compile": "./dist/core/compile.js",
-    "./linters": "./dist/core/linters.js",
-    "./eval": "./dist/eval.js",
-    "./harness-test": "./dist/harness-test.js",
-    "./harness-assert": "./dist/harness-assert.js",
-    "./run-hook": "./dist/run-hook.js",
-    "./plugin-loader": "./dist/adapters/claude-code/plugin-loader.js",
-    "./mcp": "./dist/core/mcp.js",
-    "./judge": "./dist/judge.js",
-    "./mock-model": "./dist/mock-model.js",
     "./vitest": {
       "types": "./dist/vitest.d.mts",
       "default": "./dist/vitest.mjs"
     },
-    "./jest": "./dist/jest.js",
-    "./check": "./dist/check.js"
+    "./jest": "./dist/jest.js"
   },
   "files": [
     "dist/**/*.js",
@@ -67,10 +56,15 @@
     "test:vitest": "npm run build && vitest run --project runners",
     "test:jest": "npm run build && jest",
     "test:types": "npm run build && tsc --noEmit -p test/types/tsconfig.json",
-    "demo:plugin": "npm run build && node examples/plugin-test-demo.mjs"
+    "demo:plugin": "npm run build && node examples/plugin-test-demo.mjs",
+    "api:report": "npm run build && node scripts/api-extractor.mjs --local",
+    "api:check": "npm run build && node scripts/api-extractor.mjs",
+    "docs:api": "npm run api:report && api-documenter markdown -i temp -o api-reference"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
+    "@microsoft/api-documenter": "^7.30.7",
+    "@microsoft/api-extractor": "^7.58.9",
     "@types/js-yaml": "^4.0.9",
     "@types/minimatch": "^5.1.2",
     "@types/node": "^20.19.39",

package/skills/test-harness/SKILL.md

@@ -22,7 +22,8 @@ Match what you're testing to the cheapest tier that can answer it:
 | "Is the hook actually **wired into** the assembled plugin and does it fire in a real session?"                                         | **Deterministic** | free, no API key (real `claude` + scripted mock) | `runHarnessTest` + `scriptModel`                                                              |
 | "Did the injected context (a SessionStart hook, a `/command`) actually **reach the model**?"                                           | **Deterministic** | free, no API key                                 | `runHarnessTest` → `trace.modelRequests` / `assertRequestContains`                            |
 | "Does this skill's **description trigger** when it should (recall) **and stay quiet** when it shouldn't (precision)?"                  | **Eval**          | **paid** (real model)                            | `measureTriggerRate` (+ `irrelevantPrompts`) → `assertTriggerRate({ min, maxFalsePositive })` |
-| "Does this harness change **move what the agent does**?" (A/B, signal vs noise)                                                        | **Eval**          | **paid** (real model)                            | `runEval` + `assertSignificant`                                                               |
+| "Is this exact skill's **output any good**?" — absolute quality, no on/off baseline (the default for testing one skill)                | **Eval**          | **paid** (real model)                            | `measure({ checks: [judged(rubric)] })` → `assertRates({ min })`                              |
+| "Does this harness change **move what the agent does**, _relative_ to off?" — A/B lift, regression, signal vs noise                    | **Eval**          | **paid** (real model)                            | `runEval` (arms) + `assertSignificant`                                                        |
 
 Most harness questions — block/allow, wired-in, context-landed — never need a
 model. Only "does the model trigger / behave differently" needs the eval tier.
@@ -30,6 +31,37 @@ model. Only "does the model trigger / behave differently" needs the eval tier.
 If the unit and deterministic tiers can both answer it, **prefer unit**: it's
 faster and reaches events the deterministic mock can't drive.
 
+## Step 0.5 — Set honest expectations (what's testable, and at what cost)
+
+Be explicit with the user about which bucket each surface falls into — never let
+"we'll test it" hide whether that's free, sub-priced, or needs a container. Every
+surface sorts into one of three buckets:
+
+- **A — Free & deterministic** (no model, runs in CI on every commit): a hook's
+  block/allow decision (`runHook`), a tool-contract / "did NOT call the forbidden
+  tool" check, structural facts (`vigiles scan`), and **record-replay** of any tool
+  a skill shells out to (record the real result once, replay it via a PATH stub).
+- **B — Model-gated, on your subscription** (real model, **no metered API**): does a
+  skill's description **fire** (`measureTriggerRate`, recall + precision) **and**
+  does its guidance actually **produce good output** (score it directly:
+  `measure({ checks: [judged(rubric)] })` + `assertRates` — the absolute oracle;
+  use a `runEval` A/B on-vs-off only when you need the _relative_ lift). This is
+  the half a **prose / guidance skill** lives in —
+  its worth is behavioral, so only a model can judge it. That is **not** "uncovered"
+  and **not** free: it's fully testable on the sub. State it that way.
+- **C — Needs a real service** (a real browser / DB / redis / a11y runtime): vigiles
+  **composes with a container** here; it does not fake real semantics. Name the
+  service and hand off — don't pretend a cheap tier substitutes for it.
+
+So a prose-skill library is roughly **~100% testable (some free, most on your sub),
+~0% needs-a-container** — not "poorly covered." An accessibility/browser plugin is
+the worst case, with a large bucket C. When you report coverage, give **two
+numbers**: "% testable at all (free + sub)" vs "% that needs a container", and say
+which surfaces are free vs sub-priced. The model-gated half is the **point** of the
+eval pillar (affordable on the sub), not a gap — and testing a prose skill's
+_behavior_ requires a real model for **everyone** (promptfoo, the SDKs, all of it);
+vigiles just does it on your subscription instead of metered API.
+
 ## Step 1 — Ensure vigiles is installed
 
 Check whether `vigiles` is a dependency (`package.json`), and install it as a
@@ -103,8 +135,29 @@ assertHookFired(r, "SessionStart");
 assertRequestContains(r, "expected injected text"); // did it actually land?
 ```
 
-**Eval (`runEval`)** — A/B the change on vs off across real-model trials, then
-gate on significance, not eyeballing:
+**Eval — absolute (`measure` + `judged`)** — testing _one_ skill, the usual case:
+score its output directly against a rubric. No on/off baseline — this is the
+"is it any good?" oracle (what promptfoo/DeepEval lead with), and the right
+default when there's nothing to compare against:
+
+```ts
+import { measure, judged, skill, assertRates } from "vigiles/testing";
+
+const report = await measure({
+  pluginDir: "./",
+  task: "…a task the skill should handle…",
+  checks: [
+    skill("my-plugin:my-skill"), // it fired
+    judged("the answer correctly does X and avoids Y"), // …and the output is good
+  ],
+  trials: 6,
+});
+assertRates(report, { min: 0.8 }); // each check passes ≥ 80% of trials
+```
+
+**Eval — relative (`runEval` + `assertSignificant`)** — when the question is
+_lift over no-skill_ (regression, or proving a change isn't noise): A/B the
+change on vs off and gate on significance, not eyeballing:
 
 ```ts
 import { runEval, assertSignificant } from "vigiles/testing";