vigiles 4.0.0 → 4.0.2

package/dist/core/spec.js

@@ -93,7 +93,7 @@ function cmd(command) {
  * Reference a symbol defined in a file — verified at compile time that the
  * named file exists AND defines the named symbol (via ast-grep, cross-language).
  * Compiles to the file-qualified inline form `` `file#symbol` `` so the markdown
- * `audit` / `refs-hook` re-verify the same reference.
+ * `lint` / `refs-hook` re-verify the same reference.
  */
 function symbol(file, name) {
     return { _ref: "symbol", file: file, symbol: name };

package/dist/core/types.d.ts

@@ -97,7 +97,7 @@ export interface RulesConfig {
     "untested-surface"?: RuleWithOptions<TestCoverageConfig>;
     /**
      * Nudge (or block) when an instruction file has code-shaped references that
-     * aren't expressed as vigiles marks (so the audit can't verify them), or a
+     * aren't expressed as vigiles marks (so the lint can't verify them), or a
      * `vigiles:symbol` mark that points at a missing symbol. Drives the
      * PostToolUse refs-hook: "warn" (default) → a non-blocking nudge, "error" →
      * block the edit, false → off.

package/dist/core/validate.js

@@ -157,9 +157,9 @@ function validate(content, { ruleMarkers, rules: rulesConfig, filePath, dialect
             const specPath = filePath + ".spec.ts";
             // Inline mode counts as a spec — any parseable
             // `<!-- vigiles:enforce ... -->` comment means the file is
-            // verified on `vigiles audit` even without a .spec.ts sibling.
+            // verified on `vigiles lint` even without a .spec.ts sibling.
             // Delegate to the real parser so a malformed marker can't
-            // satisfy require-spec with a rule that audit can't verify.
+            // satisfy require-spec with a rule that lint can't verify.
             const hasInline = (0, inline_js_1.hasInlineRules)(content) || (0, frontmatter_js_1.hasFrontmatterRules)(content);
             if (!(0, node_fs_1.existsSync)(specPath) && !hasInline) {
                 const msg = {

package/dist/e2e.d.ts

@@ -1,19 +1,16 @@
 /**
- * `vigiles/e2e` — the **deterministic** end-to-end tier (real sandbox + real
- * network, but a definite pass/fail).
+ * `vigiles/e2e` — DEPRECATED back-compat alias for [`vigiles/integration`](./integration.ts).
  *
- * Re-exports everything in [`vigiles/integration`](./integration.ts) and adds the
- * real-egress capability: `egressRoutes()` (probe whether allowlisted egress can
- * actually route) and — by re-export — `runHook` used with `egress: { allow }`
- * (allowlisted real outbound). Capability contract: needs a **routable rootless
- * sandbox + real network**, and each test self-skips via `egressRoutes()` where
- * that's unavailable. Still a **verification** tier — you assert pass/fail.
+ * There is no separate "e2e" tier: real **egress** is a *capability* of the
+ * harness/integration scope (`egressRoutes()` + `runHook`'s `egress: { allow }`),
+ * not a different kind of test — the old `e2e` barrel added exactly one symbol
+ * over `integration`, which is the definition of a non-tier. It now lives on
+ * `vigiles/integration`; this entry re-exports it unchanged so existing imports
+ * keep working. See `research/testing-api-design.md` Part 4 (two scopes, not
+ * four tiers). Prefer `vigiles/integration`.
  *
- * NOT here: **evals** (`runEval` / `measureTriggerRate` / `judge`). Those are a
- * different axis — **non-deterministic measurement** (real model, mean ± se), run
- * via `vigiles eval` on `*.eval.mjs`. Import them from
- * [`vigiles/eval`](./eval.ts) + [`vigiles/judge`](./judge.ts), not from here.
+ * NOT here: **evals** (`runEval` / `measure` / `measureTriggerRate` / `judge`) —
+ * those are non-deterministic measurement (`vigiles/eval`), a different axis.
  */
 export * from "./integration.js";
-export { egressRoutes } from "./adapters/claude-code/run-hook.js";
 //# sourceMappingURL=e2e.d.ts.map

package/dist/e2e.js

@@ -14,26 +14,19 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.egressRoutes = void 0;
 /**
- * `vigiles/e2e` — the **deterministic** end-to-end tier (real sandbox + real
- * network, but a definite pass/fail).
+ * `vigiles/e2e` — DEPRECATED back-compat alias for [`vigiles/integration`](./integration.ts).
  *
- * Re-exports everything in [`vigiles/integration`](./integration.ts) and adds the
- * real-egress capability: `egressRoutes()` (probe whether allowlisted egress can
- * actually route) and — by re-export — `runHook` used with `egress: { allow }`
- * (allowlisted real outbound). Capability contract: needs a **routable rootless
- * sandbox + real network**, and each test self-skips via `egressRoutes()` where
- * that's unavailable. Still a **verification** tier — you assert pass/fail.
+ * There is no separate "e2e" tier: real **egress** is a *capability* of the
+ * harness/integration scope (`egressRoutes()` + `runHook`'s `egress: { allow }`),
+ * not a different kind of test — the old `e2e` barrel added exactly one symbol
+ * over `integration`, which is the definition of a non-tier. It now lives on
+ * `vigiles/integration`; this entry re-exports it unchanged so existing imports
+ * keep working. See `research/testing-api-design.md` Part 4 (two scopes, not
+ * four tiers). Prefer `vigiles/integration`.
  *
- * NOT here: **evals** (`runEval` / `measureTriggerRate` / `judge`). Those are a
- * different axis — **non-deterministic measurement** (real model, mean ± se), run
- * via `vigiles eval` on `*.eval.mjs`. Import them from
- * [`vigiles/eval`](./eval.ts) + [`vigiles/judge`](./judge.ts), not from here.
+ * NOT here: **evals** (`runEval` / `measure` / `measureTriggerRate` / `judge`) —
+ * those are non-deterministic measurement (`vigiles/eval`), a different axis.
  */
 __exportStar(require("./integration.js"), exports);
-// The real-egress capability probe (the egress-using runHook is already re-exported
-// via the integration→unit chain).
-var run_hook_js_1 = require("./adapters/claude-code/run-hook.js");
-Object.defineProperty(exports, "egressRoutes", { enumerable: true, get: function () { return run_hook_js_1.egressRoutes; } });
 //# sourceMappingURL=e2e.js.map

package/dist/{adapters/claude-code/eval-cache.d.ts → eval-cache.d.ts}

@@ -1,4 +1,4 @@
-import { type SHA256Hash } from "../../core/hash.js";
+import { type SHA256Hash } from "./core/hash.js";
 import type { RunOut } from "./eval.js";
 /** Cache behaviour: never touch the cache / read-only / read-and-write. */
 export type CacheMode = "off" | "read" | "readwrite";

package/dist/{adapters/claude-code/eval-cache.js → eval-cache.js}

@@ -24,7 +24,7 @@ exports.restoreDir = restoreDir;
  */
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
-const hash_js_1 = require("../../core/hash.js");
+const hash_js_1 = require("./core/hash.js");
 const MAX_SNAPSHOT_FILE_BYTES = 1024 * 1024;
 const SKIP_DIRS = new Set(["node_modules", ".git"]);
 /**

package/dist/{adapters/claude-code/eval.d.ts → eval.d.ts}

@@ -1,5 +1,7 @@
 import { type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
+import type { Check, CheckJSON } from "./check.js";
+import { type Comparison } from "./stats.js";
 /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
 export interface EvalArm {
     /** Files written on top of the base fixture for this arm. */
@@ -172,6 +174,127 @@ export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
  * {@link runEvalWith} with the real agent runner.
  */
 export declare function runEval<M extends Metrics>(spec: EvalSpec<M>): Promise<EvalReport>;
+/** A task run N times, scored against a `Trace` check vocabulary. */
+export interface MeasureSpec {
+    /** Base fixture files written for every run (path → contents). */
+    readonly fixture?: Record<string, string>;
+    /** `.claude/settings.json` (hooks/permissions) for the run. */
+    readonly settings?: unknown;
+    /** A real plugin/repo to load (materialized) — see `EvalArm.plugin`. */
+    readonly plugin?: string;
+    /** A complete plugin dir to install natively (`--plugin-dir`) so skills activate. */
+    readonly pluginDir?: string;
+    /**
+     * Stub each skill BODY in `pluginDir` (frontmatter/trigger surface kept) before
+     * the run — for checks about whether a skill FIRES (`skill()`), not what it
+     * produces. A selected skill stops at selection instead of running its (often
+     * expensive) procedure, so a description/firing run costs a fraction of the
+     * tokens. Do NOT combine with `judged`/quality checks: the body is gone, so
+     * there's nothing to grade. Requires `pluginDir`. See {@link stubSkillBody}.
+     */
+    readonly stubSkillBodies?: boolean;
+    /** The task prompt given to the agent. */
+    readonly task: string;
+    /**
+     * The checks to score across the trials. Any check over the run is accepted:
+     * `Trace` checks (`tool`/`skill`/`output`/`mcp`/`judged`) and resource checks
+     * (`cost`/`latency`/`tokens`, which read the eval-only `usage`) — all fit
+     * `Check<RunContext>`.
+     */
+    readonly checks: readonly Check<RunContext>[];
+    /** Trials. Default 5. */
+    readonly trials?: number;
+    /** Model alias. Default "sonnet" — measure on the model your users run. */
+    readonly model?: string;
+    /** Tools the agent may use. */
+    readonly allowedTools?: readonly string[];
+    /** Per-run timeout ms. */
+    readonly timeoutMs?: number;
+    /** Seconds between runs. */
+    readonly spacingSec?: number;
+}
+/** One check's measured rate across the trials. */
+export interface CheckRate {
+    readonly check: CheckJSON;
+    /** Fraction of trials the check passed (0..1). */
+    readonly rate: number;
+    /** Standard error of the rate. */
+    readonly se: number;
+    /** pass^k — 1 iff the check passed on EVERY trial. */
+    readonly passK: number;
+    /** Trials observed. */
+    readonly n: number;
+}
+export interface CheckReport {
+    readonly n: number;
+    readonly perCheck: readonly CheckRate[];
+}
+/**
+ * Score a check vocabulary across trials — the scored counterpart to
+ * `assertChecks` (strict). Each check yields a `rate ± se` and `pass^k` over `n`
+ * runs. Reuses the tested `runEvalWith` aggregation (one arm), so the loop,
+ * cache, concurrency, and stats come for free. Exported with an injectable
+ * `runner` so the orchestration is unit-testable without a model.
+ */
+export declare function measureWith(spec: MeasureSpec, runner: AgentRunner): Promise<CheckReport>;
+/** Score a check vocabulary across trials against the real `claude` CLI. */
+export declare function measure(spec: MeasureSpec): Promise<CheckReport>;
+/** A task scored against checks across NAMED arms (the harness variable on/off). */
+export interface ArmsMeasureSpec {
+    readonly fixture?: Record<string, string>;
+    /** The arms to compare (settings / plugin / pluginDir per arm). */
+    readonly arms: Record<string, EvalArm>;
+    readonly task: string;
+    readonly checks: readonly Check<RunContext>[];
+    /**
+     * Stub each arm's skill BODIES (frontmatter kept) before the run — the A/B
+     * counterpart to {@link MeasureSpec.stubSkillBodies}. For firing comparisons
+     * (does description variant A fire more than B?), every arm that sets
+     * `pluginDir` is repackaged with bodies stripped so each run stops at
+     * selection — a fraction of the tokens. Arms without a `pluginDir` are left
+     * untouched. Don't combine with `judged`/quality checks. See {@link stubSkillBody}.
+     */
+    readonly stubSkillBodies?: boolean;
+    readonly trials?: number;
+    readonly model?: string;
+    readonly allowedTools?: readonly string[];
+    readonly timeoutMs?: number;
+    readonly spacingSec?: number;
+}
+/** Per-arm {@link CheckReport}s — `arms[name].perCheck[i]` aligns across arms. */
+export interface ArmsCheckReport {
+    readonly arms: Record<string, CheckReport>;
+}
+/** Score checks across arms (injectable runner). Reuses `runEvalWith`. */
+export declare function measureArmsWith(spec: ArmsMeasureSpec, runner: AgentRunner): Promise<ArmsCheckReport>;
+/** Score checks across arms against the real `claude` CLI. */
+export declare function measureArms(spec: ArmsMeasureSpec): Promise<ArmsCheckReport>;
+/**
+ * Welch significance on one check's rate between two arms (`arm` vs `baseline`),
+ * by index in `perCheck`. So "the gated arm resolves the skill significantly more
+ * than vanilla" is a p-value, not a vibe. Reuses `welchTTest` from stats.ts.
+ */
+export declare function compareCheck(report: ArmsCheckReport, baseline: string, arm: string, checkIndex: number): Comparison;
+/** Format a {@link CheckReport}: one line per check with its rate ± se and pass^k. */
+export declare function formatCheckReport(report: CheckReport): string;
+/**
+ * The scored gate (Phase 4): throw if any check's measured rate is below `min` —
+ * the `measure` counterpart to `assertChecks` (strict). Reads the rate, not a
+ * single run, so it never trips on one noisy trial.
+ */
+export declare function assertRates(report: CheckReport, opts: {
+    min: number;
+}): void;
+/**
+ * Serialize a {@link CheckReport} to JUnit XML (Phase 4) — each check a
+ * `<testcase>`, failing when its rate is below `min`. Because a check is *data*,
+ * this falls out for free: CI test reporters, regression baselines, and a
+ * promptfoo bridge all consume the same shape.
+ */
+export declare function checkReportToJUnit(report: CheckReport, opts?: {
+    min?: number;
+    name?: string;
+}): string;
 /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
 export declare function parseUsage(stdout: string): EvalUsage;
 /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */
@@ -209,8 +332,19 @@ export declare function formatEvalReport(report: EvalReport): string;
  * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`).
  */
 export interface TriggerRateSpec {
-    /** Plugin dir installed natively (`--plugin-dir`) so its skills/commands activate. */
-    readonly pluginDir: string;
+    /**
+     * Plugin dir installed natively (`--plugin-dir`) so its skills/commands
+     * activate. Provide this OR {@link skillsDir}, not both.
+     */
+    readonly pluginDir?: string;
+    /**
+     * A directory of LOOSE skills (`<skillsDir>/<name>/SKILL.md`, e.g. a repo's
+     * `.claude/skills`) to trigger-test directly. vigiles packages them into a
+     * throwaway `--plugin-dir` for you and removes it afterward — the one-liner
+     * for repo-local skills that aren't a published plugin. Provide this OR
+     * {@link pluginDir}, not both.
+     */
+    readonly skillsDir?: string;
     /** The varied prompts to test the trigger against. */
     readonly prompts: readonly string[];
     /**
@@ -223,6 +357,30 @@ export interface TriggerRateSpec {
     readonly irrelevantPrompts?: readonly string[];
     /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */
     readonly fired: (trace: Trace) => boolean;
+    /**
+     * Replace each skill's BODY with a no-op stub (keeping its frontmatter — name +
+     * description) before running. Trigger-rate is decided by the frontmatter alone
+     * (the model selects a skill before its body loads), so stubbing the body can't
+     * change what's measured but stops the run from executing an expensive
+     * procedure once the skill fires — far cheaper, faster, side-effect-free. All
+     * skills' descriptions stay present, so the selection competition is faithful.
+     * Default false (off) for now; recommended `true` for trigger evals. See
+     * {@link stubSkillBody}.
+     */
+    readonly stubSkillBodies?: boolean;
+    /**
+     * Minimum number of prompts each set (relevant + irrelevant) must have. A
+     * handful of prompts can't tell a real recall/precision rate from noise, so
+     * the run is rejected before it spends a token. Default 10; lower it
+     * deliberately for a genuinely narrow skill.
+     */
+    readonly minPrompts?: number;
+    /**
+     * Reject the run when two prompts in a set are closer than this in NCD
+     * (gzip-based distance, 0..1; 0 = identical) — near-duplicate prompts inflate
+     * a rate without testing varied phrasings. Default 0.3 (the rule-dup threshold).
+     */
+    readonly minDistance?: number;
     /** Trials per prompt. Default 1. */
     readonly trials?: number;
     /** Model alias. Default "haiku". */
@@ -263,6 +421,63 @@ export interface TriggerRateReport {
     /** Per-prompt stats for the irrelevant set. Present with irrelevant prompts. */
     readonly perIrrelevant?: readonly PromptTriggerStat[];
 }
+/**
+ * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
+ * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`)
+ * can be trigger-tested without hand-rolling a `plugin.json`. Writes a minimal
+ * `.claude-plugin/plugin.json` and copies each `<name>/` (recursively, so
+ * `references/` etc. come along) under `skills/<name>/`. Returns the temp plugin
+ * dir; the caller removes it (`measureTriggerRate` does). Throws if the directory
+ * is missing or holds no `<name>/SKILL.md`.
+ */
+export declare function packageSkillsDir(skillsDir: string, opts?: {
+    name?: string;
+    stub?: boolean;
+}): string;
+/**
+ * Rewrite a SKILL.md to keep its YAML frontmatter (the trigger surface — name +
+ * description) but replace the body with a no-op stub. Trigger-rate is a property
+ * of the frontmatter ONLY: the model picks a skill from its name + description
+ * before the body is ever loaded, so the body is causally downstream of selection
+ * and irrelevant to whether the skill fires. Stubbing it lets a trigger run stop
+ * AT selection instead of executing an expensive multi-step procedure — cheaper,
+ * faster, and side-effect-free, without changing what's measured. Pure.
+ */
+export declare function stubSkillBody(skillMd: string): string;
+/**
+ * Build a throwaway plugin dir mirroring `pluginDir`'s skills with their BODIES
+ * stripped (frontmatter kept) — the trigger surface a description/firing check
+ * needs, without paying to run each skill's procedure. Keeps the original plugin
+ * NAME so `<name>:<skill>` ids still match. The caller removes the returned dir.
+ * See {@link stubSkillBody} for why the body is irrelevant to selection.
+ */
+export declare function stubbedPluginDir(pluginDir: string): string;
+/**
+ * Distance between two prompts in ~0..1 (0 = identical, higher = more
+ * different) via Normalized Compression Distance over the normalized text.
+ * Reuses {@link ncd} from the proof engine.
+ */
+export declare function promptDistance(a: string, b: string): number;
+export interface PromptDiversityIssue {
+    readonly kind: "too-few" | "too-similar";
+    readonly message: string;
+}
+/**
+ * Deterministically check a prompt set is big and varied enough to measure a
+ * trigger rate: at least `minPrompts` entries, and no two closer than
+ * `minDistance` in NCD. Pure — no model. `label` names the set in messages.
+ */
+export declare function checkPromptDiversity(prompts: readonly string[], opts?: {
+    minPrompts?: number;
+    minDistance?: number;
+    label?: string;
+}): PromptDiversityIssue[];
+/** Throw if a prompt set isn't big/varied enough. See {@link checkPromptDiversity}. */
+export declare function assertPromptDiversity(prompts: readonly string[], opts?: {
+    minPrompts?: number;
+    minDistance?: number;
+    label?: string;
+}): void;
 /**
  * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
  * predicate evaluated per run and aggregated into an overall + per-prompt rate.