vigiles 4.0.2 → 5.0.0

package/dist/eval-cache.js

@@ -1,10 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.CACHE_FORMAT_VERSION = void 0;
 exports.cacheKey = cacheKey;
 exports.readCache = readCache;
 exports.writeCache = writeCache;
 exports.snapshotDir = snapshotDir;
 exports.restoreDir = restoreDir;
+exports.hashDir = hashDir;
 /**
  * vigiles — record/replay cache for the eval tier.
  *
@@ -43,20 +45,70 @@ function canonical(value) {
     }
     return value;
 }
+/**
+ * Cache record-format version, SALTED into every key (Jest `CACHE_VERSION` /
+ * webpack `cache.version` pattern). Bump when the `CacheRecord` shape — or how a
+ * record is produced in a way the key can't otherwise see — changes, so old
+ * entries become *unreachable* rather than deserializing into a stale shape (no
+ * brittle read-time version gate needed). A major bump means orphaned files on
+ * disk; reclaim them by deleting the cache dir.
+ */
+exports.CACHE_FORMAT_VERSION = 2;
+/**
+ * Per-run env keys that are PURE NOISE for the cache key — a fresh random path
+ * every run, never model-affecting. The opt-in ephemeral run env
+ * ({@link ephemeralRunEnv} in `eval.ts`) points `HOME`/`TMPDIR` at a throwaway
+ * dir generated per trial, so folding them into the key would make every
+ * ephemeral run unique → the cache could NEVER hit. We drop them here so the
+ * exclusion holds however the env was assembled. A non-ephemeral run normally
+ * carries neither in its per-run `env` (it's an overlay over `process.env`, not
+ * a complete env), so dropping them is a no-op there.
+ */
+const CACHE_KEY_ENV_EXCLUDE = ["HOME", "TMPDIR"];
+/** Strip the per-run-noise env keys ({@link CACHE_KEY_ENV_EXCLUDE}) from a keyed
+ *  env, returning `undefined` when nothing model-affecting remains (so the key is
+ *  byte-identical to a run that had no env at all). */
+function keyedEnv(env) {
+    if (env === undefined)
+        return undefined;
+    const out = {};
+    for (const [k, v] of Object.entries(env)) {
+        if (CACHE_KEY_ENV_EXCLUDE.includes(k))
+            continue;
+        out[k] = v;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}
 /** Deterministic content hash of the key inputs (order-independent). */
 function cacheKey(input) {
-    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(input)));
+    const normalized = {
+        ...input,
+        // The tool list is logically a SET, so ["Read","Bash"] and ["Bash","Read"]
+        // must hash the same — sort it to avoid phantom-distinct keys. (canonical()
+        // already sorts object keys; it deliberately keeps other array order.)
+        tools: [...input.tools].sort(),
+        // Drop the throwaway ephemeral HOME/TMPDIR — per-run noise, not model input.
+        env: keyedEnv(input.env),
+        cacheFormatVersion: exports.CACHE_FORMAT_VERSION,
+    };
+    return (0, hash_js_1.sha256short)(JSON.stringify(canonical(normalized)));
 }
-/** Read a cached record by key, or null on miss / unreadable / malformed. */
+/**
+ * Read a cached record by key. A MISS (no file) returns `null` — normal, the run
+ * proceeds. A CORRUPT record (file present but not valid JSON) **throws** instead
+ * of silently degrading to a re-run: a broken cassette is a real failure the CI
+ * gate must surface, not mask. The message tells you how to recover.
+ */
 function readCache(dir, key) {
     const path = (0, node_path_1.join)(dir, `${key}.json`);
     if (!(0, node_fs_1.existsSync)(path))
         return null;
+    const raw = (0, node_fs_1.readFileSync)(path, "utf-8");
     try {
-        return JSON.parse((0, node_fs_1.readFileSync)(path, "utf-8"));
+        return JSON.parse(raw);
     }
     catch {
-        return null;
+        throw new Error(`eval cache: corrupt record ${path} (invalid JSON) — delete it or clear the cache dir`);
     }
 }
 /** Write a cached record by key (creating the cache dir as needed). */
@@ -91,4 +143,32 @@ function restoreDir(cwd, files) {
         (0, node_fs_1.writeFileSync)(full, content);
     }
 }
+/**
+ * Content digest of a directory: a lexicographically-sorted list of
+ * `relativePath:contentHash` for every file, hashed to one value. Editing,
+ * adding, removing, or moving any file changes the digest. It hashes file
+ * CONTENT (not mtime — CI checkouts reset mtimes, the classic stale-cache
+ * anti-pattern) and includes the relative path (so a rename invalidates and two
+ * files can't swap contents undetected). A flat sorted list, NOT a Merkle tree —
+ * sufficient at plugin-dir scale; the tree's incremental-recompute payoff isn't
+ * worth the complexity here (cf. Bazel/Turborepo hash content per file).
+ */
+function hashDir(dir) {
+    const root = (0, node_path_1.resolve)(dir);
+    const parts = [];
+    const walk = (d) => {
+        for (const entry of (0, node_fs_1.readdirSync)(d).sort()) {
+            if (SKIP_DIRS.has(entry))
+                continue;
+            const full = (0, node_path_1.join)(d, entry);
+            const st = (0, node_fs_1.statSync)(full);
+            if (st.isDirectory())
+                walk(full);
+            else if (st.isFile())
+                parts.push(`${(0, node_path_1.relative)(root, full)}:${(0, hash_js_1.sha256short)((0, node_fs_1.readFileSync)(full))}`);
+        }
+    };
+    walk(root);
+    return (0, hash_js_1.sha256short)(parts.join("\n"));
+}
 //# sourceMappingURL=eval-cache.js.map

package/dist/eval.d.ts

@@ -2,6 +2,8 @@ import { type ToolCall, type Trace } from "./harness-test.js";
 import { type CacheMode } from "./eval-cache.js";
 import type { Check, CheckJSON } from "./check.js";
 import { type Comparison } from "./stats.js";
+import { type ToolIntercept } from "./tool-intercept.js";
+import { type ToolStub } from "./tool-stub.js";
 /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */
 export interface EvalArm {
     /** Files written on top of the base fixture for this arm. */
@@ -22,6 +24,27 @@ export interface EvalArm {
      * an arm be "skill installed" vs "off" to measure real activation.
      */
     readonly pluginDir?: string;
+    /**
+     * Tools to intercept for this arm (the tool-call spy). Each
+     * {@link ToolIntercept} is denied its real execution by an auto-wired PreToolUse
+     * hook — the model still emits the `tool_use` (so its arguments land in the
+     * `Trace` for `toolWith` / `notTool`), but the side effect (a paid API call, a
+     * `git push`, a paid subagent) never happens: the call is intercepted
+     * (prevented), NOT executed. This makes a real-model eval **side-effect-free and
+     * safe** (it does NOT cut the model-call cost); and because CC surfaces the
+     * denial as a *blocked* call, it's for asserting the agent's ATTEMPT, not for
+     * stubbing a tool to continue a multi-step flow. See `src/tool-intercept.ts`.
+     */
+    readonly interceptTools?: readonly ToolIntercept[];
+    /**
+     * Model alias/id for THIS arm, overriding the eval-level `model`. A model
+     * comparison IS a harness A/B — `arms: { sonnet: { model: "claude-sonnet-4-6" },
+     * opus: { model: "claude-opus-4-8" } }` — so model-as-an-arm answers "does my
+     * harness still hold on the cheaper tier / after a model upgrade?" through the
+     * same significance machinery, with no separate model-matrix runner. Omit to
+     * use the eval-level model. See `docs/eval-architecture.md` (model strategy).
+     */
+    readonly model?: string;
 }
 /** Per-run resource use, parsed from the terminal `result` event (0 when absent). */
 export interface EvalUsage {
@@ -29,8 +52,13 @@ export interface EvalUsage {
     readonly costUsd: number;
     /** Wall-clock `duration_ms` of the run. */
     readonly durationMs: number;
+    /** Fresh (uncached) input tokens, billed at full input price. */
     readonly inputTokens: number;
     readonly outputTokens: number;
+    /** Tokens written to the prompt cache this run (~1.25× input price). */
+    readonly cacheCreationTokens: number;
+    /** Tokens served from the prompt cache this run (~0.1× input price). */
+    readonly cacheReadTokens: number;
 }
 /**
  * Context handed to `measure` after a run, to compute that run's metrics. It is
@@ -98,6 +126,39 @@ export interface EvalSpec<M extends Metrics> {
     readonly rateLimitRetries?: number;
     /** Base backoff ms (doubled each retry). Default 1000. */
     readonly retryBackoffMs?: number;
+    /**
+     * **Opt-in, default OFF.** Run each trial in an *ephemeral run environment* — a
+     * throwaway `$HOME` + scrubbed env, re-injecting only the harness's own auth (see
+     * {@link ephemeralRunEnv}). Running a model-driven skill/agent is itself a side
+     * effect (the *model*, not the author, chose the actions), so a `git push` /
+     * write to `~` should land in a disposable HOME, not the real `~/.gitconfig` /
+     * `~/.ssh` / `~/.aws`. This is the cross-platform STATE-protection floor (no
+     * kernel features), orthogonal to the bubblewrap host-confinement in
+     * `src/sandbox.ts`.
+     *
+     * **Ships default-OFF** because a too-narrow auth allowlist would silently break
+     * the real `claude` CLI's authentication; leaving it off keeps every existing
+     * eval (including one running right now) authenticating exactly as before. When
+     * absent / `false`, the per-trial env is byte-identical to today
+     * (`{ ...process.env, ...arm.env }`). See `docs/safety.md` (ephemerality) and
+     * `research/cross-platform-sandboxing.md`.
+     */
+    readonly ephemeralEnv?: boolean;
+    /**
+     * **Tool stubs on PATH (rung R2).** A list of fake binaries to shadow on PATH
+     * for every trial, so a skill/hook/agent that calls a CLI tool (`gh`, `psql`,
+     * `redis-cli`, `z3`, …) and works with its RESULT can be tested against a
+     * **recorded / author-provided canned output** — no live service. vigiles writes
+     * one executable stub per {@link ToolStub} into a bin dir under the trial cwd and
+     * PREPENDS that dir to the run's PATH (both the default and the ephemeral env
+     * path), so the fake wins over the real binary.
+     *
+     * The stubs are author/recorded fixtures, **never** model-synthesized — a
+     * synthesized tool output looks plausible but diverges from the real
+     * tool/version (false confidence). Absent → no change (the PATH is byte-identical
+     * to today). See {@link ToolStub} and `research/eval-coverage-and-isolation.md`.
+     */
+    readonly stubs?: readonly ToolStub[];
 }
 /** Per-metric summary statistics across an arm's runs. */
 export interface MetricStat {
@@ -124,6 +185,8 @@ export interface ArmUsage {
     readonly meanDurationMs: number;
     readonly totalInputTokens: number;
     readonly totalOutputTokens: number;
+    readonly totalCacheCreationTokens: number;
+    readonly totalCacheReadTokens: number;
 }
 export interface ArmReport {
     readonly runs: number;
@@ -159,6 +222,15 @@ export interface AgentRunArgs {
     readonly hasSettings: boolean;
     readonly pluginDir: string | undefined;
     readonly timeoutMs: number;
+    /** Extra env layered over `process.env` for this run (e.g. `VIGILES_INTERCEPT_TOOLS`). */
+    readonly env?: Record<string, string>;
+    /**
+     * When true, `env` is the COMPLETE spawn environment (an ephemeral run env from
+     * {@link ephemeralRunEnv}) — the runner does NOT prepend `process.env`, so the
+     * real `$HOME` / secrets are scrubbed. Default false: `env` is an overlay over
+     * `process.env` (the byte-identical-to-today path). Set only by `ephemeralEnv`.
+     */
+    readonly replaceEnv?: boolean;
 }
 /**
  * Runs one trial and returns its raw output. The default ({@link spawnAgent})
@@ -167,6 +239,17 @@ export interface AgentRunArgs {
  * stream-json) and a custom runtime can be plugged in.
  */
 export type AgentRunner = (args: AgentRunArgs) => Promise<RunOut>;
+/**
+ * Resolve the environment a trial's subprocess actually runs with — the
+ * SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
+ * scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
+ * secrets are DROPPED; otherwise `env` is an overlay on `base` (byte-identical to
+ * the pre-ephemeral behaviour). Extracted from the `v8 ignore`d `spawnAgent` so
+ * the one line that enforces the scrub is both unit- and behaviourally-tested — a
+ * regression to an always-merge would otherwise silently defeat ephemerality and
+ * leak the host environment into an untrusted, model-driven run.
+ */
+export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
 /**
  * Run the eval: every arm × every trial against the real `claude` CLI, with the
  * metric computed per run and aggregated per arm. Requires `claude` on PATH and
@@ -193,6 +276,8 @@ export interface MeasureSpec {
      * there's nothing to grade. Requires `pluginDir`. See {@link stubSkillBody}.
      */
     readonly stubSkillBodies?: boolean;
+    /** Tools to intercept (the tool-call spy) — see {@link EvalArm.interceptTools}. */
+    readonly interceptTools?: readonly ToolIntercept[];
     /** The task prompt given to the agent. */
     readonly task: string;
     /**
@@ -278,21 +363,31 @@ export declare function compareCheck(report: ArmsCheckReport, baseline: string,
 /** Format a {@link CheckReport}: one line per check with its rate ± se and pass^k. */
 export declare function formatCheckReport(report: CheckReport): string;
 /**
- * The scored gate (Phase 4): throw if any check's measured rate is below `min` —
- * the `measure` counterpart to `assertChecks` (strict). Reads the rate, not a
- * single run, so it never trips on one noisy trial.
+ * The scored gate (Phase 4): throw if any check's measured rate is below its
+ * threshold — the `measure` counterpart to `assertChecks` (strict). Reads the
+ * rate, not a single run, so it never trips on one noisy trial.
+ *
+ * `min` is the default threshold for every check. `per` overrides it for a check
+ * KIND (e.g. `{ min: 0.8, per: { skill: 1.0 } }` — "every check ≥ 80%, but the
+ * skill must FIRE on every trial"), so a strict firing/safety check and a
+ * lenient quality check gate in one call — the single-skill absolute oracle
+ * (`measure({ checks: [skill(), judged()] }) + assertRates`) needs exactly this.
  */
 export declare function assertRates(report: CheckReport, opts: {
     min: number;
+    per?: Readonly<Record<string, number>>;
 }): void;
 /**
  * Serialize a {@link CheckReport} to JUnit XML (Phase 4) — each check a
- * `<testcase>`, failing when its rate is below `min`. Because a check is *data*,
- * this falls out for free: CI test reporters, regression baselines, and a
- * promptfoo bridge all consume the same shape.
+ * `<testcase>`, failing when its rate is below its threshold. `min` is the
+ * default; `per` overrides it by check KIND, matching `assertRates` exactly (one
+ * shared threshold helper) so the gate and the report can never disagree about
+ * which checks failed. Because a check is *data*, this falls out for free: CI
+ * test reporters, regression baselines, and a promptfoo bridge all consume it.
  */
 export declare function checkReportToJUnit(report: CheckReport, opts?: {
     min?: number;
+    per?: Readonly<Record<string, number>>;
     name?: string;
 }): string;
 /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */
@@ -307,6 +402,86 @@ export declare function aggregate(rows: readonly Metrics[]): Record<string, numb
 export declare function aggregateStats(rows: readonly Metrics[]): Record<string, MetricStat>;
 /** Aggregate per-run usage into an arm's cost / latency / token totals + means. */
 export declare function aggregateUsage(usages: readonly EvalUsage[]): ArmUsage;
+/**
+ * A model id is "dated" (honestly pinned) when it ends in an 8-digit date stamp,
+ * e.g. `claude-haiku-4-5-20251001`. A floating alias (`haiku`, `sonnet`, or even
+ * `claude-sonnet-4-6` with no date) can change underneath you — so a cached or
+ * baselined result pinned to it can silently hide model drift. See
+ * `docs/eval-architecture.md` (honest model pinning).
+ */
+export declare function isDatedModel(model: string): boolean;
+/**
+ * Capability tier of a model by FAMILY: haiku=1 < sonnet=2 < opus=3 (version is
+ * ignored, so `claude-sonnet-4-6` and a dated Sonnet rank equal). An unrecognized
+ * family returns `null` — unrankable, so the floor never blocks a model we can't
+ * judge (fail-open on ranking). Used by the model floor; aliases and full/dated
+ * ids both work.
+ */
+export declare function modelTier(id: string): number | null;
+/**
+ * Is `model` a weaker tier than `floor`? Both must be rankable (see
+ * {@link modelTier}); an unrankable model/floor is never "below" (fail-open).
+ */
+export declare function belowModelFloor(model: string, floor: string): boolean;
+/**
+ * Reduce a raw `--version` string to the **major.minor** cache-key token. We key
+ * the cache on major.minor, NOT the patch: a patch release rarely changes agent
+ * behaviour, so keying patches would churn the cache on every release for no
+ * signal; a minor/major bump is where the system prompt / tool defs actually move.
+ * (If a specific patch is known to matter, clear the cache or bump
+ * `CACHE_FORMAT_VERSION`.) Falls back to the trimmed raw string when no semver is
+ * found. Pure + tested.
+ */
+export declare function harnessVersionKey(raw: string): string;
+/**
+ * Build an **ephemeral run environment** for a model-driven run: a NEW env object
+ * with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, only
+ * an allowlist of auth + runtime vars passed through from `base`, and everything
+ * else DROPPED. Pure — no fs, no spawn.
+ *
+ * The rationale is "fresh HOME + only the harness credential injected, **not** a
+ * blanket wipe": running a model-driven skill/agent is itself a side effect (the
+ * *model*, not the author, chose the actions), so it should not be able to read
+ * the real `~/.gitconfig` / `~/.ssh` / `~/.aws` or write to the real `~`. But the
+ * real `claude` CLI must still AUTHENTICATE, so the harness's own credentials
+ * ({@link EPHEMERAL_ALLOW} — `ANTHROPIC_*`, `CLAUDE_*`, locale/PATH) are
+ * re-injected; a blanket `--clearenv`-style wipe would break every eval. Because
+ * this needs no kernel features, it is the cross-platform STATE-protection floor
+ * (lands on macOS immediately), orthogonal to the Linux bubblewrap HOST
+ * confinement in `src/sandbox.ts`.
+ *
+ * @param base  the source environment to filter (usually `process.env`).
+ * @param opts.home  the throwaway dir to set as `HOME`/`TMPDIR`.
+ * @param opts.allow  extra var NAMES to pass through (e.g. the `VIGILES_*` keys
+ *   the eval already injects). Layered ON TOP of the default allowlist.
+ */
+export declare function ephemeralRunEnv(base: NodeJS.ProcessEnv | Record<string, string | undefined>, opts: {
+    home: string;
+    allow?: readonly string[];
+}): Record<string, string>;
+/**
+ * Home-relative AUTH files to carry from the real HOME into the throwaway one.
+ *
+ * A local subscription credential (OAuth token) often lives in a FILE under HOME,
+ * not an env var — so scrubbing HOME would lose it and break a local-authed run.
+ * This is the file half of the auth allowlist; {@link EPHEMERAL_ALLOW} covers the
+ * env-var / host-brokered half. Kept a named constant so it's easy to extend, and
+ * deliberately NARROW — only the explicit auth files, never `.gitconfig` / `.ssh`
+ * / `.aws`, which are exactly what an ephemeral run must not see.
+ */
+export declare const EPHEMERAL_HOME_KEEP: readonly string[];
+/**
+ * Seed the throwaway HOME with the harness's own auth FILE(s) — best-effort;
+ * covers local file-based OAuth; the env-var/host-brokered path is covered by the
+ * allowlist in {@link ephemeralRunEnv}.
+ *
+ * COPIES (never symlinks) each {@link EPHEMERAL_HOME_KEEP} path from `realHome`
+ * into `throwawayHome`, creating parent dirs as needed; a symlink would let the
+ * model-driven run write back to the real credential file, defeating ephemerality.
+ * A path that doesn't exist in the real HOME is skipped silently (that user auths
+ * via env-var / host broker instead). Pure fs — no env, no spawn.
+ */
+export declare function seedEphemeralHome(throwawayHome: string, realHome: string, keep?: readonly string[]): void;
 /** Whether a run's captured output looks like a rate-limit / overload. Pure. */
 export declare function isRateLimited(out: RunOut): boolean;
 /** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */
@@ -357,15 +532,30 @@ export interface TriggerRateSpec {
     readonly irrelevantPrompts?: readonly string[];
     /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */
     readonly fired: (trace: Trace) => boolean;
+    /**
+     * Co-install these skill sources ALONGSIDE the skill-under-test so it competes
+     * for selection as it does in the real harness — the **whole-harness** tier.
+     * Each entry is a plugin dir (`skills/` or `.claude/skills/`) or a loose skills
+     * dir (`<name>/SKILL.md`); their skills are merged into one install under the
+     * under-test plugin's name (the under-test skill wins a name collision, so its
+     * `<name>:<skill>` id still matches `fired`).
+     *
+     * WHY: skill selection is competitive and Claude Code evicts the least-used
+     * skill descriptions under a context budget, so an ISOLATED trigger-rate (the
+     * default, `installSet` absent/empty) **overstates recall and understates
+     * false-positives**. Isolated is the cheap authoring loop; populate the set for
+     * a release gate. See `research/isolated-vs-whole-harness-eval.md`.
+     */
+    readonly installSet?: readonly string[];
     /**
      * Replace each skill's BODY with a no-op stub (keeping its frontmatter — name +
-     * description) before running. Trigger-rate is decided by the frontmatter alone
-     * (the model selects a skill before its body loads), so stubbing the body can't
-     * change what's measured but stops the run from executing an expensive
-     * procedure once the skill fires — far cheaper, faster, side-effect-free. All
-     * skills' descriptions stay present, so the selection competition is faithful.
-     * Default false (off) for now; recommended `true` for trigger evals. See
-     * {@link stubSkillBody}.
+     * description). Trigger-rate is decided by the frontmatter ALONE (the model
+     * selects a skill before its body loads), so stubbing can't change what's
+     * measured — it just stops the run executing an expensive procedure once the
+     * skill fires. All descriptions stay present, so selection competition is
+     * faithful. **Defaults to `true`** here: testing a description never needs the
+     * body, so it's automated, not a knob you must remember. Set `false` only in the
+     * rare case you want the real body to run. See {@link stubSkillBody}.
      */
     readonly stubSkillBodies?: boolean;
     /**
@@ -383,8 +573,20 @@ export interface TriggerRateSpec {
     readonly minDistance?: number;
     /** Trials per prompt. Default 1. */
     readonly trials?: number;
-    /** Model alias. Default "haiku". */
+    /**
+     * Model alias/id. Default `"sonnet"` — the realistic selector most Claude Code
+     * users run. NOT haiku: trigger-rate is a *selection* measurement and haiku is a
+     * much weaker selector, so it under-reports recall (dogfooded: a skill scored
+     * 0.50 on haiku vs 0.90 on Sonnet). Override for a cheaper-but-pessimistic run.
+     */
     readonly model?: string;
+    /**
+     * Minimum model tier this eval may run on (haiku<sonnet<opus by family). The
+     * run **fails** if the resolved `model` is weaker — trigger-rate under-measures
+     * selection on a too-weak model, so this stops a cheap model from producing
+     * false-negative recall. Default `"sonnet"`. Lower it deliberately for a cheap run.
+     */
+    readonly minModel?: string;
     /** Tools the agent may use. Default: Read Edit Write Bash Skill. */
     readonly allowedTools?: readonly string[];
     /** Per-run timeout ms. Default 240000. */
@@ -420,6 +622,14 @@ export interface TriggerRateReport {
     readonly precision?: number;
     /** Per-prompt stats for the irrelevant set. Present with irrelevant prompts. */
     readonly perIrrelevant?: readonly PromptTriggerStat[];
+    /**
+     * Competitor skills co-installed via {@link TriggerRateSpec.installSet}. `0`
+     * (the default) means the skill was measured **ISOLATED** — so `rate` is an
+     * UPPER bound on real recall and `falsePositiveRate` a LOWER bound, because
+     * selection is competitive (a populated harness can evict or out-compete the
+     * description). A non-zero count is the whole-harness measurement.
+     */
+    readonly competitors: number;
 }
 /**
  * Package loose `<skillsDir>/<name>/SKILL.md` skills into a throwaway plugin dir
@@ -478,6 +688,25 @@ export declare function assertPromptDiversity(prompts: readonly string[], opts?:
     minDistance?: number;
     label?: string;
 }): void;
+/**
+ * Build a combined plugin: the under-test skills PLUS every `installSet` source's
+ * skills, so the skill-under-test competes for selection as in the real harness.
+ * Named after the under-test plugin so `<name>:<skill>` ids still match; the
+ * under-test skills win a name collision. Returns the dir + `added` = how many
+ * installSet skills were merged in (excludes collisions). The report's
+ * `competitors` is derived separately from the FULL pool (see `countSkills`), so
+ * sibling skills already in the under-test source count too. Caller removes the
+ * dir. Pure (filesystem only).
+ */
+export declare function packageInstallSet(opts: {
+    underTestSrc: string;
+    name: string;
+    installSet: readonly string[];
+    stub: boolean;
+}): {
+    dir: string;
+    added: number;
+};
 /**
  * Trigger-rate orchestration — every prompt × trial via `runner`, the `fired`
  * predicate evaluated per run and aggregated into an overall + per-prompt rate.