vigiles 33.2.0 → 33.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/adapter.d.ts +1 -1
- package/dist/adapters/claude-code/runtime.js +70 -0
- package/dist/adapters/claude-code/skill-reachability.d.ts +42 -21
- package/dist/adapters/claude-code/skill-reachability.js +91 -28
- package/dist/cli-main.js +224 -17
- package/dist/core/adapter.d.ts +7 -0
- package/dist/core/install-reader.js +15 -0
- package/dist/core/layout.d.ts +10 -0
- package/dist/core/layout.js +17 -0
- package/dist/core/run-env.d.ts +50 -0
- package/dist/core/run-env.js +51 -0
- package/dist/core/runtime.d.ts +44 -0
- package/dist/core/skill-link-target.d.ts +20 -0
- package/dist/core/skill-link-target.js +33 -0
- package/dist/eval.d.ts +39 -39
- package/dist/eval.js +62 -115
- package/dist/harness-test.d.ts +35 -0
- package/dist/harness-test.js +49 -7
- package/dist/output-style-arms.js +7 -2
- package/dist/plugin-declaration.d.ts +6 -6
- package/dist/plugin-declaration.js +6 -6
- package/dist/setup-plan.d.ts +49 -9
- package/dist/setup-plan.js +102 -68
- package/dist/skill-links.d.ts +183 -0
- package/dist/skill-links.js +255 -0
- package/dist/test-coverage.js +37 -1
- package/package.json +1 -1
package/dist/eval.d.ts
CHANGED
|
@@ -168,8 +168,9 @@ export interface EvalSpec<M extends Metrics> {
|
|
|
168
168
|
* **Ships default-OFF** because a too-narrow auth allowlist would silently break
|
|
169
169
|
* the real `claude` CLI's authentication; leaving it off keeps every existing
|
|
170
170
|
* eval (including one running right now) authenticating exactly as before. When
|
|
171
|
-
* absent / `false`, the per-trial env is
|
|
172
|
-
*
|
|
171
|
+
* absent / `false`, the per-trial env is the caller's (`{ ...process.env,
|
|
172
|
+
* ...arm.env }`) minus the harness's declared session identity, which no child
|
|
173
|
+
* run inherits in either mode. See `docs/safety.md` (ephemerality) and
|
|
173
174
|
* `research/cross-platform-sandboxing.md`.
|
|
174
175
|
*/
|
|
175
176
|
readonly ephemeralEnv?: boolean;
|
|
@@ -249,11 +250,13 @@ export interface EvalReport {
|
|
|
249
250
|
* Resolve the environment a trial's subprocess actually runs with — the
|
|
250
251
|
* SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
|
|
251
252
|
* scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
|
|
252
|
-
* secrets are DROPPED; otherwise `env` is an overlay on `base
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
256
|
-
*
|
|
253
|
+
* secrets are DROPPED; otherwise `env` is an overlay on `base`. In BOTH, the
|
|
254
|
+
* harness's declared session identity (`runEnv.sessionIdentity`) is removed: a
|
|
255
|
+
* run started from inside a live session must not run as that session, scrubbed
|
|
256
|
+
* or not. Extracted from the `v8 ignore`d `spawnAgent` so the one line that
|
|
257
|
+
* enforces the scrub is both unit- and behaviourally-tested — a regression to an
|
|
258
|
+
* always-merge would otherwise silently defeat ephemerality and leak the host
|
|
259
|
+
* environment into an untrusted, model-driven run.
|
|
257
260
|
*/
|
|
258
261
|
export declare function resolveSpawnEnv(a: Pick<AgentRunArgs, "env" | "replaceEnv" | "effort">, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
259
262
|
/**
|
|
@@ -268,10 +271,10 @@ export declare const EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
|
|
|
268
271
|
*
|
|
269
272
|
* WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
|
|
270
273
|
* `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
|
|
271
|
-
* — and the env var wins over the flag.
|
|
272
|
-
*
|
|
273
|
-
*
|
|
274
|
-
*
|
|
274
|
+
* — and the env var wins over the flag. The default (inherited) spawn env
|
|
275
|
+
* carries the caller's whole environment, so an ambient
|
|
276
|
+
* `CLAUDE_CODE_EFFORT_LEVEL=max` in the author's shell reaches the run unless
|
|
277
|
+
* something removes it. Without this pin,
|
|
275
278
|
* hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
|
|
276
279
|
* record `low` over a run that executed at `max` — the exact defect the feature
|
|
277
280
|
* exists to prevent, reintroduced by the fix for it.
|
|
@@ -601,51 +604,48 @@ export declare function modelTier(id: string): number | null;
|
|
|
601
604
|
export declare function belowModelFloor(model: string, floor: string): boolean;
|
|
602
605
|
/**
|
|
603
606
|
* Build an **ephemeral run environment** for a model-driven run: a NEW env object
|
|
604
|
-
* with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`,
|
|
605
|
-
*
|
|
606
|
-
* else
|
|
607
|
+
* with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, the
|
|
608
|
+
* OS essentials (`PATH`, `LANG`, `LC_*`, `TERM`), the harness's OWN auth, and
|
|
609
|
+
* nothing else. Pure — no fs, no spawn.
|
|
607
610
|
*
|
|
608
611
|
* The rationale is "fresh HOME + only the harness credential injected, **not** a
|
|
609
612
|
* blanket wipe": running a model-driven skill/agent is itself a side effect (the
|
|
610
613
|
* *model*, not the author, chose the actions), so it should not be able to read
|
|
611
614
|
* the real `~/.gitconfig` / `~/.ssh` / `~/.aws` or write to the real `~`. But the
|
|
612
|
-
* real
|
|
613
|
-
* (
|
|
614
|
-
*
|
|
615
|
-
*
|
|
616
|
-
*
|
|
617
|
-
*
|
|
615
|
+
* real CLI must still AUTHENTICATE, so the auth names the harness declares
|
|
616
|
+
* (`runtime.runEnv.keep`) are re-injected, and the parent session's identity
|
|
617
|
+
* (`runtime.runEnv.sessionIdentity`) never is. Which names those are is the
|
|
618
|
+
* adapter's fact, not this file's: see `RunEnvPolicy` in `core/runtime.ts`.
|
|
619
|
+
* Because this needs no kernel features, it is the cross-platform
|
|
620
|
+
* STATE-protection floor (lands on macOS immediately), orthogonal to the Linux
|
|
621
|
+
* bubblewrap HOST confinement in `src/sandbox.ts`.
|
|
622
|
+
*
|
|
623
|
+
* Notably it does NOT pass through `GIT_*`, `GH_TOKEN`, `SSH_*`, `AWS_*` access
|
|
624
|
+
* keys, a config-dir override, or any other undeclared variable — those are
|
|
625
|
+
* exactly what an ephemeral run must not see. A too-narrow `keep` breaks auth
|
|
626
|
+
* loudly (the run fails to start); a too-broad one leaks silently, which is why
|
|
627
|
+
* it is an allowlist and not a prefix.
|
|
618
628
|
*
|
|
619
629
|
* @param base the source environment to filter (usually `process.env`).
|
|
620
630
|
* @param opts.home the throwaway dir to set as `HOME`/`TMPDIR`.
|
|
621
631
|
* @param opts.allow extra var NAMES to pass through (e.g. the `VIGILES_*` keys
|
|
622
|
-
* the eval already injects). Layered ON TOP of the
|
|
632
|
+
* the eval already injects). Layered ON TOP of the harness's `keep`; a
|
|
633
|
+
* session-identity name is dropped even when listed here.
|
|
623
634
|
*/
|
|
624
635
|
export declare function ephemeralRunEnv(base: NodeJS.ProcessEnv | Record<string, string | undefined>, opts: {
|
|
625
636
|
home: string;
|
|
626
637
|
allow?: readonly string[];
|
|
627
638
|
}): Record<string, string>;
|
|
628
|
-
/**
|
|
629
|
-
* Home-relative AUTH files to carry from the real HOME into the throwaway one.
|
|
630
|
-
*
|
|
631
|
-
* A local subscription credential (OAuth token) often lives in a FILE under HOME,
|
|
632
|
-
* not an env var — so scrubbing HOME would lose it and break a local-authed run.
|
|
633
|
-
* This is the file half of the auth allowlist; {@link EPHEMERAL_ALLOW} covers the
|
|
634
|
-
* env-var / host-brokered half. Kept a named constant so it's easy to extend, and
|
|
635
|
-
* deliberately NARROW — only the explicit auth files, never `.gitconfig` / `.ssh`
|
|
636
|
-
* / `.aws`, which are exactly what an ephemeral run must not see.
|
|
637
|
-
*/
|
|
638
|
-
export declare const EPHEMERAL_HOME_KEEP: readonly string[];
|
|
639
639
|
/**
|
|
640
640
|
* Seed the throwaway HOME with the harness's own auth FILE(s) — best-effort;
|
|
641
|
-
* covers local file-based OAuth; the env-var/host-brokered path is covered by
|
|
642
|
-
*
|
|
641
|
+
* covers local file-based OAuth; the env-var/host-brokered path is covered by
|
|
642
|
+
* `ephemeralRunEnv`. The files are the runtime's declared `keepHomeFiles`.
|
|
643
643
|
*
|
|
644
|
-
* COPIES (never symlinks) each
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
644
|
+
* COPIES (never symlinks) each path from `realHome` into `throwawayHome`,
|
|
645
|
+
* creating parent dirs as needed; a symlink would let the model-driven run write
|
|
646
|
+
* back to the real credential file, defeating ephemerality. A path that doesn't
|
|
647
|
+
* exist in the real HOME is skipped silently (that user auths via env-var / host
|
|
648
|
+
* broker instead). Pure fs — no env, no spawn.
|
|
649
649
|
*/
|
|
650
650
|
export declare function seedEphemeralHome(throwawayHome: string, realHome: string, keep?: readonly string[]): void;
|
|
651
651
|
/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
|
package/dist/eval.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.claudeCodeLiveDriver = exports.claudeEvalDriver = exports.
|
|
3
|
+
exports.claudeCodeLiveDriver = exports.claudeEvalDriver = exports.spawnAgent = exports.EFFORT_ENV_VAR = void 0;
|
|
4
4
|
exports.resolveSpawnEnv = resolveSpawnEnv;
|
|
5
5
|
exports.pinEffortEnv = pinEffortEnv;
|
|
6
6
|
exports.effortRejection = effortRejection;
|
|
@@ -76,6 +76,7 @@ const node_path_1 = require("node:path");
|
|
|
76
76
|
const observe_js_1 = require("./observe.js");
|
|
77
77
|
const plugin_loader_js_1 = require("./adapters/claude-code/plugin-loader.js");
|
|
78
78
|
const runtime_js_1 = require("./adapters/claude-code/runtime.js");
|
|
79
|
+
const run_env_js_1 = require("./core/run-env.js");
|
|
79
80
|
const eval_cost_js_1 = require("./eval-cost.js");
|
|
80
81
|
const proofs_js_1 = require("./core/proofs.js");
|
|
81
82
|
const model_access_js_1 = require("./adapters/claude-code/model-access.js");
|
|
@@ -90,6 +91,14 @@ const tool_intercept_js_1 = require("./tool-intercept.js");
|
|
|
90
91
|
const tool_stub_js_1 = require("./tool-stub.js");
|
|
91
92
|
const tmp_root_js_1 = require("./core/tmp-root.js");
|
|
92
93
|
const edit_distance_js_1 = require("./core/edit-distance.js");
|
|
94
|
+
/**
|
|
95
|
+
* The harness the eval tier spawns. ONE binding on purpose: the paid tier is
|
|
96
|
+
* still wired to the Claude Code runner rather than chosen through the adapter's
|
|
97
|
+
* eval driver (#325), and every fact this file needs about that harness — its
|
|
98
|
+
* binary, its version key, which env a child may inherit — is read off this
|
|
99
|
+
* runtime's port fields, never spelled here.
|
|
100
|
+
*/
|
|
101
|
+
const EVAL_RUNTIME = runtime_js_1.claudeCodeRuntime;
|
|
93
102
|
function writeFiles(cwd, files) {
|
|
94
103
|
for (const [p, content] of Object.entries(files)) {
|
|
95
104
|
const full = (0, node_path_1.resolve)(cwd, p);
|
|
@@ -101,15 +110,17 @@ function writeFiles(cwd, files) {
|
|
|
101
110
|
* Resolve the environment a trial's subprocess actually runs with — the
|
|
102
111
|
* SECURITY-CRITICAL decision behind `ephemeralEnv`. When `replaceEnv` is set, the
|
|
103
112
|
* scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited
|
|
104
|
-
* secrets are DROPPED; otherwise `env` is an overlay on `base
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
113
|
+
* secrets are DROPPED; otherwise `env` is an overlay on `base`. In BOTH, the
|
|
114
|
+
* harness's declared session identity (`runEnv.sessionIdentity`) is removed: a
|
|
115
|
+
* run started from inside a live session must not run as that session, scrubbed
|
|
116
|
+
* or not. Extracted from the `v8 ignore`d `spawnAgent` so the one line that
|
|
117
|
+
* enforces the scrub is both unit- and behaviourally-tested — a regression to an
|
|
118
|
+
* always-merge would otherwise silently defeat ephemerality and leak the host
|
|
119
|
+
* environment into an untrusted, model-driven run.
|
|
109
120
|
*/
|
|
110
121
|
function resolveSpawnEnv(a, base = process.env) {
|
|
111
122
|
const resolved = a.replaceEnv ? (a.env ?? {}) : { ...base, ...a.env };
|
|
112
|
-
return pinEffortEnv(resolved, a.effort);
|
|
123
|
+
return pinEffortEnv((0, run_env_js_1.withoutSessionIdentity)(resolved, EVAL_RUNTIME.runEnv), a.effort);
|
|
113
124
|
}
|
|
114
125
|
/**
|
|
115
126
|
* The env var name the harness reads for the reasoning budget. It sits ABOVE the
|
|
@@ -123,10 +134,10 @@ exports.EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL";
|
|
|
123
134
|
*
|
|
124
135
|
* WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the
|
|
125
136
|
* `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL`
|
|
126
|
-
* — and the env var wins over the flag.
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
137
|
+
* — and the env var wins over the flag. The default (inherited) spawn env
|
|
138
|
+
* carries the caller's whole environment, so an ambient
|
|
139
|
+
* `CLAUDE_CODE_EFFORT_LEVEL=max` in the author's shell reaches the run unless
|
|
140
|
+
* something removes it. Without this pin,
|
|
130
141
|
* hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would
|
|
131
142
|
* record `low` over a run that executed at `max` — the exact defect the feature
|
|
132
143
|
* exists to prevent, reintroduced by the fix for it.
|
|
@@ -269,7 +280,7 @@ function spawnAgentRaw(a) {
|
|
|
269
280
|
(0, foreign_runner_js_1.refuseUnderForeignRunner)("spawning `claude`");
|
|
270
281
|
return new Promise((resolvePromise) => {
|
|
271
282
|
const args = buildAgentArgs(a);
|
|
272
|
-
const child = (0, node_child_process_1.spawn)(
|
|
283
|
+
const child = (0, node_child_process_1.spawn)(EVAL_RUNTIME.agentBinary, args, {
|
|
273
284
|
cwd: a.cwd,
|
|
274
285
|
// The security-critical env resolution (overlay vs. scrubbed replacement)
|
|
275
286
|
// lives in the tested `resolveSpawnEnv` seam above, not inline here.
|
|
@@ -866,7 +877,7 @@ let cachedHarnessVersion;
|
|
|
866
877
|
function harnessVersion() {
|
|
867
878
|
if (cachedHarnessVersion === undefined) {
|
|
868
879
|
try {
|
|
869
|
-
cachedHarnessVersion =
|
|
880
|
+
cachedHarnessVersion = EVAL_RUNTIME.versionKey((0, node_child_process_1.execSync)(`${EVAL_RUNTIME.agentBinary} --version`, {
|
|
870
881
|
encoding: "utf-8",
|
|
871
882
|
stdio: ["ignore", "pipe", "ignore"],
|
|
872
883
|
}));
|
|
@@ -913,132 +924,68 @@ function withInterceptToolHook(settings, intercepts) {
|
|
|
913
924
|
},
|
|
914
925
|
};
|
|
915
926
|
}
|
|
916
|
-
/**
|
|
917
|
-
* The default allowlist `ephemeralRunEnv` passes through from the real
|
|
918
|
-
* environment. Two groups, both load-bearing for a real-model `claude` run:
|
|
919
|
-
*
|
|
920
|
-
* - **Auth** — the harness's OWN credentials. The eval drives the real `claude`
|
|
921
|
-
* CLI, which authenticates via the user's subscription (`~/.claude`, reached
|
|
922
|
-
* through the fresh HOME's allowed config — see below) OR via these env vars.
|
|
923
|
-
* We mirror the auth surface the runtime port already names
|
|
924
|
-
* (`ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL`, see
|
|
925
|
-
* `adapters/claude-code/runtime.ts`) plus the OAuth/token + region variants the
|
|
926
|
-
* CLI accepts, so a token-authed user is not broken by a too-narrow list.
|
|
927
|
-
* - **Runtime** — what any spawned process needs to *function*: `PATH` (resolve
|
|
928
|
-
* `node` / `claude`), the locale/terminal vars (`LANG` / `LC_*` / `TERM`), and
|
|
929
|
-
* `TMPDIR` (which we override to the fresh HOME). Mirrors what `bwrapArgs` /
|
|
930
|
-
* `setenvArgs` in `src/sandbox.ts` set back after `--clearenv`.
|
|
931
|
-
*
|
|
932
|
-
* Notably it does NOT pass through `GIT_*`, `GH_TOKEN`, `SSH_*`, `AWS_*`, or any
|
|
933
|
-
* other non-allowlisted secret-shaped var — those are exactly what an ephemeral
|
|
934
|
-
* run must not see. `CLAUDE_*` is allowlisted by prefix because the CLI reads
|
|
935
|
-
* several `CLAUDE_*` knobs (config dir, etc.) and omitting one is the failure
|
|
936
|
-
* mode this whole guard is conservative against.
|
|
937
|
-
*
|
|
938
|
-
* Conservative by design: a too-broad allowlist is safe (it just leaks a benign
|
|
939
|
-
* var); a too-narrow one silently breaks auth — which is why the feature ships
|
|
940
|
-
* default-OFF until validated against a real run.
|
|
941
|
-
*/
|
|
942
|
-
const EPHEMERAL_ALLOW = [
|
|
943
|
-
// Runtime essentials (mirror sandbox.ts setenv-after-clearenv).
|
|
944
|
-
"PATH",
|
|
945
|
-
"LANG",
|
|
946
|
-
"TERM",
|
|
947
|
-
// Anthropic / Claude Code auth + endpoint (mirror runtime.ts + CLI auth vars).
|
|
948
|
-
runtime_js_1.claudeCodeRuntime.modelApiKeyEnv, // ANTHROPIC_API_KEY
|
|
949
|
-
runtime_js_1.claudeCodeRuntime.modelBaseUrlEnv, // ANTHROPIC_BASE_URL
|
|
950
|
-
"ANTHROPIC_AUTH_TOKEN",
|
|
951
|
-
"ANTHROPIC_API_URL",
|
|
952
|
-
"ANTHROPIC_MODEL",
|
|
953
|
-
"ANTHROPIC_DEFAULT_HEADERS",
|
|
954
|
-
"CLAUDE_CODE_USE_BEDROCK",
|
|
955
|
-
"CLAUDE_CODE_USE_VERTEX",
|
|
956
|
-
// Cloud-provider auth the CLI uses for Bedrock/Vertex backends (region/profile
|
|
957
|
-
// only — NOT the secret-shaped AWS_* access keys, which stay dropped).
|
|
958
|
-
"AWS_REGION",
|
|
959
|
-
"AWS_DEFAULT_REGION",
|
|
960
|
-
"AWS_PROFILE",
|
|
961
|
-
"CLOUD_ML_REGION",
|
|
962
|
-
"GOOGLE_CLOUD_PROJECT",
|
|
963
|
-
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
964
|
-
];
|
|
965
|
-
/** Prefixes passed through wholesale — the CLI reads several `CLAUDE_*` knobs and
|
|
966
|
-
* a `LC_*` locale family; allowlist by prefix so omitting one isn't the silent
|
|
967
|
-
* auth/locale break this guard exists to avoid. */
|
|
968
|
-
const EPHEMERAL_ALLOW_PREFIXES = ["CLAUDE_", "LC_"];
|
|
969
927
|
/**
|
|
970
928
|
* Build an **ephemeral run environment** for a model-driven run: a NEW env object
|
|
971
|
-
* with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`,
|
|
972
|
-
*
|
|
973
|
-
* else
|
|
929
|
+
* with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, the
|
|
930
|
+
* OS essentials (`PATH`, `LANG`, `LC_*`, `TERM`), the harness's OWN auth, and
|
|
931
|
+
* nothing else. Pure — no fs, no spawn.
|
|
974
932
|
*
|
|
975
933
|
* The rationale is "fresh HOME + only the harness credential injected, **not** a
|
|
976
934
|
* blanket wipe": running a model-driven skill/agent is itself a side effect (the
|
|
977
935
|
* *model*, not the author, chose the actions), so it should not be able to read
|
|
978
936
|
* the real `~/.gitconfig` / `~/.ssh` / `~/.aws` or write to the real `~`. But the
|
|
979
|
-
* real
|
|
980
|
-
* (
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
*
|
|
984
|
-
*
|
|
937
|
+
* real CLI must still AUTHENTICATE, so the auth names the harness declares
|
|
938
|
+
* (`runtime.runEnv.keep`) are re-injected, and the parent session's identity
|
|
939
|
+
* (`runtime.runEnv.sessionIdentity`) never is. Which names those are is the
|
|
940
|
+
* adapter's fact, not this file's: see `RunEnvPolicy` in `core/runtime.ts`.
|
|
941
|
+
* Because this needs no kernel features, it is the cross-platform
|
|
942
|
+
* STATE-protection floor (lands on macOS immediately), orthogonal to the Linux
|
|
943
|
+
* bubblewrap HOST confinement in `src/sandbox.ts`.
|
|
944
|
+
*
|
|
945
|
+
* Notably it does NOT pass through `GIT_*`, `GH_TOKEN`, `SSH_*`, `AWS_*` access
|
|
946
|
+
* keys, a config-dir override, or any other undeclared variable — those are
|
|
947
|
+
* exactly what an ephemeral run must not see. A too-narrow `keep` breaks auth
|
|
948
|
+
* loudly (the run fails to start); a too-broad one leaks silently, which is why
|
|
949
|
+
* it is an allowlist and not a prefix.
|
|
985
950
|
*
|
|
986
951
|
* @param base the source environment to filter (usually `process.env`).
|
|
987
952
|
* @param opts.home the throwaway dir to set as `HOME`/`TMPDIR`.
|
|
988
953
|
* @param opts.allow extra var NAMES to pass through (e.g. the `VIGILES_*` keys
|
|
989
|
-
* the eval already injects). Layered ON TOP of the
|
|
954
|
+
* the eval already injects). Layered ON TOP of the harness's `keep`; a
|
|
955
|
+
* session-identity name is dropped even when listed here.
|
|
990
956
|
*/
|
|
991
957
|
function ephemeralRunEnv(base, opts) {
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
...
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
const allowed = allowExact.has(k) ||
|
|
1001
|
-
EPHEMERAL_ALLOW_PREFIXES.some((p) => k.startsWith(p));
|
|
1002
|
-
if (allowed)
|
|
1003
|
-
out[k] = v;
|
|
1004
|
-
}
|
|
1005
|
-
// Fresh HOME + TMPDIR last so they always win over anything passed through.
|
|
1006
|
-
out.HOME = opts.home;
|
|
1007
|
-
out.TMPDIR = opts.home;
|
|
1008
|
-
return out;
|
|
958
|
+
// A fresh object: the trial layers its overlay and stub PATH onto it.
|
|
959
|
+
return {
|
|
960
|
+
...(0, run_env_js_1.scrubbedRunEnv)(base, {
|
|
961
|
+
home: opts.home,
|
|
962
|
+
policy: EVAL_RUNTIME.runEnv,
|
|
963
|
+
allow: opts.allow,
|
|
964
|
+
}),
|
|
965
|
+
};
|
|
1009
966
|
}
|
|
1010
|
-
/**
|
|
1011
|
-
* Home-relative AUTH files to carry from the real HOME into the throwaway one.
|
|
1012
|
-
*
|
|
1013
|
-
* A local subscription credential (OAuth token) often lives in a FILE under HOME,
|
|
1014
|
-
* not an env var — so scrubbing HOME would lose it and break a local-authed run.
|
|
1015
|
-
* This is the file half of the auth allowlist; {@link EPHEMERAL_ALLOW} covers the
|
|
1016
|
-
* env-var / host-brokered half. Kept a named constant so it's easy to extend, and
|
|
1017
|
-
* deliberately NARROW — only the explicit auth files, never `.gitconfig` / `.ssh`
|
|
1018
|
-
* / `.aws`, which are exactly what an ephemeral run must not see.
|
|
1019
|
-
*/
|
|
1020
|
-
exports.EPHEMERAL_HOME_KEEP = [
|
|
1021
|
-
".claude/.credentials.json", // the Claude Code OAuth token
|
|
1022
|
-
];
|
|
1023
967
|
/**
|
|
1024
968
|
* Seed the throwaway HOME with the harness's own auth FILE(s) — best-effort;
|
|
1025
|
-
* covers local file-based OAuth; the env-var/host-brokered path is covered by
|
|
1026
|
-
*
|
|
969
|
+
* covers local file-based OAuth; the env-var/host-brokered path is covered by
|
|
970
|
+
* `ephemeralRunEnv`. The files are the runtime's declared `keepHomeFiles`.
|
|
1027
971
|
*
|
|
1028
|
-
* COPIES (never symlinks) each
|
|
1029
|
-
*
|
|
1030
|
-
*
|
|
1031
|
-
*
|
|
1032
|
-
*
|
|
972
|
+
* COPIES (never symlinks) each path from `realHome` into `throwawayHome`,
|
|
973
|
+
* creating parent dirs as needed; a symlink would let the model-driven run write
|
|
974
|
+
* back to the real credential file, defeating ephemerality. A path that doesn't
|
|
975
|
+
* exist in the real HOME is skipped silently (that user auths via env-var / host
|
|
976
|
+
* broker instead). Pure fs — no env, no spawn.
|
|
1033
977
|
*/
|
|
1034
|
-
function seedEphemeralHome(throwawayHome, realHome, keep =
|
|
978
|
+
function seedEphemeralHome(throwawayHome, realHome, keep = EVAL_RUNTIME.runEnv?.keepHomeFiles ?? []) {
|
|
1035
979
|
for (const rel of keep) {
|
|
1036
980
|
const src = (0, node_path_1.join)(realHome, rel);
|
|
1037
981
|
if (!(0, node_fs_1.existsSync)(src))
|
|
1038
982
|
continue; // env-var / host-brokered auth covers this.
|
|
1039
983
|
const dest = (0, node_path_1.join)(throwawayHome, rel);
|
|
1040
984
|
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(dest), { recursive: true });
|
|
1041
|
-
|
|
985
|
+
// The DATA, never a link: `cpSync` recreates a symlinked credential
|
|
986
|
+
// (dotfiles, a secret manager) as a link, and the run would write through
|
|
987
|
+
// it into the real file. `copyFileSync` always copies the contents.
|
|
988
|
+
(0, node_fs_1.copyFileSync)(src, dest);
|
|
1042
989
|
}
|
|
1043
990
|
}
|
|
1044
991
|
/** Execute one trial in a fresh sandbox; returns its metric row + usage. */
|
package/dist/harness-test.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ export type { Trace, SubagentTrace } from "./core/eval-driver.js";
|
|
|
5
5
|
import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js";
|
|
6
6
|
/** Re-exported for the eval tier's parser, which reads the same stream. */
|
|
7
7
|
export { parseReplies, reachedModel };
|
|
8
|
+
import type { HarnessRuntime } from "./core/runtime.js";
|
|
8
9
|
import { type SandboxMode } from "./sandbox.js";
|
|
9
10
|
export { scriptModel } from "./mock-model.js";
|
|
10
11
|
export type { ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, } from "./core/harness-driver.js";
|
|
@@ -126,6 +127,31 @@ export declare function parseHooks(stdout: string): HookFire[];
|
|
|
126
127
|
export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[];
|
|
127
128
|
/** Parse the `claude` stdout/stream into the unified trace fields. */
|
|
128
129
|
export declare function parseClaudeRun(stdout: string): ParsedRun;
|
|
130
|
+
/**
|
|
131
|
+
* Where a harness-tier run's agent gets HOME and the rest of its environment.
|
|
132
|
+
*
|
|
133
|
+
* - `inherit` — the caller's env (real HOME), minus the harness's declared
|
|
134
|
+
* session identity. The default of `runHarnessTest`: a test that reads the
|
|
135
|
+
* machine's config keeps doing so, but no run executes AS the caller's live
|
|
136
|
+
* session.
|
|
137
|
+
* - `throwaway` — a fresh HOME/TMPDIR at `dir`, the OS essentials, and nothing
|
|
138
|
+
* else from the caller: no harness auth (the scripted mock needs none, and a
|
|
139
|
+
* config-dir variable would point the run back at the real config), no
|
|
140
|
+
* identity, no secrets. The same scrubbed shape as the eval tier's
|
|
141
|
+
* `ephemeralEnv` (`scrubbedRunEnv`), without the auth.
|
|
142
|
+
*/
|
|
143
|
+
export type HarnessRunHome = {
|
|
144
|
+
readonly home: "inherit";
|
|
145
|
+
} | {
|
|
146
|
+
readonly home: "throwaway";
|
|
147
|
+
readonly dir: string;
|
|
148
|
+
};
|
|
149
|
+
/**
|
|
150
|
+
* The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
|
|
151
|
+
* `base`, with the runtime's mock wiring on top. Pure — the testable seam of
|
|
152
|
+
* the un-coverable spawn.
|
|
153
|
+
*/
|
|
154
|
+
export declare function harnessSpawnEnv(runtime: HarnessRuntime, wiredEnv: Readonly<Record<string, string>>, home: HarnessRunHome, base?: NodeJS.ProcessEnv): Readonly<Record<string, string>>;
|
|
129
155
|
/** Whether the agent CLI is available — harness tests need it. */
|
|
130
156
|
export declare function claudeAvailable(): boolean;
|
|
131
157
|
/**
|
|
@@ -157,6 +183,15 @@ export interface RunHarnessTestOptions {
|
|
|
157
183
|
* only — requesting confinement for another harness throws.
|
|
158
184
|
*/
|
|
159
185
|
export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise<HarnessTestResult>;
|
|
186
|
+
/**
|
|
187
|
+
* {@link runHarnessTest} with the run's HOME chosen by the caller — see
|
|
188
|
+
* {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
|
|
189
|
+
* a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
|
|
190
|
+
* run must not depend on, or act as, the machine it happens to run on. The
|
|
191
|
+
* confined (bubblewrap) path ignores `home`: it already runs with its own HOME
|
|
192
|
+
* and a cleared env.
|
|
193
|
+
*/
|
|
194
|
+
export declare function runHarnessTestIn(spec: HarnessTestSpec, opts: RunHarnessTestOptions, home: HarnessRunHome["home"]): Promise<HarnessTestResult>;
|
|
160
195
|
/**
|
|
161
196
|
* `runHarness` — the harness-scope entry of the revamped API (Phase 2 of
|
|
162
197
|
* `research/testing-api-design.md`). The harness has two execution scopes, `hook`
|
package/dist/harness-test.js
CHANGED
|
@@ -8,8 +8,10 @@ exports.parseOutput = parseOutput;
|
|
|
8
8
|
exports.parseHooks = parseHooks;
|
|
9
9
|
exports.buildClaudeArgs = buildClaudeArgs;
|
|
10
10
|
exports.parseClaudeRun = parseClaudeRun;
|
|
11
|
+
exports.harnessSpawnEnv = harnessSpawnEnv;
|
|
11
12
|
exports.claudeAvailable = claudeAvailable;
|
|
12
13
|
exports.runHarnessTest = runHarnessTest;
|
|
14
|
+
exports.runHarnessTestIn = runHarnessTestIn;
|
|
13
15
|
exports.runHarness = runHarness;
|
|
14
16
|
exports.outputStyleFixture = outputStyleFixture;
|
|
15
17
|
/**
|
|
@@ -55,6 +57,7 @@ Object.defineProperty(exports, "parseReplies", { enumerable: true, get: function
|
|
|
55
57
|
Object.defineProperty(exports, "reachedModel", { enumerable: true, get: function () { return replies_js_1.reachedModel; } });
|
|
56
58
|
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
57
59
|
const output_style_js_1 = require("./core/output-style.js");
|
|
60
|
+
const run_env_js_1 = require("./core/run-env.js");
|
|
58
61
|
const sandbox_js_1 = require("./sandbox.js");
|
|
59
62
|
var mock_model_js_2 = require("./mock-model.js");
|
|
60
63
|
Object.defineProperty(exports, "scriptModel", { enumerable: true, get: function () { return mock_model_js_2.scriptModel; } });
|
|
@@ -314,6 +317,19 @@ function parseClaudeRun(stdout) {
|
|
|
314
317
|
replies: (0, replies_js_1.parseReplies)(stdout),
|
|
315
318
|
};
|
|
316
319
|
}
|
|
320
|
+
/**
|
|
321
|
+
* The spawn env of one harness-tier run: {@link HarnessRunHome} applied to
|
|
322
|
+
* `base`, with the runtime's mock wiring on top. Pure — the testable seam of
|
|
323
|
+
* the un-coverable spawn.
|
|
324
|
+
*/
|
|
325
|
+
function harnessSpawnEnv(runtime, wiredEnv, home, base = process.env) {
|
|
326
|
+
switch (home.home) {
|
|
327
|
+
case "inherit":
|
|
328
|
+
return { ...(0, run_env_js_1.withoutSessionIdentity)(base, runtime.runEnv), ...wiredEnv };
|
|
329
|
+
case "throwaway":
|
|
330
|
+
return { ...(0, run_env_js_1.scrubbedRunEnv)(base, { home: home.dir }), ...wiredEnv };
|
|
331
|
+
}
|
|
332
|
+
}
|
|
317
333
|
/* v8 ignore start -- spawns the real claude CLI + filesystem; exercised by the
|
|
318
334
|
claude-backed suite, excluded from the deterministic coverage gate (the parse
|
|
319
335
|
helpers above carry the testable logic). */
|
|
@@ -345,13 +361,24 @@ function writeFixture(cwd, files, settings) {
|
|
|
345
361
|
(0, node_fs_1.writeFileSync)((0, node_path_1.join)(cwd, "settings.json"), json);
|
|
346
362
|
}
|
|
347
363
|
}
|
|
364
|
+
/**
|
|
365
|
+
* The {@link HarnessRunHome} for a run in `cwd`. A throwaway HOME lives under
|
|
366
|
+
* the run's own temp dir, as the eval tier's does, so `cleanup()` removes it
|
|
367
|
+
* with everything else.
|
|
368
|
+
*/
|
|
369
|
+
function runHomeIn(cwd, home) {
|
|
370
|
+
return home === "throwaway"
|
|
371
|
+
? { home, dir: (0, node_fs_1.mkdtempSync)((0, node_path_1.join)(cwd, "home-")) }
|
|
372
|
+
: { home };
|
|
373
|
+
}
|
|
348
374
|
/**
|
|
349
375
|
* Spawn the agent binary against the mock. The mock-wiring *args* are already in
|
|
350
376
|
* `args` (the driver placed `wireMock(url).args` at the correct argv position
|
|
351
|
-
* via `ctx.mockArgs`); here
|
|
352
|
-
* env. Driver-agnostic at the transport seam.
|
|
377
|
+
* via `ctx.mockArgs`); here the env is {@link harnessSpawnEnv} of the caller's,
|
|
378
|
+
* with `wireMock(url).env` on top. Driver-agnostic at the transport seam.
|
|
353
379
|
*/
|
|
354
|
-
function spawnAgent(runtime, args,
|
|
380
|
+
function spawnAgent(runtime, args, run) {
|
|
381
|
+
const { cwd, baseUrl, timeoutMs, home } = run;
|
|
355
382
|
const wired = runtime.wireMock(baseUrl);
|
|
356
383
|
return new Promise((resolvePromise) => {
|
|
357
384
|
// vigiles:free-tier — the deterministic harness tier. `runtime.wireMock`
|
|
@@ -361,9 +388,8 @@ function spawnAgent(runtime, args, cwd, baseUrl, timeoutMs) {
|
|
|
361
388
|
const child = (0, node_child_process_1.spawn)(runtime.agentBinary, [...args], {
|
|
362
389
|
cwd,
|
|
363
390
|
// Any key works — the mock ignores auth. wireMock supplies the overlay
|
|
364
|
-
// env (base-URL var + dummy key for CC; the dummy key for Codex)
|
|
365
|
-
|
|
366
|
-
env: { ...process.env, ...wired.env },
|
|
391
|
+
// env (base-URL var + dummy key for CC; the dummy key for Codex).
|
|
392
|
+
env: harnessSpawnEnv(runtime, wired.env, home),
|
|
367
393
|
stdio: ["ignore", "pipe", "pipe"],
|
|
368
394
|
});
|
|
369
395
|
let stdout = "";
|
|
@@ -431,6 +457,17 @@ function makeResult(cwd, out, parsed, turns, modelRequests) {
|
|
|
431
457
|
* only — requesting confinement for another harness throws.
|
|
432
458
|
*/
|
|
433
459
|
async function runHarnessTest(spec, opts = {}) {
|
|
460
|
+
return runHarnessTestIn(spec, opts, "inherit");
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* {@link runHarnessTest} with the run's HOME chosen by the caller — see
|
|
464
|
+
* {@link HarnessRunHome}. Not on a public entry point: the one caller that needs
|
|
465
|
+
* a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose
|
|
466
|
+
* run must not depend on, or act as, the machine it happens to run on. The
|
|
467
|
+
* confined (bubblewrap) path ignores `home`: it already runs with its own HOME
|
|
468
|
+
* and a cleared env.
|
|
469
|
+
*/
|
|
470
|
+
async function runHarnessTestIn(spec, opts, home) {
|
|
434
471
|
// Tell the CLI runner this script exercised the harness, so a file that runs
|
|
435
472
|
// NOTHING can be told apart from one that ran and passed. See check-count.ts.
|
|
436
473
|
(0, check_count_js_1.recordCheck)();
|
|
@@ -492,7 +529,12 @@ async function runHarnessTest(spec, opts = {}) {
|
|
|
492
529
|
const mock = await driver.startMock(spec.model);
|
|
493
530
|
try {
|
|
494
531
|
const args = buildArgs(driver.runtime.wireMock(mock.url).args);
|
|
495
|
-
const out = await spawnAgent(driver.runtime, args,
|
|
532
|
+
const out = await spawnAgent(driver.runtime, args, {
|
|
533
|
+
cwd,
|
|
534
|
+
baseUrl: mock.url,
|
|
535
|
+
timeoutMs,
|
|
536
|
+
home: runHomeIn(cwd, home),
|
|
537
|
+
});
|
|
496
538
|
// A script that was never consumed is otherwise invisible — the run just
|
|
497
539
|
// looks empty. `scriptUnconsumedWarning` names the one shape that produces
|
|
498
540
|
// it (every request arriving without tool declarations, so nothing looked
|
|
@@ -14,7 +14,8 @@ exports.outputStyleArmsWith = outputStyleArmsWith;
|
|
|
14
14
|
* all: it drives the real API, so no request is captured. The free run here
|
|
15
15
|
* uses the scripted model, where requests ARE captured, with the same fixture
|
|
16
16
|
* the paid arm gets — so a style that would not load stops the eval before the
|
|
17
|
-
* first billed trial.
|
|
17
|
+
* first billed trial. That free run uses a throwaway HOME and none of the
|
|
18
|
+
* caller's environment beyond the OS essentials.
|
|
18
19
|
*/
|
|
19
20
|
const adapter_registry_js_1 = require("./adapter-registry.js");
|
|
20
21
|
const harness_test_js_1 = require("./harness-test.js");
|
|
@@ -37,7 +38,11 @@ async function experimental_outputStyleArms(path) {
|
|
|
37
38
|
* run by a harness they were not checked on.
|
|
38
39
|
*/
|
|
39
40
|
async function outputStyleArmsWith(path, adapter) {
|
|
40
|
-
|
|
41
|
+
// A throwaway HOME: this run happens on every import of an eval file (even
|
|
42
|
+
// `vigiles eval --check`), so it must not read the machine's user config or
|
|
43
|
+
// run as the session that imported it. What it proves — the style reaches the
|
|
44
|
+
// model — is a fact about the fixture, not about this machine.
|
|
45
|
+
const preflight = await (0, harness_test_js_1.runHarnessTestIn)({ outputStyle: path, prompt: "hi", model: [{ text: "ok" }] }, { adapter }, "throwaway");
|
|
41
46
|
preflight.cleanup();
|
|
42
47
|
const { files, settings } = (0, harness_test_js_1.outputStyleFixture)(path, adapter);
|
|
43
48
|
const none = adapter.layout.outputStyles?.selectNone ?? {};
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
* boundary is deliberate: plugins "can execute arbitrary code on your machine
|
|
18
18
|
* with your user privileges", so a repo is not allowed to install one for you.
|
|
19
19
|
*
|
|
20
|
-
* The declaration buys exactly one thing
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
20
|
+
* The declaration buys exactly one thing: a collaborator who clones the repo is
|
|
21
|
+
* told the project wants this plugin, with the command that installs it.
|
|
22
|
+
* **Silent absence becomes a prompt.** It does not become a working install. What
|
|
23
|
+
* the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
|
|
24
|
+
* any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
|
|
25
|
+
* the links travel with the clone.
|
|
26
26
|
*
|
|
27
27
|
* Nothing is vendored: what lands in the repo is a *reference* (a marketplace
|
|
28
28
|
* source plus an enabled flag). The plugin content still lives in the global
|
|
@@ -18,12 +18,12 @@
|
|
|
18
18
|
* boundary is deliberate: plugins "can execute arbitrary code on your machine
|
|
19
19
|
* with your user privileges", so a repo is not allowed to install one for you.
|
|
20
20
|
*
|
|
21
|
-
* The declaration buys exactly one thing
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
21
|
+
* The declaration buys exactly one thing: a collaborator who clones the repo is
|
|
22
|
+
* told the project wants this plugin, with the command that installs it.
|
|
23
|
+
* **Silent absence becomes a prompt.** It does not become a working install. What
|
|
24
|
+
* the plugin adds on their machine is the HOOKS; the SKILLS do not depend on it
|
|
25
|
+
* any more — `init` links them into `.claude/skills/` (`./skill-links.ts`), and
|
|
26
|
+
* the links travel with the clone.
|
|
27
27
|
*
|
|
28
28
|
* Nothing is vendored: what lands in the repo is a *reference* (a marketplace
|
|
29
29
|
* source plus an enabled flag). The plugin content still lives in the global
|