vigiles 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -378
- package/dist/claude-code.d.ts +10 -0
- package/dist/claude-code.js +26 -0
- package/dist/egress-proxy.d.ts +2 -0
- package/dist/egress-proxy.js +60 -0
- package/dist/eval-baseline.d.ts +68 -0
- package/dist/eval-baseline.js +173 -0
- package/dist/harness-assert.d.ts +35 -1
- package/dist/harness-assert.js +76 -1
- package/dist/linting.d.ts +9 -0
- package/dist/linting.js +25 -0
- package/dist/run-hook.d.ts +81 -1
- package/dist/run-hook.js +189 -11
- package/dist/sandbox.d.ts +35 -4
- package/dist/sandbox.js +71 -5
- package/dist/testing.d.ts +12 -0
- package/dist/testing.js +28 -0
- package/package.json +6 -2
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { EvalReport } from "./eval.js";
|
|
2
|
+
import { type Comparison } from "./stats.js";
|
|
3
|
+
/** Bumped only on a breaking change to the on-disk shape. */
|
|
4
|
+
export declare const BASELINE_VERSION = 1;
|
|
5
|
+
/** The committed baseline: the recorded `EvalReport`s, keyed by report name. */
|
|
6
|
+
export interface BaselineFile {
|
|
7
|
+
readonly version: number;
|
|
8
|
+
/** ISO-8601 timestamp the baseline was recorded (provenance / future trend). */
|
|
9
|
+
readonly recordedAt: string;
|
|
10
|
+
/** Recorded reports, keyed by `report.name` (so multiple eval files coexist). */
|
|
11
|
+
readonly reports: Record<string, EvalReport>;
|
|
12
|
+
}
|
|
13
|
+
/** How a metric moved between baseline and current run. */
|
|
14
|
+
export type DiffStatus = "regressed" | "improved" | "unchanged";
|
|
15
|
+
/** One arm×metric comparison of a current run against the baseline. */
|
|
16
|
+
export interface MetricDiff {
|
|
17
|
+
/** The `report.name` this entry belongs to. */
|
|
18
|
+
readonly report: string;
|
|
19
|
+
readonly arm: string;
|
|
20
|
+
readonly metric: string;
|
|
21
|
+
readonly status: DiffStatus;
|
|
22
|
+
/** Welch comparison, current vs. baseline (`delta = current − baseline`). */
|
|
23
|
+
readonly comparison: Comparison;
|
|
24
|
+
}
|
|
25
|
+
export interface BaselineDiff {
|
|
26
|
+
/** Every arm×metric present in BOTH the baseline and the current run. */
|
|
27
|
+
readonly entries: readonly MetricDiff[];
|
|
28
|
+
/** The subset that regressed (significant move in the bad direction). */
|
|
29
|
+
readonly regressions: readonly MetricDiff[];
|
|
30
|
+
/** The subset that improved (significant move in the good direction). */
|
|
31
|
+
readonly improvements: readonly MetricDiff[];
|
|
32
|
+
/** True when there are no regressions — the gate. */
|
|
33
|
+
readonly passed: boolean;
|
|
34
|
+
}
|
|
35
|
+
export interface DiffOptions {
|
|
36
|
+
/** Significance level for the Welch test. Default 0.05. */
|
|
37
|
+
readonly alpha?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Metrics where a DECREASE is the improvement (e.g. `cost`, `latency`,
|
|
40
|
+
* `turns`). For these, a significant increase is the regression. Everything
|
|
41
|
+
* else is treated as higher-is-better.
|
|
42
|
+
*/
|
|
43
|
+
readonly lowerIsBetter?: readonly string[];
|
|
44
|
+
}
|
|
45
|
+
/** Build a `BaselineFile` envelope from a run's reports (keyed by name). */
|
|
46
|
+
export declare function toBaselineFile(reports: readonly EvalReport[], recordedAt?: string): BaselineFile;
|
|
47
|
+
/** Parse + validate a baseline JSON string (throws on a bad version/shape). */
|
|
48
|
+
export declare function parseBaselineFile(json: string): BaselineFile;
|
|
49
|
+
/**
|
|
50
|
+
* Diff a current run against a committed baseline. Compares every arm×metric
|
|
51
|
+
* present in both (by report name), flagging a *significant* move in the
|
|
52
|
+
* undesired direction as a regression. Metrics absent from one side are skipped
|
|
53
|
+
* (a new arm/metric is not a regression).
|
|
54
|
+
*/
|
|
55
|
+
export declare function diffReports(baseline: BaselineFile, current: readonly EvalReport[], opts?: DiffOptions): BaselineDiff;
|
|
56
|
+
/** Format a baseline diff as a compact console report. */
|
|
57
|
+
export declare function formatBaselineDiff(diff: BaselineDiff): string;
|
|
58
|
+
/**
|
|
59
|
+
* Render a baseline diff as JUnit XML — one `<testcase>` per arm×metric, a
|
|
60
|
+
* `<failure>` for each regression. Lets a CI provider show eval regressions in
|
|
61
|
+
* the same place as unit-test failures.
|
|
62
|
+
*/
|
|
63
|
+
export declare function diffToJUnit(diff: BaselineDiff): string;
|
|
64
|
+
/** Read + parse a baseline file, or null if it doesn't exist yet. */
|
|
65
|
+
export declare function readBaseline(path: string): BaselineFile | null;
|
|
66
|
+
/** Write reports as the committed baseline (pretty JSON, parent dirs created). */
|
|
67
|
+
export declare function writeBaseline(path: string, reports: readonly EvalReport[]): void;
|
|
68
|
+
//# sourceMappingURL=eval-baseline.d.ts.map
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BASELINE_VERSION = void 0;
|
|
4
|
+
exports.toBaselineFile = toBaselineFile;
|
|
5
|
+
exports.parseBaselineFile = parseBaselineFile;
|
|
6
|
+
exports.diffReports = diffReports;
|
|
7
|
+
exports.formatBaselineDiff = formatBaselineDiff;
|
|
8
|
+
exports.diffToJUnit = diffToJUnit;
|
|
9
|
+
exports.readBaseline = readBaseline;
|
|
10
|
+
exports.writeBaseline = writeBaseline;
|
|
11
|
+
/**
|
|
12
|
+
* vigiles — eval regression gating (Phase C).
|
|
13
|
+
*
|
|
14
|
+
* The eval tier reports mean ± se per arm; `src/stats.ts` turns a gap into a
|
|
15
|
+
* significance verdict. This module points that machinery at a *committed
|
|
16
|
+
* baseline*: record one run's `EvalReport`s to `.vigiles/eval-baseline.json`,
|
|
17
|
+
* then on a later run flag any arm×metric that moved **significantly in the bad
|
|
18
|
+
* direction** vs. that baseline. "jest snapshots for agent behaviour, with a real
|
|
19
|
+
* noise floor" — a bare pass-rate can't tell a true regression from sampling
|
|
20
|
+
* noise, but a Welch t-test over the two runs' summary stats can.
|
|
21
|
+
*
|
|
22
|
+
* Pure + model-free (the diff/serialize/JUnit are fully unit-tested); the only
|
|
23
|
+
* side effects are the two small fs helpers (`readBaseline` / `writeBaseline`).
|
|
24
|
+
* Reuses `welchTTest` from `src/stats.ts` — the current run is the "arm", the
|
|
25
|
+
* baseline is the "baseline", so `delta = current − baseline`.
|
|
26
|
+
*/
|
|
27
|
+
const node_fs_1 = require("node:fs");
|
|
28
|
+
const node_path_1 = require("node:path");
|
|
29
|
+
const stats_js_1 = require("./stats.js");
|
|
30
|
+
/** Bumped only on a breaking change to the on-disk shape. */
|
|
31
|
+
exports.BASELINE_VERSION = 1;
|
|
32
|
+
/** Build a `BaselineFile` envelope from a run's reports (keyed by name). */
|
|
33
|
+
function toBaselineFile(reports, recordedAt = new Date().toISOString()) {
|
|
34
|
+
const byName = {};
|
|
35
|
+
for (const r of reports)
|
|
36
|
+
byName[r.name] = r;
|
|
37
|
+
return { version: exports.BASELINE_VERSION, recordedAt, reports: byName };
|
|
38
|
+
}
|
|
39
|
+
/** Parse + validate a baseline JSON string (throws on a bad version/shape). */
|
|
40
|
+
function parseBaselineFile(json) {
|
|
41
|
+
const data = JSON.parse(json);
|
|
42
|
+
if (typeof data !== "object" || data === null) {
|
|
43
|
+
throw new Error("baseline: expected a JSON object");
|
|
44
|
+
}
|
|
45
|
+
const obj = data;
|
|
46
|
+
if (obj.version !== exports.BASELINE_VERSION) {
|
|
47
|
+
throw new Error(`baseline: unsupported version ${String(obj.version)} (expected ${String(exports.BASELINE_VERSION)})`);
|
|
48
|
+
}
|
|
49
|
+
if (typeof obj.reports !== "object" || obj.reports === null) {
|
|
50
|
+
throw new Error("baseline: missing `reports`");
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
version: exports.BASELINE_VERSION,
|
|
54
|
+
recordedAt: typeof obj.recordedAt === "string" ? obj.recordedAt : "",
|
|
55
|
+
reports: obj.reports,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Classify one comparison given the metric's direction. */
|
|
59
|
+
function classify(cmp, lowerIsBetter) {
|
|
60
|
+
if (!cmp.significant || cmp.delta === 0)
|
|
61
|
+
return "unchanged";
|
|
62
|
+
const improved = lowerIsBetter ? cmp.delta < 0 : cmp.delta > 0;
|
|
63
|
+
return improved ? "improved" : "regressed";
|
|
64
|
+
}
|
|
65
|
+
/** Append a diff entry for every arm×metric common to both reports. */
|
|
66
|
+
function collectReportDiffs(baseline, current, cfg, out) {
|
|
67
|
+
for (const [arm, curArm] of Object.entries(current.arms)) {
|
|
68
|
+
const baseArm = baseline.arms[arm];
|
|
69
|
+
if (!baseArm)
|
|
70
|
+
continue;
|
|
71
|
+
for (const [metric, curStat] of Object.entries(curArm.stats)) {
|
|
72
|
+
const baseStat = baseArm.stats[metric];
|
|
73
|
+
if (!baseStat)
|
|
74
|
+
continue;
|
|
75
|
+
const comparison = (0, stats_js_1.welchTTest)(curStat, baseStat, cfg.alpha);
|
|
76
|
+
out.push({
|
|
77
|
+
report: current.name,
|
|
78
|
+
arm,
|
|
79
|
+
metric,
|
|
80
|
+
status: classify(comparison, cfg.lower.has(metric)),
|
|
81
|
+
comparison,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Diff a current run against a committed baseline. Compares every arm×metric
|
|
88
|
+
* present in both (by report name), flagging a *significant* move in the
|
|
89
|
+
* undesired direction as a regression. Metrics absent from one side are skipped
|
|
90
|
+
* (a new arm/metric is not a regression).
|
|
91
|
+
*/
|
|
92
|
+
function diffReports(baseline, current, opts = {}) {
|
|
93
|
+
const cfg = {
|
|
94
|
+
alpha: opts.alpha ?? 0.05,
|
|
95
|
+
lower: new Set(opts.lowerIsBetter ?? []),
|
|
96
|
+
};
|
|
97
|
+
const entries = [];
|
|
98
|
+
for (const cur of current) {
|
|
99
|
+
const base = baseline.reports[cur.name];
|
|
100
|
+
if (base)
|
|
101
|
+
collectReportDiffs(base, cur, cfg, entries);
|
|
102
|
+
}
|
|
103
|
+
const regressions = entries.filter((e) => e.status === "regressed");
|
|
104
|
+
const improvements = entries.filter((e) => e.status === "improved");
|
|
105
|
+
return {
|
|
106
|
+
entries,
|
|
107
|
+
regressions,
|
|
108
|
+
improvements,
|
|
109
|
+
passed: regressions.length === 0,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
const STATUS_MARK = {
|
|
113
|
+
regressed: "✗",
|
|
114
|
+
improved: "✓",
|
|
115
|
+
unchanged: "·",
|
|
116
|
+
};
|
|
117
|
+
function formatDelta(c) {
|
|
118
|
+
const sign = c.delta >= 0 ? "+" : "";
|
|
119
|
+
return `Δ=${sign}${c.delta.toFixed(3)} p=${c.pValue.toFixed(3)}`;
|
|
120
|
+
}
|
|
121
|
+
/** Format a baseline diff as a compact console report. */
|
|
122
|
+
function formatBaselineDiff(diff) {
|
|
123
|
+
const head = diff.passed
|
|
124
|
+
? "baseline OK — no significant regressions"
|
|
125
|
+
: `baseline FAIL — ${String(diff.regressions.length)} regression(s)`;
|
|
126
|
+
const lines = [head];
|
|
127
|
+
for (const e of diff.entries) {
|
|
128
|
+
lines.push(` ${STATUS_MARK[e.status]} ${e.report}/${e.arm}/${e.metric} ${formatDelta(e.comparison)}`);
|
|
129
|
+
}
|
|
130
|
+
return lines.join("\n");
|
|
131
|
+
}
|
|
132
|
+
function xmlEscape(s) {
|
|
133
|
+
return s
|
|
134
|
+
.replaceAll("&", "&")
|
|
135
|
+
.replaceAll("<", "<")
|
|
136
|
+
.replaceAll(">", ">")
|
|
137
|
+
.replaceAll('"', """);
|
|
138
|
+
}
|
|
139
|
+
function junitCase(e) {
|
|
140
|
+
const name = xmlEscape(`${e.report}.${e.arm}.${e.metric}`);
|
|
141
|
+
const open = ` <testcase classname="${xmlEscape(e.report)}" name="${name}">`;
|
|
142
|
+
if (e.status !== "regressed")
|
|
143
|
+
return `${open}</testcase>`;
|
|
144
|
+
const msg = xmlEscape(`regression: ${formatDelta(e.comparison)}`);
|
|
145
|
+
return `${open}\n <failure message="${msg}"/>\n </testcase>`;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Render a baseline diff as JUnit XML — one `<testcase>` per arm×metric, a
|
|
149
|
+
* `<failure>` for each regression. Lets a CI provider show eval regressions in
|
|
150
|
+
* the same place as unit-test failures.
|
|
151
|
+
*/
|
|
152
|
+
function diffToJUnit(diff) {
|
|
153
|
+
const cases = diff.entries.map(junitCase).join("\n");
|
|
154
|
+
return [
|
|
155
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
156
|
+
`<testsuite name="vigiles-eval" tests="${String(diff.entries.length)}" failures="${String(diff.regressions.length)}">`,
|
|
157
|
+
cases,
|
|
158
|
+
"</testsuite>",
|
|
159
|
+
"",
|
|
160
|
+
].join("\n");
|
|
161
|
+
}
|
|
162
|
+
/** Read + parse a baseline file, or null if it doesn't exist yet. */
|
|
163
|
+
function readBaseline(path) {
|
|
164
|
+
if (!(0, node_fs_1.existsSync)(path))
|
|
165
|
+
return null;
|
|
166
|
+
return parseBaselineFile((0, node_fs_1.readFileSync)(path, "utf-8"));
|
|
167
|
+
}
|
|
168
|
+
/** Write reports as the committed baseline (pretty JSON, parent dirs created). */
|
|
169
|
+
function writeBaseline(path, reports) {
|
|
170
|
+
(0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
|
|
171
|
+
(0, node_fs_1.writeFileSync)(path, JSON.stringify(toBaselineFile(reports), null, 2) + "\n");
|
|
172
|
+
}
|
|
173
|
+
//# sourceMappingURL=eval-baseline.js.map
|
package/dist/harness-assert.d.ts
CHANGED
|
@@ -15,11 +15,14 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
|
|
17
17
|
import type { EvalReport, TriggerRateReport } from "./eval.js";
|
|
18
|
-
import type { HookRunResult } from "./run-hook.js";
|
|
18
|
+
import type { HookRunResult, EgressAttempt } from "./run-hook.js";
|
|
19
19
|
import type { OutputContract } from "./spec.js";
|
|
20
20
|
import { type ParsedAgentResult } from "./agent-result.js";
|
|
21
|
+
import { type BaselineFile, type DiffOptions } from "./eval-baseline.js";
|
|
21
22
|
export { compareArms } from "./stats.js";
|
|
22
23
|
export type { Comparison } from "./stats.js";
|
|
24
|
+
export { diffReports, toBaselineFile, parseBaselineFile, readBaseline, writeBaseline, formatBaselineDiff, diffToJUnit, } from "./eval-baseline.js";
|
|
25
|
+
export type { BaselineFile, BaselineDiff, MetricDiff, DiffStatus, DiffOptions, } from "./eval-baseline.js";
|
|
23
26
|
/**
|
|
24
27
|
* Run a harness test, hand the result to `fn`, and always clean up the sandbox.
|
|
25
28
|
* Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
|
|
@@ -36,6 +39,28 @@ export declare function assertServedTurns(r: HarnessTestResult, n: number): void
|
|
|
36
39
|
export declare function assertHookBlocked(r: HookRunResult): void;
|
|
37
40
|
/** Assert a `runHook` result allowed (did not block). */
|
|
38
41
|
export declare function assertHookAllowed(r: HookRunResult): void;
|
|
42
|
+
/** Anything carrying recorded egress attempts (a runHook recordEgress result). */
|
|
43
|
+
interface HasEgress {
|
|
44
|
+
readonly egress: readonly EgressAttempt[];
|
|
45
|
+
}
|
|
46
|
+
/** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
|
|
47
|
+
export declare function egressHosts(r: HasEgress): string[];
|
|
48
|
+
/** Assert the confined run made NO network egress attempt at all. */
|
|
49
|
+
export declare function assertNoEgress(r: HasEgress): void;
|
|
50
|
+
/**
|
|
51
|
+
* Assert every egress attempt went to an allowed host. `allowed` matches a host
|
|
52
|
+
* (exact string or regex), or a specific `host:port`. Any attempt outside the
|
|
53
|
+
* allowlist fails, naming the offender — exfil / unexpected-registry detection.
|
|
54
|
+
*/
|
|
55
|
+
export declare function assertEgressOnly(r: HasEgress, allowed: ReadonlyArray<string | RegExp>): void;
|
|
56
|
+
/** Anything carrying recorded file writes (a confined runHook result). */
|
|
57
|
+
interface HasWrites {
|
|
58
|
+
readonly filesWritten: readonly string[];
|
|
59
|
+
}
|
|
60
|
+
/** Assert the run wrote NO file matching `pattern` (substring or regex). */
|
|
61
|
+
export declare function assertNoWrite(r: HasWrites, pattern: string | RegExp): void;
|
|
62
|
+
/** Assert every file the run wrote matches one of `allowed` (substring or regex). */
|
|
63
|
+
export declare function assertWroteOnly(r: HasWrites, allowed: ReadonlyArray<string | RegExp>): void;
|
|
39
64
|
/**
|
|
40
65
|
* Assert the worker's output is a success result, and return its `value`. With a
|
|
41
66
|
* `contract`, the value is validated against the success shape (a wrong/missing
|
|
@@ -207,6 +232,15 @@ export declare function assertImproves(report: EvalReport, opts: {
|
|
|
207
232
|
significant?: boolean;
|
|
208
233
|
alpha?: number;
|
|
209
234
|
}): void;
|
|
235
|
+
/**
|
|
236
|
+
* Assert the current run has not *regressed* against a committed baseline — the
|
|
237
|
+
* CI gate (Phase C). A regression is an arm×metric that moved **significantly in
|
|
238
|
+
* the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
|
|
239
|
+
* trip it; see `src/eval-baseline.ts`). Higher is better by default; list
|
|
240
|
+
* `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
|
|
241
|
+
* `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
|
|
242
|
+
*/
|
|
243
|
+
export declare function assertNoRegression(current: EvalReport | readonly EvalReport[], baseline: BaselineFile, opts?: DiffOptions): void;
|
|
210
244
|
/**
|
|
211
245
|
* Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
|
|
212
246
|
* reliability gate for a skill's *activation* (does its description fire on the
|
package/dist/harness-assert.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.vigilesMatchers = exports.compareArms = void 0;
|
|
3
|
+
exports.vigilesMatchers = exports.diffToJUnit = exports.formatBaselineDiff = exports.writeBaseline = exports.readBaseline = exports.parseBaselineFile = exports.toBaselineFile = exports.diffReports = exports.compareArms = void 0;
|
|
4
4
|
exports.withHarness = withHarness;
|
|
5
5
|
exports.assertCreated = assertCreated;
|
|
6
6
|
exports.assertNotCreated = assertNotCreated;
|
|
7
7
|
exports.assertServedTurns = assertServedTurns;
|
|
8
8
|
exports.assertHookBlocked = assertHookBlocked;
|
|
9
9
|
exports.assertHookAllowed = assertHookAllowed;
|
|
10
|
+
exports.egressHosts = egressHosts;
|
|
11
|
+
exports.assertNoEgress = assertNoEgress;
|
|
12
|
+
exports.assertEgressOnly = assertEgressOnly;
|
|
13
|
+
exports.assertNoWrite = assertNoWrite;
|
|
14
|
+
exports.assertWroteOnly = assertWroteOnly;
|
|
10
15
|
exports.assertAgentOk = assertAgentOk;
|
|
11
16
|
exports.assertAgentErr = assertAgentErr;
|
|
12
17
|
exports.assertAgentResult = assertAgentResult;
|
|
@@ -34,6 +39,7 @@ exports.improvement = improvement;
|
|
|
34
39
|
exports.significantlyBeats = significantlyBeats;
|
|
35
40
|
exports.assertSignificant = assertSignificant;
|
|
36
41
|
exports.assertImproves = assertImproves;
|
|
42
|
+
exports.assertNoRegression = assertNoRegression;
|
|
37
43
|
exports.assertTriggerRate = assertTriggerRate;
|
|
38
44
|
/**
|
|
39
45
|
* vigiles — runner-agnostic helpers for harness tests / evals.
|
|
@@ -53,10 +59,19 @@ exports.assertTriggerRate = assertTriggerRate;
|
|
|
53
59
|
const harness_test_js_1 = require("./harness-test.js");
|
|
54
60
|
const agent_result_js_1 = require("./agent-result.js");
|
|
55
61
|
const stats_js_1 = require("./stats.js");
|
|
62
|
+
const eval_baseline_js_1 = require("./eval-baseline.js");
|
|
56
63
|
// Re-export the significance primitives so the whole eval-analysis surface lives
|
|
57
64
|
// behind `vigiles/harness-assert` (no separate entry point).
|
|
58
65
|
var stats_js_2 = require("./stats.js");
|
|
59
66
|
Object.defineProperty(exports, "compareArms", { enumerable: true, get: function () { return stats_js_2.compareArms; } });
|
|
67
|
+
var eval_baseline_js_2 = require("./eval-baseline.js");
|
|
68
|
+
Object.defineProperty(exports, "diffReports", { enumerable: true, get: function () { return eval_baseline_js_2.diffReports; } });
|
|
69
|
+
Object.defineProperty(exports, "toBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.toBaselineFile; } });
|
|
70
|
+
Object.defineProperty(exports, "parseBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.parseBaselineFile; } });
|
|
71
|
+
Object.defineProperty(exports, "readBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.readBaseline; } });
|
|
72
|
+
Object.defineProperty(exports, "writeBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.writeBaseline; } });
|
|
73
|
+
Object.defineProperty(exports, "formatBaselineDiff", { enumerable: true, get: function () { return eval_baseline_js_2.formatBaselineDiff; } });
|
|
74
|
+
Object.defineProperty(exports, "diffToJUnit", { enumerable: true, get: function () { return eval_baseline_js_2.diffToJUnit; } });
|
|
60
75
|
/**
|
|
61
76
|
* Run a harness test, hand the result to `fn`, and always clean up the sandbox.
|
|
62
77
|
* Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
|
|
@@ -105,6 +120,46 @@ function assertHookAllowed(r) {
|
|
|
105
120
|
fail(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)}, decision ${String(r.decision)})`);
|
|
106
121
|
}
|
|
107
122
|
}
|
|
123
|
+
const hostPort = (e) => `${e.host}:${String(e.port)}`;
|
|
124
|
+
/** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
|
|
125
|
+
function egressHosts(r) {
|
|
126
|
+
return r.egress.map(hostPort);
|
|
127
|
+
}
|
|
128
|
+
/** Assert the confined run made NO network egress attempt at all. */
|
|
129
|
+
function assertNoEgress(r) {
|
|
130
|
+
if (r.egress.length > 0) {
|
|
131
|
+
fail(`expected no egress, but it tried to reach: ${egressHosts(r).join(", ")}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Assert every egress attempt went to an allowed host. `allowed` matches a host
|
|
136
|
+
* (exact string or regex), or a specific `host:port`. Any attempt outside the
|
|
137
|
+
* allowlist fails, naming the offender — exfil / unexpected-registry detection.
|
|
138
|
+
*/
|
|
139
|
+
function assertEgressOnly(r, allowed) {
|
|
140
|
+
const ok = (e) => allowed.some((a) => typeof a === "string"
|
|
141
|
+
? a === e.host || a === hostPort(e)
|
|
142
|
+
: a.test(e.host) || a.test(hostPort(e)));
|
|
143
|
+
const bad = r.egress.filter((e) => !ok(e));
|
|
144
|
+
if (bad.length > 0) {
|
|
145
|
+
fail(`egress to non-allowlisted host(s): ${bad.map(hostPort).join(", ")}`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const matches = (f, p) => typeof p === "string" ? f.includes(p) : p.test(f);
|
|
149
|
+
/** Assert the run wrote NO file matching `pattern` (substring or regex). */
|
|
150
|
+
function assertNoWrite(r, pattern) {
|
|
151
|
+
const bad = r.filesWritten.filter((f) => matches(f, pattern));
|
|
152
|
+
if (bad.length > 0) {
|
|
153
|
+
fail(`expected no write matching ${String(pattern)}, but wrote: ${bad.join(", ")}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Assert every file the run wrote matches one of `allowed` (substring or regex). */
|
|
157
|
+
function assertWroteOnly(r, allowed) {
|
|
158
|
+
const bad = r.filesWritten.filter((f) => !allowed.some((a) => matches(f, a)));
|
|
159
|
+
if (bad.length > 0) {
|
|
160
|
+
fail(`run wrote file(s) outside the allowlist: ${bad.join(", ")}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
108
163
|
// --- subagent railway outcome (parse the worker's result block) ------------
|
|
109
164
|
//
|
|
110
165
|
// A subagent with a result() contract ends its turn with a vigiles:ok/err block.
|
|
@@ -430,6 +485,26 @@ function assertImproves(report, opts) {
|
|
|
430
485
|
fail(`expected ${opts.arm} to beat ${opts.baseline} on ${opts.metric} by > ${String(by)}, got ${delta.toFixed(3)}`);
|
|
431
486
|
}
|
|
432
487
|
}
|
|
488
|
+
/**
|
|
489
|
+
* Assert the current run has not *regressed* against a committed baseline — the
|
|
490
|
+
* CI gate (Phase C). A regression is an arm×metric that moved **significantly in
|
|
491
|
+
* the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
|
|
492
|
+
* trip it; see `src/eval-baseline.ts`). Higher is better by default; list
|
|
493
|
+
* `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
|
|
494
|
+
* `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
|
|
495
|
+
*/
|
|
496
|
+
function assertNoRegression(current, baseline, opts) {
|
|
497
|
+
const reports = Array.isArray(current)
|
|
498
|
+
? current
|
|
499
|
+
: [current];
|
|
500
|
+
const diff = (0, eval_baseline_js_1.diffReports)(baseline, reports, opts);
|
|
501
|
+
if (!diff.passed) {
|
|
502
|
+
const detail = diff.regressions
|
|
503
|
+
.map((r) => `${r.report}/${r.arm}/${r.metric} Δ=${r.comparison.delta.toFixed(3)} p=${r.comparison.pValue.toFixed(3)}`)
|
|
504
|
+
.join("; ");
|
|
505
|
+
fail(`regression vs baseline: ${detail}`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
433
508
|
/**
|
|
434
509
|
* Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
|
|
435
510
|
* reliability gate for a skill's *activation* (does its description fire on the
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
|
|
3
|
+
* files. Re-exports the spec builders/types and the compiler under one
|
|
4
|
+
* concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
|
|
5
|
+
* keep working; this just groups them so the import name matches the pillar.
|
|
6
|
+
*/
|
|
7
|
+
export * from "./spec.js";
|
|
8
|
+
export * from "./compile.js";
|
|
9
|
+
//# sourceMappingURL=linting.d.ts.map
|
package/dist/linting.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
/**
|
|
18
|
+
* `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
|
|
19
|
+
* files. Re-exports the spec builders/types and the compiler under one
|
|
20
|
+
* concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
|
|
21
|
+
* keep working; this just groups them so the import name matches the pillar.
|
|
22
|
+
*/
|
|
23
|
+
__exportStar(require("./spec.js"), exports);
|
|
24
|
+
__exportStar(require("./compile.js"), exports);
|
|
25
|
+
//# sourceMappingURL=linting.js.map
|
package/dist/run-hook.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { type SandboxMode, type EgressAttempt } from "./sandbox.js";
|
|
2
|
+
export type { EgressAttempt };
|
|
1
3
|
/** A hook event payload (the JSON Claude Code writes to the hook's stdin). */
|
|
2
4
|
export interface HookInput {
|
|
3
5
|
/** e.g. "PreToolUse", "PostToolUse", "Stop", "SessionStart", "PreCompact". */
|
|
@@ -38,6 +40,41 @@ export interface RunHookOptions {
|
|
|
38
40
|
readonly env?: Record<string, string>;
|
|
39
41
|
/** Per-run timeout ms. Default 10000. */
|
|
40
42
|
readonly timeoutMs?: number;
|
|
43
|
+
/**
|
|
44
|
+
* Provenance of the hook command. `true` (default) means YOU authored it — the
|
|
45
|
+
* usual case at this tier, a command written inline in the test — so it runs
|
|
46
|
+
* directly. `false` marks it foreign (a vendored third-party hook script),
|
|
47
|
+
* which makes confinement the DEFAULT: with no explicit `sandbox`, an untrusted
|
|
48
|
+
* hook behaves as `sandbox: "auto"` — confined under bubblewrap, or refused if
|
|
49
|
+
* none is available — so foreign code is never run unconfined by accident. This
|
|
50
|
+
* mirrors the harness tier, where trust follows `plugin`/`pluginDir`
|
|
51
|
+
* provenance (`specTrusted` in `src/sandbox.ts`); the unit tier takes a raw
|
|
52
|
+
* command string with no provenance signal, so you declare it here.
|
|
53
|
+
*/
|
|
54
|
+
readonly trusted?: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Confine the hook under bubblewrap (Linux). When unset, the mode follows
|
|
57
|
+
* {@link RunHookOptions.trusted}: a trusted hook runs directly (`false`), an
|
|
58
|
+
* untrusted one is confined-or-refused (`"auto"`). Set it explicitly to
|
|
59
|
+
* override: `"auto"`/`"strict"` force confinement (a no-egress namespace with a
|
|
60
|
+
* cleared environment — your `opts.env` is added back — or a **refusal** if no
|
|
61
|
+
* bwrap is available), and `false` is the opt-out that runs even untrusted code
|
|
62
|
+
* unconfined. macOS/Windows have no bwrap, so `"auto"`/`"strict"` throw there —
|
|
63
|
+
* see `src/sandbox.ts`.
|
|
64
|
+
*/
|
|
65
|
+
readonly sandbox?: SandboxMode;
|
|
66
|
+
/**
|
|
67
|
+
* Record the hook's network egress. Implies confinement (the recorder lives in
|
|
68
|
+
* the sandbox netns, so this forces a sandboxed run and refuses if no sandbox is
|
|
69
|
+
* available). A recording proxy on loopback captures every `host:port` a
|
|
70
|
+
* proxy-honoring tool (npm/pip/curl/fetch) tries to reach — surfaced as
|
|
71
|
+
* {@link HookRunResult.egress} — while the netns still **blocks** it (nothing
|
|
72
|
+
* actually leaves). Use it to test what a hook/skill phones home to, or which
|
|
73
|
+
* registry an install would hit. Raw-socket egress is blocked but not recorded
|
|
74
|
+
* (it never reaches the proxy) — the block is the boundary, the record is
|
|
75
|
+
* best-effort observability over it.
|
|
76
|
+
*/
|
|
77
|
+
readonly recordEgress?: boolean;
|
|
41
78
|
}
|
|
42
79
|
export interface HookRunResult {
|
|
43
80
|
readonly exitCode: number;
|
|
@@ -50,6 +87,17 @@ export interface HookRunResult {
|
|
|
50
87
|
* `permissionDecision:"deny"` all set `blocked = true`.
|
|
51
88
|
*/
|
|
52
89
|
readonly blocked: boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Network egress the hook attempted, recorded then blocked. Empty unless
|
|
92
|
+
* {@link RunHookOptions.recordEgress} was set (and the run was confined).
|
|
93
|
+
*/
|
|
94
|
+
readonly egress: readonly EgressAttempt[];
|
|
95
|
+
/**
|
|
96
|
+
* Files the hook wrote to its work dir (relative paths), recorded on confined
|
|
97
|
+
* runs — what a hook touched on disk. Empty on a direct (unconfined) run.
|
|
98
|
+
* Assert over it with `assertNoWrite` / `assertWroteOnly`.
|
|
99
|
+
*/
|
|
100
|
+
readonly filesWritten: readonly string[];
|
|
53
101
|
/**
|
|
54
102
|
* The decision the hook expressed, preferring the structured
|
|
55
103
|
* `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
|
|
@@ -67,11 +115,43 @@ export declare function decideHook(exitCode: number, json: HookOutput | null): {
|
|
|
67
115
|
blocked: boolean;
|
|
68
116
|
decision: HookRunResult["decision"];
|
|
69
117
|
};
|
|
118
|
+
/** The raw fields of a hook spawn that the result parser needs. */
|
|
119
|
+
export interface HookSpawnResult {
|
|
120
|
+
readonly status: number | null;
|
|
121
|
+
readonly signal: string | null;
|
|
122
|
+
readonly stdout: string;
|
|
123
|
+
readonly stderr: string;
|
|
124
|
+
/** Egress attempts captured by the in-sandbox recorder (recordEgress only). */
|
|
125
|
+
readonly egress?: readonly EgressAttempt[];
|
|
126
|
+
/** Files the hook wrote to its work dir (confined runs). */
|
|
127
|
+
readonly filesWritten?: readonly string[];
|
|
128
|
+
}
|
|
129
|
+
/** Spawn a hook (command + piped event) — the injectable seam over the real spawn. */
|
|
130
|
+
export type HookSpawner = (command: string, input: HookInput, opts: RunHookOptions) => HookSpawnResult;
|
|
131
|
+
/** The spawn seams `runHookWith` needs, so its decision logic is testable. */
|
|
132
|
+
export interface RunHookDeps {
|
|
133
|
+
/** Whether bubblewrap confinement is available (Linux + bwrap). */
|
|
134
|
+
readonly available: boolean;
|
|
135
|
+
/** Run the command directly (unconfined). */
|
|
136
|
+
readonly direct: HookSpawner;
|
|
137
|
+
/** Run the command confined under bubblewrap. */
|
|
138
|
+
readonly sandboxed: HookSpawner;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The hook-run orchestration with injectable spawn seams: pick direct vs.
|
|
142
|
+
* confined via the safe-by-default policy (`decideSandbox`), then parse the exit
|
|
143
|
+
* code + stdout into a normalized decision. Exported so all three branches
|
|
144
|
+
* (direct / sandbox / refuse) are unit-tested with fake spawners — no real
|
|
145
|
+
* bwrap. `runHook` is this with the real seams.
|
|
146
|
+
*/
|
|
147
|
+
export declare function runHookWith(command: string, input: HookInput, opts: RunHookOptions, deps: RunHookDeps): HookRunResult;
|
|
70
148
|
/**
|
|
71
149
|
* Run a hook command, piping `input` as JSON to its stdin, and report the exit
|
|
72
150
|
* code + parsed decision. Synchronous (so it can be used inside an eval's
|
|
73
151
|
* `measure` too). `command` is run through a shell, so the same command string a
|
|
74
|
-
* plugin ships (with args / env refs) works verbatim.
|
|
152
|
+
* plugin ships (with args / env refs) works verbatim. Mark a hook you didn't
|
|
153
|
+
* write with `trusted: false` and it is confined by default (or pass `sandbox:
|
|
154
|
+
* "auto"` directly) — see {@link RunHookOptions.trusted}.
|
|
75
155
|
*/
|
|
76
156
|
export declare function runHook(command: string, input: HookInput, opts?: RunHookOptions): HookRunResult;
|
|
77
157
|
//# sourceMappingURL=run-hook.d.ts.map
|