vigiles 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,68 @@
1
+ import type { EvalReport } from "./eval.js";
2
+ import { type Comparison } from "./stats.js";
3
+ /** Bumped only on a breaking change to the on-disk shape. */
4
+ export declare const BASELINE_VERSION = 1;
5
+ /** The committed baseline: the recorded `EvalReport`s, keyed by report name. */
6
+ export interface BaselineFile {
7
+ readonly version: number;
8
+ /** ISO-8601 timestamp the baseline was recorded (provenance / future trend). */
9
+ readonly recordedAt: string;
10
+ /** Recorded reports, keyed by `report.name` (so multiple eval files coexist). */
11
+ readonly reports: Record<string, EvalReport>;
12
+ }
13
+ /** How a metric moved between baseline and current run. */
14
+ export type DiffStatus = "regressed" | "improved" | "unchanged";
15
+ /** One arm×metric comparison of a current run against the baseline. */
16
+ export interface MetricDiff {
17
+ /** The `report.name` this entry belongs to. */
18
+ readonly report: string;
19
+ readonly arm: string;
20
+ readonly metric: string;
21
+ readonly status: DiffStatus;
22
+ /** Welch comparison, current vs. baseline (`delta = current − baseline`). */
23
+ readonly comparison: Comparison;
24
+ }
25
+ export interface BaselineDiff {
26
+ /** Every arm×metric present in BOTH the baseline and the current run. */
27
+ readonly entries: readonly MetricDiff[];
28
+ /** The subset that regressed (significant move in the bad direction). */
29
+ readonly regressions: readonly MetricDiff[];
30
+ /** The subset that improved (significant move in the good direction). */
31
+ readonly improvements: readonly MetricDiff[];
32
+ /** True when there are no regressions — the gate. */
33
+ readonly passed: boolean;
34
+ }
35
+ export interface DiffOptions {
36
+ /** Significance level for the Welch test. Default 0.05. */
37
+ readonly alpha?: number;
38
+ /**
39
+ * Metrics where a DECREASE is the improvement (e.g. `cost`, `latency`,
40
+ * `turns`). For these, a significant increase is the regression. Everything
41
+ * else is treated as higher-is-better.
42
+ */
43
+ readonly lowerIsBetter?: readonly string[];
44
+ }
45
+ /** Build a `BaselineFile` envelope from a run's reports (keyed by name). */
46
+ export declare function toBaselineFile(reports: readonly EvalReport[], recordedAt?: string): BaselineFile;
47
+ /** Parse + validate a baseline JSON string (throws on a bad version/shape). */
48
+ export declare function parseBaselineFile(json: string): BaselineFile;
49
+ /**
50
+ * Diff a current run against a committed baseline. Compares every arm×metric
51
+ * present in both (by report name), flagging a *significant* move in the
52
+ * undesired direction as a regression. Metrics absent from one side are skipped
53
+ * (a new arm/metric is not a regression).
54
+ */
55
+ export declare function diffReports(baseline: BaselineFile, current: readonly EvalReport[], opts?: DiffOptions): BaselineDiff;
56
+ /** Format a baseline diff as a compact console report. */
57
+ export declare function formatBaselineDiff(diff: BaselineDiff): string;
58
+ /**
59
+ * Render a baseline diff as JUnit XML — one `<testcase>` per arm×metric, a
60
+ * `<failure>` for each regression. Lets a CI provider show eval regressions in
61
+ * the same place as unit-test failures.
62
+ */
63
+ export declare function diffToJUnit(diff: BaselineDiff): string;
64
+ /** Read + parse a baseline file, or null if it doesn't exist yet. */
65
+ export declare function readBaseline(path: string): BaselineFile | null;
66
+ /** Write reports as the committed baseline (pretty JSON, parent dirs created). */
67
+ export declare function writeBaseline(path: string, reports: readonly EvalReport[]): void;
68
+ //# sourceMappingURL=eval-baseline.d.ts.map
@@ -0,0 +1,173 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BASELINE_VERSION = void 0;
4
+ exports.toBaselineFile = toBaselineFile;
5
+ exports.parseBaselineFile = parseBaselineFile;
6
+ exports.diffReports = diffReports;
7
+ exports.formatBaselineDiff = formatBaselineDiff;
8
+ exports.diffToJUnit = diffToJUnit;
9
+ exports.readBaseline = readBaseline;
10
+ exports.writeBaseline = writeBaseline;
11
+ /**
12
+ * vigiles — eval regression gating (Phase C).
13
+ *
14
+ * The eval tier reports mean ± se per arm; `src/stats.ts` turns a gap into a
15
+ * significance verdict. This module points that machinery at a *committed
16
+ * baseline*: record one run's `EvalReport`s to `.vigiles/eval-baseline.json`,
17
+ * then on a later run flag any arm×metric that moved **significantly in the bad
18
+ * direction** vs. that baseline. "jest snapshots for agent behaviour, with a real
19
+ * noise floor" — a bare pass-rate can't tell a true regression from sampling
20
+ * noise, but a Welch t-test over the two runs' summary stats can.
21
+ *
22
+ * Pure + model-free (the diff/serialize/JUnit are fully unit-tested); the only
23
+ * side effects are the two small fs helpers (`readBaseline` / `writeBaseline`).
24
+ * Reuses `welchTTest` from `src/stats.ts` — the current run is the "arm", the
25
+ * baseline is the "baseline", so `delta = current − baseline`.
26
+ */
27
+ const node_fs_1 = require("node:fs");
28
+ const node_path_1 = require("node:path");
29
+ const stats_js_1 = require("./stats.js");
30
+ /** Bumped only on a breaking change to the on-disk shape. */
31
+ exports.BASELINE_VERSION = 1;
32
+ /** Build a `BaselineFile` envelope from a run's reports (keyed by name). */
33
+ function toBaselineFile(reports, recordedAt = new Date().toISOString()) {
34
+ const byName = {};
35
+ for (const r of reports)
36
+ byName[r.name] = r;
37
+ return { version: exports.BASELINE_VERSION, recordedAt, reports: byName };
38
+ }
39
+ /** Parse + validate a baseline JSON string (throws on a bad version/shape). */
40
+ function parseBaselineFile(json) {
41
+ const data = JSON.parse(json);
42
+ if (typeof data !== "object" || data === null) {
43
+ throw new Error("baseline: expected a JSON object");
44
+ }
45
+ const obj = data;
46
+ if (obj.version !== exports.BASELINE_VERSION) {
47
+ throw new Error(`baseline: unsupported version ${String(obj.version)} (expected ${String(exports.BASELINE_VERSION)})`);
48
+ }
49
+ if (typeof obj.reports !== "object" || obj.reports === null) {
50
+ throw new Error("baseline: missing `reports`");
51
+ }
52
+ return {
53
+ version: exports.BASELINE_VERSION,
54
+ recordedAt: typeof obj.recordedAt === "string" ? obj.recordedAt : "",
55
+ reports: obj.reports,
56
+ };
57
+ }
58
+ /** Classify one comparison given the metric's direction. */
59
+ function classify(cmp, lowerIsBetter) {
60
+ if (!cmp.significant || cmp.delta === 0)
61
+ return "unchanged";
62
+ const improved = lowerIsBetter ? cmp.delta < 0 : cmp.delta > 0;
63
+ return improved ? "improved" : "regressed";
64
+ }
65
+ /** Append a diff entry for every arm×metric common to both reports. */
66
+ function collectReportDiffs(baseline, current, cfg, out) {
67
+ for (const [arm, curArm] of Object.entries(current.arms)) {
68
+ const baseArm = baseline.arms[arm];
69
+ if (!baseArm)
70
+ continue;
71
+ for (const [metric, curStat] of Object.entries(curArm.stats)) {
72
+ const baseStat = baseArm.stats[metric];
73
+ if (!baseStat)
74
+ continue;
75
+ const comparison = (0, stats_js_1.welchTTest)(curStat, baseStat, cfg.alpha);
76
+ out.push({
77
+ report: current.name,
78
+ arm,
79
+ metric,
80
+ status: classify(comparison, cfg.lower.has(metric)),
81
+ comparison,
82
+ });
83
+ }
84
+ }
85
+ }
86
+ /**
87
+ * Diff a current run against a committed baseline. Compares every arm×metric
88
+ * present in both (by report name), flagging a *significant* move in the
89
+ * undesired direction as a regression. Metrics absent from one side are skipped
90
+ * (a new arm/metric is not a regression).
91
+ */
92
+ function diffReports(baseline, current, opts = {}) {
93
+ const cfg = {
94
+ alpha: opts.alpha ?? 0.05,
95
+ lower: new Set(opts.lowerIsBetter ?? []),
96
+ };
97
+ const entries = [];
98
+ for (const cur of current) {
99
+ const base = baseline.reports[cur.name];
100
+ if (base)
101
+ collectReportDiffs(base, cur, cfg, entries);
102
+ }
103
+ const regressions = entries.filter((e) => e.status === "regressed");
104
+ const improvements = entries.filter((e) => e.status === "improved");
105
+ return {
106
+ entries,
107
+ regressions,
108
+ improvements,
109
+ passed: regressions.length === 0,
110
+ };
111
+ }
112
+ const STATUS_MARK = {
113
+ regressed: "✗",
114
+ improved: "✓",
115
+ unchanged: "·",
116
+ };
117
+ function formatDelta(c) {
118
+ const sign = c.delta >= 0 ? "+" : "";
119
+ return `Δ=${sign}${c.delta.toFixed(3)} p=${c.pValue.toFixed(3)}`;
120
+ }
121
+ /** Format a baseline diff as a compact console report. */
122
+ function formatBaselineDiff(diff) {
123
+ const head = diff.passed
124
+ ? "baseline OK — no significant regressions"
125
+ : `baseline FAIL — ${String(diff.regressions.length)} regression(s)`;
126
+ const lines = [head];
127
+ for (const e of diff.entries) {
128
+ lines.push(` ${STATUS_MARK[e.status]} ${e.report}/${e.arm}/${e.metric} ${formatDelta(e.comparison)}`);
129
+ }
130
+ return lines.join("\n");
131
+ }
132
+ function xmlEscape(s) {
133
+ return s
134
+ .replaceAll("&", "&amp;")
135
+ .replaceAll("<", "&lt;")
136
+ .replaceAll(">", "&gt;")
137
+ .replaceAll('"', "&quot;");
138
+ }
139
+ function junitCase(e) {
140
+ const name = xmlEscape(`${e.report}.${e.arm}.${e.metric}`);
141
+ const open = ` <testcase classname="${xmlEscape(e.report)}" name="${name}">`;
142
+ if (e.status !== "regressed")
143
+ return `${open}</testcase>`;
144
+ const msg = xmlEscape(`regression: ${formatDelta(e.comparison)}`);
145
+ return `${open}\n <failure message="${msg}"/>\n </testcase>`;
146
+ }
147
+ /**
148
+ * Render a baseline diff as JUnit XML — one `<testcase>` per arm×metric, a
149
+ * `<failure>` for each regression. Lets a CI provider show eval regressions in
150
+ * the same place as unit-test failures.
151
+ */
152
+ function diffToJUnit(diff) {
153
+ const cases = diff.entries.map(junitCase).join("\n");
154
+ return [
155
+ '<?xml version="1.0" encoding="UTF-8"?>',
156
+ `<testsuite name="vigiles-eval" tests="${String(diff.entries.length)}" failures="${String(diff.regressions.length)}">`,
157
+ cases,
158
+ "</testsuite>",
159
+ "",
160
+ ].join("\n");
161
+ }
162
+ /** Read + parse a baseline file, or null if it doesn't exist yet. */
163
+ function readBaseline(path) {
164
+ if (!(0, node_fs_1.existsSync)(path))
165
+ return null;
166
+ return parseBaselineFile((0, node_fs_1.readFileSync)(path, "utf-8"));
167
+ }
168
+ /** Write reports as the committed baseline (pretty JSON, parent dirs created). */
169
+ function writeBaseline(path, reports) {
170
+ (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
171
+ (0, node_fs_1.writeFileSync)(path, JSON.stringify(toBaselineFile(reports), null, 2) + "\n");
172
+ }
173
+ //# sourceMappingURL=eval-baseline.js.map
@@ -15,11 +15,14 @@
15
15
  */
16
16
  import { type HarnessTestSpec, type HarnessTestResult, type ToolCall, type Trace } from "./harness-test.js";
17
17
  import type { EvalReport, TriggerRateReport } from "./eval.js";
18
- import type { HookRunResult } from "./run-hook.js";
18
+ import type { HookRunResult, EgressAttempt } from "./run-hook.js";
19
19
  import type { OutputContract } from "./spec.js";
20
20
  import { type ParsedAgentResult } from "./agent-result.js";
21
+ import { type BaselineFile, type DiffOptions } from "./eval-baseline.js";
21
22
  export { compareArms } from "./stats.js";
22
23
  export type { Comparison } from "./stats.js";
24
+ export { diffReports, toBaselineFile, parseBaselineFile, readBaseline, writeBaseline, formatBaselineDiff, diffToJUnit, } from "./eval-baseline.js";
25
+ export type { BaselineFile, BaselineDiff, MetricDiff, DiffStatus, DiffOptions, } from "./eval-baseline.js";
23
26
  /**
24
27
  * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
25
28
  * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
@@ -36,6 +39,28 @@ export declare function assertServedTurns(r: HarnessTestResult, n: number): void
36
39
  export declare function assertHookBlocked(r: HookRunResult): void;
37
40
  /** Assert a `runHook` result allowed (did not block). */
38
41
  export declare function assertHookAllowed(r: HookRunResult): void;
42
+ /** Anything carrying recorded egress attempts (a runHook recordEgress result). */
43
+ interface HasEgress {
44
+ readonly egress: readonly EgressAttempt[];
45
+ }
46
+ /** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
47
+ export declare function egressHosts(r: HasEgress): string[];
48
+ /** Assert the confined run made NO network egress attempt at all. */
49
+ export declare function assertNoEgress(r: HasEgress): void;
50
+ /**
51
+ * Assert every egress attempt went to an allowed host. `allowed` matches a host
52
+ * (exact string or regex), or a specific `host:port`. Any attempt outside the
53
+ * allowlist fails, naming the offender — exfil / unexpected-registry detection.
54
+ */
55
+ export declare function assertEgressOnly(r: HasEgress, allowed: ReadonlyArray<string | RegExp>): void;
56
+ /** Anything carrying recorded file writes (a confined runHook result). */
57
+ interface HasWrites {
58
+ readonly filesWritten: readonly string[];
59
+ }
60
+ /** Assert the run wrote NO file matching `pattern` (substring or regex). */
61
+ export declare function assertNoWrite(r: HasWrites, pattern: string | RegExp): void;
62
+ /** Assert every file the run wrote matches one of `allowed` (substring or regex). */
63
+ export declare function assertWroteOnly(r: HasWrites, allowed: ReadonlyArray<string | RegExp>): void;
39
64
  /**
40
65
  * Assert the worker's output is a success result, and return its `value`. With a
41
66
  * `contract`, the value is validated against the success shape (a wrong/missing
@@ -207,6 +232,15 @@ export declare function assertImproves(report: EvalReport, opts: {
207
232
  significant?: boolean;
208
233
  alpha?: number;
209
234
  }): void;
235
+ /**
236
+ * Assert the current run has not *regressed* against a committed baseline — the
237
+ * CI gate (Phase C). A regression is an arm×metric that moved **significantly in
238
+ * the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
239
+ * trip it; see `src/eval-baseline.ts`). Higher is better by default; list
240
+ * `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
241
+ * `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
242
+ */
243
+ export declare function assertNoRegression(current: EvalReport | readonly EvalReport[], baseline: BaselineFile, opts?: DiffOptions): void;
210
244
  /**
211
245
  * Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
212
246
  * reliability gate for a skill's *activation* (does its description fire on the
@@ -1,12 +1,17 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.vigilesMatchers = exports.compareArms = void 0;
3
+ exports.vigilesMatchers = exports.diffToJUnit = exports.formatBaselineDiff = exports.writeBaseline = exports.readBaseline = exports.parseBaselineFile = exports.toBaselineFile = exports.diffReports = exports.compareArms = void 0;
4
4
  exports.withHarness = withHarness;
5
5
  exports.assertCreated = assertCreated;
6
6
  exports.assertNotCreated = assertNotCreated;
7
7
  exports.assertServedTurns = assertServedTurns;
8
8
  exports.assertHookBlocked = assertHookBlocked;
9
9
  exports.assertHookAllowed = assertHookAllowed;
10
+ exports.egressHosts = egressHosts;
11
+ exports.assertNoEgress = assertNoEgress;
12
+ exports.assertEgressOnly = assertEgressOnly;
13
+ exports.assertNoWrite = assertNoWrite;
14
+ exports.assertWroteOnly = assertWroteOnly;
10
15
  exports.assertAgentOk = assertAgentOk;
11
16
  exports.assertAgentErr = assertAgentErr;
12
17
  exports.assertAgentResult = assertAgentResult;
@@ -34,6 +39,7 @@ exports.improvement = improvement;
34
39
  exports.significantlyBeats = significantlyBeats;
35
40
  exports.assertSignificant = assertSignificant;
36
41
  exports.assertImproves = assertImproves;
42
+ exports.assertNoRegression = assertNoRegression;
37
43
  exports.assertTriggerRate = assertTriggerRate;
38
44
  /**
39
45
  * vigiles — runner-agnostic helpers for harness tests / evals.
@@ -53,10 +59,19 @@ exports.assertTriggerRate = assertTriggerRate;
53
59
  const harness_test_js_1 = require("./harness-test.js");
54
60
  const agent_result_js_1 = require("./agent-result.js");
55
61
  const stats_js_1 = require("./stats.js");
62
+ const eval_baseline_js_1 = require("./eval-baseline.js");
56
63
  // Re-export the significance primitives so the whole eval-analysis surface lives
57
64
  // behind `vigiles/harness-assert` (no separate entry point).
58
65
  var stats_js_2 = require("./stats.js");
59
66
  Object.defineProperty(exports, "compareArms", { enumerable: true, get: function () { return stats_js_2.compareArms; } });
67
+ var eval_baseline_js_2 = require("./eval-baseline.js");
68
+ Object.defineProperty(exports, "diffReports", { enumerable: true, get: function () { return eval_baseline_js_2.diffReports; } });
69
+ Object.defineProperty(exports, "toBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.toBaselineFile; } });
70
+ Object.defineProperty(exports, "parseBaselineFile", { enumerable: true, get: function () { return eval_baseline_js_2.parseBaselineFile; } });
71
+ Object.defineProperty(exports, "readBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.readBaseline; } });
72
+ Object.defineProperty(exports, "writeBaseline", { enumerable: true, get: function () { return eval_baseline_js_2.writeBaseline; } });
73
+ Object.defineProperty(exports, "formatBaselineDiff", { enumerable: true, get: function () { return eval_baseline_js_2.formatBaselineDiff; } });
74
+ Object.defineProperty(exports, "diffToJUnit", { enumerable: true, get: function () { return eval_baseline_js_2.diffToJUnit; } });
60
75
  /**
61
76
  * Run a harness test, hand the result to `fn`, and always clean up the sandbox.
62
77
  * Returns whatever `fn` returns. Use this instead of calling `cleanup()` by
@@ -105,6 +120,46 @@ function assertHookAllowed(r) {
105
120
  fail(`expected the hook to allow, but it blocked (exit ${String(r.exitCode)}, decision ${String(r.decision)})`);
106
121
  }
107
122
  }
123
+ const hostPort = (e) => `${e.host}:${String(e.port)}`;
124
+ /** The `host:port` strings a run attempted, e.g. `["registry.npmjs.org:443"]`. */
125
+ function egressHosts(r) {
126
+ return r.egress.map(hostPort);
127
+ }
128
+ /** Assert the confined run made NO network egress attempt at all. */
129
+ function assertNoEgress(r) {
130
+ if (r.egress.length > 0) {
131
+ fail(`expected no egress, but it tried to reach: ${egressHosts(r).join(", ")}`);
132
+ }
133
+ }
134
+ /**
135
+ * Assert every egress attempt went to an allowed host. `allowed` matches a host
136
+ * (exact string or regex), or a specific `host:port`. Any attempt outside the
137
+ * allowlist fails, naming the offender — exfil / unexpected-registry detection.
138
+ */
139
+ function assertEgressOnly(r, allowed) {
140
+ const ok = (e) => allowed.some((a) => typeof a === "string"
141
+ ? a === e.host || a === hostPort(e)
142
+ : a.test(e.host) || a.test(hostPort(e)));
143
+ const bad = r.egress.filter((e) => !ok(e));
144
+ if (bad.length > 0) {
145
+ fail(`egress to non-allowlisted host(s): ${bad.map(hostPort).join(", ")}`);
146
+ }
147
+ }
148
+ const matches = (f, p) => typeof p === "string" ? f.includes(p) : p.test(f);
149
+ /** Assert the run wrote NO file matching `pattern` (substring or regex). */
150
+ function assertNoWrite(r, pattern) {
151
+ const bad = r.filesWritten.filter((f) => matches(f, pattern));
152
+ if (bad.length > 0) {
153
+ fail(`expected no write matching ${String(pattern)}, but wrote: ${bad.join(", ")}`);
154
+ }
155
+ }
156
+ /** Assert every file the run wrote matches one of `allowed` (substring or regex). */
157
+ function assertWroteOnly(r, allowed) {
158
+ const bad = r.filesWritten.filter((f) => !allowed.some((a) => matches(f, a)));
159
+ if (bad.length > 0) {
160
+ fail(`run wrote file(s) outside the allowlist: ${bad.join(", ")}`);
161
+ }
162
+ }
108
163
  // --- subagent railway outcome (parse the worker's result block) ------------
109
164
  //
110
165
  // A subagent with a result() contract ends its turn with a vigiles:ok/err block.
@@ -430,6 +485,26 @@ function assertImproves(report, opts) {
430
485
  fail(`expected ${opts.arm} to beat ${opts.baseline} on ${opts.metric} by > ${String(by)}, got ${delta.toFixed(3)}`);
431
486
  }
432
487
  }
488
+ /**
489
+ * Assert the current run has not *regressed* against a committed baseline — the
490
+ * CI gate (Phase C). A regression is an arm×metric that moved **significantly in
491
+ * the bad direction** vs. `baseline` (Welch t-test, so sampling noise doesn't
492
+ * trip it; see `src/eval-baseline.ts`). Higher is better by default; list
493
+ * `lowerIsBetter` metrics (cost/latency) to flip them. Load the baseline with
494
+ * `readBaseline(path)` and record a fresh one with `writeBaseline(path, reports)`.
495
+ */
496
+ function assertNoRegression(current, baseline, opts) {
497
+ const reports = Array.isArray(current)
498
+ ? current
499
+ : [current];
500
+ const diff = (0, eval_baseline_js_1.diffReports)(baseline, reports, opts);
501
+ if (!diff.passed) {
502
+ const detail = diff.regressions
503
+ .map((r) => `${r.report}/${r.arm}/${r.metric} Δ=${r.comparison.delta.toFixed(3)} p=${r.comparison.pValue.toFixed(3)}`)
504
+ .join("; ");
505
+ fail(`regression vs baseline: ${detail}`);
506
+ }
507
+ }
433
508
  /**
434
509
  * Assert a skill/behaviour triggered on at least `min` (0..1) of its runs — the
435
510
  * reliability gate for a skill's *activation* (does its description fire on the
@@ -0,0 +1,9 @@
1
+ /**
2
+ * `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
3
+ * files. Re-exports the spec builders/types and the compiler under one
4
+ * concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
5
+ * keep working; this just groups them so the import name matches the pillar.
6
+ */
7
+ export * from "./spec.js";
8
+ export * from "./compile.js";
9
+ //# sourceMappingURL=linting.d.ts.map
@@ -0,0 +1,25 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ /**
18
+ * `vigiles/linting` — Pillar 1 entry point: the **linting layer** for instruction
19
+ * files. Re-exports the spec builders/types and the compiler under one
20
+ * concern-named import. The granular paths (`vigiles/spec`, `vigiles/compile`)
21
+ * keep working; this just groups them so the import name matches the pillar.
22
+ */
23
+ __exportStar(require("./spec.js"), exports);
24
+ __exportStar(require("./compile.js"), exports);
25
+ //# sourceMappingURL=linting.js.map
@@ -1,3 +1,5 @@
1
+ import { type SandboxMode, type EgressAttempt } from "./sandbox.js";
2
+ export type { EgressAttempt };
1
3
  /** A hook event payload (the JSON Claude Code writes to the hook's stdin). */
2
4
  export interface HookInput {
3
5
  /** e.g. "PreToolUse", "PostToolUse", "Stop", "SessionStart", "PreCompact". */
@@ -38,6 +40,41 @@ export interface RunHookOptions {
38
40
  readonly env?: Record<string, string>;
39
41
  /** Per-run timeout ms. Default 10000. */
40
42
  readonly timeoutMs?: number;
43
+ /**
44
+ * Provenance of the hook command. `true` (default) means YOU authored it — the
45
+ * usual case at this tier, a command written inline in the test — so it runs
46
+ * directly. `false` marks it foreign (a vendored third-party hook script),
47
+ * which makes confinement the DEFAULT: with no explicit `sandbox`, an untrusted
48
+ * hook behaves as `sandbox: "auto"` — confined under bubblewrap, or refused if
49
+ * none is available — so foreign code is never run unconfined by accident. This
50
+ * mirrors the harness tier, where trust follows `plugin`/`pluginDir`
51
+ * provenance (`specTrusted` in `src/sandbox.ts`); the unit tier takes a raw
52
+ * command string with no provenance signal, so you declare it here.
53
+ */
54
+ readonly trusted?: boolean;
55
+ /**
56
+ * Confine the hook under bubblewrap (Linux). When unset, the mode follows
57
+ * {@link RunHookOptions.trusted}: a trusted hook runs directly (`false`), an
58
+ * untrusted one is confined-or-refused (`"auto"`). Set it explicitly to
59
+ * override: `"auto"`/`"strict"` force confinement (a no-egress namespace with a
60
+ * cleared environment — your `opts.env` is added back — or a **refusal** if no
61
+ * bwrap is available), and `false` is the opt-out that runs even untrusted code
62
+ * unconfined. macOS/Windows have no bwrap, so `"auto"`/`"strict"` throw there —
63
+ * see `src/sandbox.ts`.
64
+ */
65
+ readonly sandbox?: SandboxMode;
66
+ /**
67
+ * Record the hook's network egress. Implies confinement (the recorder lives in
68
+ * the sandbox netns, so this forces a sandboxed run and refuses if no sandbox is
69
+ * available). A recording proxy on loopback captures every `host:port` a
70
+ * proxy-honoring tool (npm/pip/curl/fetch) tries to reach — surfaced as
71
+ * {@link HookRunResult.egress} — while the netns still **blocks** it (nothing
72
+ * actually leaves). Use it to test what a hook/skill phones home to, or which
73
+ * registry an install would hit. Raw-socket egress is blocked but not recorded
74
+ * (it never reaches the proxy) — the block is the boundary, the record is
75
+ * best-effort observability over it.
76
+ */
77
+ readonly recordEgress?: boolean;
41
78
  }
42
79
  export interface HookRunResult {
43
80
  readonly exitCode: number;
@@ -50,6 +87,17 @@ export interface HookRunResult {
50
87
  * `permissionDecision:"deny"` all set `blocked = true`.
51
88
  */
52
89
  readonly blocked: boolean;
90
+ /**
91
+ * Network egress the hook attempted, recorded then blocked. Empty unless
92
+ * {@link RunHookOptions.recordEgress} was set (and the run was confined).
93
+ */
94
+ readonly egress: readonly EgressAttempt[];
95
+ /**
96
+ * Files the hook wrote to its work dir (relative paths), recorded on confined
97
+ * runs — what a hook touched on disk. Empty on a direct (unconfined) run.
98
+ * Assert over it with `assertNoWrite` / `assertWroteOnly`.
99
+ */
100
+ readonly filesWritten: readonly string[];
53
101
  /**
54
102
  * The decision the hook expressed, preferring the structured
55
103
  * `permissionDecision` ("allow"|"deny"|"ask") then legacy `decision`
@@ -67,11 +115,43 @@ export declare function decideHook(exitCode: number, json: HookOutput | null): {
67
115
  blocked: boolean;
68
116
  decision: HookRunResult["decision"];
69
117
  };
118
+ /** The raw fields of a hook spawn that the result parser needs. */
119
+ export interface HookSpawnResult {
120
+ readonly status: number | null;
121
+ readonly signal: string | null;
122
+ readonly stdout: string;
123
+ readonly stderr: string;
124
+ /** Egress attempts captured by the in-sandbox recorder (recordEgress only). */
125
+ readonly egress?: readonly EgressAttempt[];
126
+ /** Files the hook wrote to its work dir (confined runs). */
127
+ readonly filesWritten?: readonly string[];
128
+ }
129
+ /** Spawn a hook (command + piped event) — the injectable seam over the real spawn. */
130
+ export type HookSpawner = (command: string, input: HookInput, opts: RunHookOptions) => HookSpawnResult;
131
+ /** The spawn seams `runHookWith` needs, so its decision logic is testable. */
132
+ export interface RunHookDeps {
133
+ /** Whether bubblewrap confinement is available (Linux + bwrap). */
134
+ readonly available: boolean;
135
+ /** Run the command directly (unconfined). */
136
+ readonly direct: HookSpawner;
137
+ /** Run the command confined under bubblewrap. */
138
+ readonly sandboxed: HookSpawner;
139
+ }
140
+ /**
141
+ * The hook-run orchestration with injectable spawn seams: pick direct vs.
142
+ * confined via the safe-by-default policy (`decideSandbox`), then parse the exit
143
+ * code + stdout into a normalized decision. Exported so all three branches
144
+ * (direct / sandbox / refuse) are unit-tested with fake spawners — no real
145
+ * bwrap. `runHook` is this with the real seams.
146
+ */
147
+ export declare function runHookWith(command: string, input: HookInput, opts: RunHookOptions, deps: RunHookDeps): HookRunResult;
70
148
  /**
71
149
  * Run a hook command, piping `input` as JSON to its stdin, and report the exit
72
150
  * code + parsed decision. Synchronous (so it can be used inside an eval's
73
151
  * `measure` too). `command` is run through a shell, so the same command string a
74
- * plugin ships (with args / env refs) works verbatim.
152
+ * plugin ships (with args / env refs) works verbatim. Mark a hook you didn't
153
+ * write with `trusted: false` and it is confined by default (or pass `sandbox:
154
+ * "auto"` directly) — see {@link RunHookOptions.trusted}.
75
155
  */
76
156
  export declare function runHook(command: string, input: HookInput, opts?: RunHookOptions): HookRunResult;
77
157
  //# sourceMappingURL=run-hook.d.ts.map