vigiles 15.3.0 → 15.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/run-scripts.d.ts +4 -4
- package/dist/adapters/claude-code/run-scripts.js +6 -6
- package/dist/adapters/claude-code/skill-driver.d.ts +3 -0
- package/dist/adapters/claude-code/skill-driver.js +3 -0
- package/dist/adapters/claude-code/typed-spec.d.ts +1 -1
- package/dist/adapters/claude-code/typed-spec.js +3 -3
- package/dist/audit-report.template.html +1 -1
- package/dist/audit-score.d.ts +1 -1
- package/dist/audit-score.js +2 -2
- package/dist/check-count.js +1 -1
- package/dist/claude-code.d.ts +2 -2
- package/dist/claude-code.js +15 -9
- package/dist/cli.js +8 -8
- package/dist/codex.d.ts +1 -1
- package/dist/codex.js +1 -1
- package/dist/core/adopt.d.ts +1 -1
- package/dist/core/adopt.js +4 -4
- package/dist/core/compile-generator.d.ts +9 -0
- package/dist/core/compile-generator.js +9 -0
- package/dist/core/spec.d.ts +26 -20
- package/dist/core/spec.js +34 -26
- package/dist/eval-surface.d.ts +122 -0
- package/dist/eval-surface.js +130 -0
- package/dist/experimental.d.ts +1 -1
- package/dist/experimental.js +2 -2
- package/dist/harness-assert.d.ts +1 -1
- package/dist/harness-assert.js +3 -2
- package/dist/load-hook.d.ts +1 -1
- package/dist/load-hook.js +1 -1
- package/dist/run-hook.js +1 -1
- package/dist/scaffold-test.js +7 -11
- package/dist/scan-trigger-suggest.d.ts +3 -3
- package/dist/scan-trigger-suggest.js +4 -4
- package/dist/services-docker.d.ts +1 -1
- package/dist/services-docker.js +3 -3
- package/dist/skill-test.d.ts +5 -0
- package/dist/skill-test.js +5 -0
- package/dist/test-coverage.js +1 -1
- package/dist/test.d.ts +75 -0
- package/dist/test.js +192 -0
- package/package.json +8 -8
- package/skills/test-harness/SKILL.md +16 -14
- package/dist/e2e.d.ts +0 -16
- package/dist/e2e.js +0 -32
- package/dist/integration.d.ts +0 -29
- package/dist/integration.js +0 -58
- package/dist/mutations.d.ts +0 -96
- package/dist/mutations.js +0 -237
- package/dist/testing.d.ts +0 -35
- package/dist/testing.js +0 -124
- package/dist/unit.d.ts +0 -24
- package/dist/unit.js +0 -59
package/dist/audit-score.js
CHANGED
|
@@ -28,7 +28,7 @@ exports.formatAuditScore = formatAuditScore;
|
|
|
28
28
|
* design). NB the EXECUTING
|
|
29
29
|
* "do your hooks actually block?" disaster-battery is STILL not an `audit` ring:
|
|
30
30
|
* running arbitrary hooks safely needs cross-platform confinement that isn't
|
|
31
|
-
* shipped yet, so the battery lives in the `vigiles
|
|
31
|
+
* shipped yet, so the battery lives in the `vigiles` testing API via
|
|
32
32
|
* `guardrail-check`/`assertBlocksDisasters`, where you opt in explicitly.
|
|
33
33
|
*
|
|
34
34
|
* TESTED vs EVALUATED are two rings, not one, because a harness and an eval differ
|
|
@@ -49,7 +49,7 @@ const score_core_js_1 = require("./score-core.js");
|
|
|
49
49
|
const W_UNTESTED = 3;
|
|
50
50
|
/** The command that answers the firing question — named, not alluded to. */
|
|
51
51
|
const MEASURE_FIRING_COMMAND = "run `npx vigiles audit` interactively to measure, or add a `*.eval.mjs` " +
|
|
52
|
-
"(`
|
|
52
|
+
"(`paid_measureTriggerRate`, vigiles/eval)";
|
|
53
53
|
/** Resolve the terse "thing(s)" plural placeholder against a count:
|
|
54
54
|
* n===1 drops the "(s)" ("1 tool"); otherwise it becomes "s" ("3 tools"). */
|
|
55
55
|
function pluralizeLabel(n, label) {
|
package/dist/check-count.js
CHANGED
|
@@ -46,7 +46,7 @@ exports.armCheckReport = armCheckReport;
|
|
|
46
46
|
* tool avoids.
|
|
47
47
|
*
|
|
48
48
|
* WHAT A MISSING COUNT MEANS: nothing at all. A script that never imports
|
|
49
|
-
* `vigiles
|
|
49
|
+
* `vigiles` cannot report, so the runner sees no file and treats it
|
|
50
50
|
* exactly as before — a plain pass. Silence is the legacy branch, never a
|
|
51
51
|
* verdict; only a count of literally zero is a finding. (The alternative —
|
|
52
52
|
* force-loading this module into every child with `node --import` so silence
|
package/dist/claude-code.d.ts
CHANGED
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
* `vigiles/claude-code` — the Claude Code-specific harness pieces a *different*
|
|
3
3
|
* harness would swap out: the plugin/repo loader (reads real Claude Code plugin
|
|
4
4
|
* layouts) and the scriptable Anthropic Messages mock. Split from
|
|
5
|
-
* `vigiles
|
|
5
|
+
* the `vigiles` root on purpose — the test API above is the stable surface; this
|
|
6
6
|
* is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
|
|
7
7
|
*/
|
|
8
8
|
export * from "./adapters/claude-code/plugin-loader.js";
|
|
9
9
|
export * from "./mock-model.js";
|
|
10
10
|
export { claudeCodeDriver, buildClaudeArgs, parseClaudeRun, claudeAvailable, } from "./harness-test.js";
|
|
11
11
|
export * from "./adapters/claude-code/dialect.js";
|
|
12
|
-
export { agent,
|
|
12
|
+
export { agent, experimental_skill, type ClaudeCodeToolVocabulary, } from "./adapters/claude-code/typed-spec.js";
|
|
13
13
|
export { measureSelectionMatrix, assertNoCollision, formatSelectionReport, } from "./scan-behavioral.js";
|
|
14
14
|
export type { SelectionReport, SkillSelectionStat, SelectionOptions, SelectionMatrixOptions, } from "./scan-behavioral.js";
|
|
15
15
|
export * from "./adapters/claude-code/layout.js";
|
package/dist/claude-code.js
CHANGED
|
@@ -14,12 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.formatSelectionReport = exports.assertNoCollision = exports.measureSelectionMatrix = exports.
|
|
17
|
+
exports.formatSelectionReport = exports.assertNoCollision = exports.measureSelectionMatrix = exports.experimental_skill = exports.agent = exports.claudeAvailable = exports.parseClaudeRun = exports.buildClaudeArgs = exports.claudeCodeDriver = void 0;
|
|
18
18
|
/**
|
|
19
19
|
* `vigiles/claude-code` — the Claude Code-specific harness pieces a *different*
|
|
20
20
|
* harness would swap out: the plugin/repo loader (reads real Claude Code plugin
|
|
21
21
|
* layouts) and the scriptable Anthropic Messages mock. Split from
|
|
22
|
-
* `vigiles
|
|
22
|
+
* the `vigiles` root on purpose — the test API above is the stable surface; this
|
|
23
23
|
* is the adapter, so a future `vigiles/<other-harness>` can sit beside it.
|
|
24
24
|
*/
|
|
25
25
|
__exportStar(require("./adapters/claude-code/plugin-loader.js"), exports);
|
|
@@ -28,23 +28,29 @@ __exportStar(require("./mock-model.js"), exports);
|
|
|
28
28
|
// helpers + the `claude` capability probe. Agnostic users never need these
|
|
29
29
|
// (`runHarnessTest` defaults to the CC driver), but they're exposed here — beside
|
|
30
30
|
// the Codex driver in `vigiles/codex` — for CC-specific tests/tooling. They are
|
|
31
|
-
// deliberately NOT on the agnostic `vigiles
|
|
31
|
+
// deliberately NOT on the agnostic `vigiles` surface.
|
|
32
32
|
var harness_test_js_1 = require("./harness-test.js");
|
|
33
33
|
Object.defineProperty(exports, "claudeCodeDriver", { enumerable: true, get: function () { return harness_test_js_1.claudeCodeDriver; } });
|
|
34
34
|
Object.defineProperty(exports, "buildClaudeArgs", { enumerable: true, get: function () { return harness_test_js_1.buildClaudeArgs; } });
|
|
35
35
|
Object.defineProperty(exports, "parseClaudeRun", { enumerable: true, get: function () { return harness_test_js_1.parseClaudeRun; } });
|
|
36
36
|
Object.defineProperty(exports, "claudeAvailable", { enumerable: true, get: function () { return harness_test_js_1.claudeAvailable; } });
|
|
37
37
|
__exportStar(require("./adapters/claude-code/dialect.js"), exports);
|
|
38
|
-
// The typed Claude Code authoring surface: `agent` / `
|
|
39
|
-
// floor enforced AT COMPILE TIME against the CC tool catalog (a
|
|
40
|
-
// e.g. `purity: "pure"` + `"Bash"`). A strict addition to the
|
|
41
|
-
// purity checks; the bare core `agent()`/`
|
|
38
|
+
// The typed Claude Code authoring surface: `agent` / `experimental_skill` with
|
|
39
|
+
// the `purity` floor enforced AT COMPILE TIME against the CC tool catalog (a
|
|
40
|
+
// `tsc` error for e.g. `purity: "pure"` + `"Bash"`). A strict addition to the
|
|
41
|
+
// runtime/compile purity checks; the bare core `agent()`/`experimental_skill()`
|
|
42
|
+
// (`vigiles/spec`) stay open.
|
|
43
|
+
//
|
|
44
|
+
// NOTE the collision this rename also resolves: `vigiles/testing` exports a
|
|
45
|
+
// `skill()` too, and it is a DIFFERENT function — a `Check<Trace>` asking "did
|
|
46
|
+
// this skill fire?", taking an id string. One word, two concepts, told apart
|
|
47
|
+
// only by which door you imported from. Only the authoring builder is prefixed.
|
|
42
48
|
var typed_spec_js_1 = require("./adapters/claude-code/typed-spec.js");
|
|
43
49
|
Object.defineProperty(exports, "agent", { enumerable: true, get: function () { return typed_spec_js_1.agent; } });
|
|
44
|
-
Object.defineProperty(exports, "
|
|
50
|
+
Object.defineProperty(exports, "experimental_skill", { enumerable: true, get: function () { return typed_spec_js_1.experimental_skill; } });
|
|
45
51
|
// Selection-collision — a Claude-Code-ONLY behavioral measurement (Codex has no
|
|
46
52
|
// skill-selection event to read), so it lives on this surface, not the agnostic
|
|
47
|
-
// `vigiles
|
|
53
|
+
// `vigiles`. `measureSelectionMatrix` builds the N×N "which skill fired?"
|
|
48
54
|
// matrix (diagonal = recall, off-diagonal = collision); `assertNoCollision` gates it.
|
|
49
55
|
var scan_behavioral_js_1 = require("./scan-behavioral.js");
|
|
50
56
|
Object.defineProperty(exports, "measureSelectionMatrix", { enumerable: true, get: function () { return scan_behavioral_js_1.measureSelectionMatrix; } });
|
package/dist/cli.js
CHANGED
|
@@ -1702,7 +1702,7 @@ function formatTriggerNudge(triggerableSkills) {
|
|
|
1702
1702
|
return "";
|
|
1703
1703
|
const n = triggerableSkills;
|
|
1704
1704
|
return (`ℹ Do your ${String(n)} skill${n === 1 ? "" : "s"} actually fire? The deterministic read can't tell — ` +
|
|
1705
|
-
`run \`audit\` interactively to measure, or test with \`
|
|
1705
|
+
`run \`audit\` interactively to measure, or test with \`paid_measureTriggerRate\` (vigiles/eval).`);
|
|
1706
1706
|
}
|
|
1707
1707
|
/** A terminal summary of the rule map: the CONFIDENT lane counts + the POSSIBLE
|
|
1708
1708
|
* (review) and SKIPPED tiers, with the honest caveat that detection is a
|
|
@@ -1973,7 +1973,7 @@ function vigilesWorkflow(plan, harnesses, hasPackageJson) {
|
|
|
1973
1973
|
`
|
|
1974
1974
|
: "";
|
|
1975
1975
|
// The test pillar's harness job runs `*.harness.mjs` files that
|
|
1976
|
-
// `import "vigiles
|
|
1976
|
+
// `import "vigiles"`, so it needs vigiles RESOLVABLE — a package.json +
|
|
1977
1977
|
// install. A repo with no package.json can't run the JS test tier in CI at all
|
|
1978
1978
|
// (the import won't resolve even though the CLI itself came from npx), so the
|
|
1979
1979
|
// harness job is emitted ONLY when a package.json exists (Codex review / #111 /
|
|
@@ -2136,7 +2136,7 @@ const STARTER_HARNESS = `/**
|
|
|
2136
2136
|
*
|
|
2137
2137
|
* Guide: https://github.com/zernie/vigiles/blob/main/docs/harness-testing.md
|
|
2138
2138
|
*/
|
|
2139
|
-
import { runHook } from "vigiles
|
|
2139
|
+
import { runHook } from "vigiles";
|
|
2140
2140
|
import assert from "node:assert/strict";
|
|
2141
2141
|
|
|
2142
2142
|
// EXAMPLE — replace with one of YOUR hooks. This PreToolUse Bash guard blocks a
|
|
@@ -4384,7 +4384,7 @@ const COMMAND_HELP = {
|
|
|
4384
4384
|
usage: " vigiles audit [dir...] Lighthouse for your harness — a LOCAL report: rings + what's broken + fixes (a deterministic read; 2+ dirs → leaderboard)",
|
|
4385
4385
|
detail: [
|
|
4386
4386
|
" writes vigiles-report.html + .json (auto-gitignored; --out=<dir> · --no-html/--no-json · --no-open · --json for machine output). NOT a CI step — use `vigiles lint` in CI.",
|
|
4387
|
-
" the executing checks (run your hooks · live MCP · do skills fire?) run only interactively — `audit` asks once (remembered); automation uses the vigiles
|
|
4387
|
+
" the executing checks (run your hooks · live MCP · do skills fire?) run only interactively — `audit` asks once (remembered); automation uses the vigiles testing API",
|
|
4388
4388
|
" --serve opens a LIVE local report whose buttons create specs in one click (own repo only; loopback + token-guarded) · --no-serve to skip the prompt",
|
|
4389
4389
|
],
|
|
4390
4390
|
},
|
|
@@ -4970,7 +4970,7 @@ function refsHookCommand() {
|
|
|
4970
4970
|
// ---------------------------------------------------------------------------
|
|
4971
4971
|
/**
|
|
4972
4972
|
* Load a compiled-hook program's default export — the SHARED loader, also the
|
|
4973
|
-
* public `vigiles
|
|
4973
|
+
* public `vigiles` `loadHook` a `.harness.mjs` test uses, so a hook that
|
|
4974
4974
|
* loads in a test loads identically here (one loader, no drift).
|
|
4975
4975
|
*/
|
|
4976
4976
|
const loadHookProgram = load_hook_js_1.loadHook;
|
|
@@ -6135,7 +6135,7 @@ async function main() {
|
|
|
6135
6135
|
// uses `vigiles lint`). The executing checks (safety battery + live MCP +
|
|
6136
6136
|
// skill firing) run only on consent: at a TTY `audit` asks once (remembered
|
|
6137
6137
|
// in `.vigilesrc.json`), headless it stays a read + a one-line nudge. There
|
|
6138
|
-
// is NO execution flag — automation tests the harness via the vigiles
|
|
6138
|
+
// is NO execution flag — automation tests the harness via the vigiles testing
|
|
6139
6139
|
// API + skills, not the report verb. See the `audit-side-effect-free` rule.
|
|
6140
6140
|
const dirs = restArgs.length > 0 ? restArgs : ["."];
|
|
6141
6141
|
const json = args.includes("--json");
|
|
@@ -6337,10 +6337,10 @@ async function main() {
|
|
|
6337
6337
|
// linter also executes it). A plain `audit` is a deterministic READ;
|
|
6338
6338
|
// these run only on consent — ASK once at a TTY (remembered); headless
|
|
6339
6339
|
// stays a read + a nudge (no execution flag — automation uses the
|
|
6340
|
-
// vigiles
|
|
6340
|
+
// vigiles testing API). Resolved AFTER the report (so the read leads) but
|
|
6341
6341
|
// BEFORE the rule map is routed, so a first-time "yes" enriches THIS run.
|
|
6342
6342
|
// (The safety battery is NOT here — it needs cross-platform confinement
|
|
6343
|
-
// that isn't shipped, so it lives in the vigiles
|
|
6343
|
+
// that isn't shipped, so it lives in the vigiles testing API.)
|
|
6344
6344
|
// `surfaces` + `execDecision` were resolved above (the ring needed them);
|
|
6345
6345
|
// this only turns an `ask` into a prompt and remembers the answer.
|
|
6346
6346
|
const { execute, note: execNote } = await resolveExecution(surfaces, execDecision, json, adapter.name);
|
package/dist/codex.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `vigiles/codex` — the OpenAI Codex harness adapter. Sits beside
|
|
3
|
-
* `vigiles/claude-code`: same harness-agnostic `vigiles
|
|
3
|
+
* `vigiles/claude-code`: same harness-agnostic `vigiles` testing core, a
|
|
4
4
|
* different transport (the `codex` binary + the OpenAI **Responses** SSE mock).
|
|
5
5
|
*
|
|
6
6
|
* Pillar 2 (harness testing) is proven here — `startCodexMock` serves the
|
package/dist/codex.js
CHANGED
|
@@ -16,7 +16,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/**
|
|
18
18
|
* `vigiles/codex` — the OpenAI Codex harness adapter. Sits beside
|
|
19
|
-
* `vigiles/claude-code`: same harness-agnostic `vigiles
|
|
19
|
+
* `vigiles/claude-code`: same harness-agnostic `vigiles` testing core, a
|
|
20
20
|
* different transport (the `codex` binary + the OpenAI **Responses** SSE mock).
|
|
21
21
|
*
|
|
22
22
|
* Pillar 2 (harness testing) is proven here — `startCodexMock` serves the
|
package/dist/core/adopt.d.ts
CHANGED
|
@@ -80,7 +80,7 @@ export interface AdoptSurfaceResult {
|
|
|
80
80
|
unmappedKeys: string[];
|
|
81
81
|
}
|
|
82
82
|
/**
|
|
83
|
-
* Adopt an existing SKILL.md into
|
|
83
|
+
* Adopt an existing SKILL.md into an `experimental_skill()` spec. The body is carried verbatim
|
|
84
84
|
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
85
85
|
* skill round-trips below the integrity header.
|
|
86
86
|
*
|
package/dist/core/adopt.js
CHANGED
|
@@ -289,7 +289,7 @@ function unmappedNote(kind, keys) {
|
|
|
289
289
|
const SURFACE_HEADER = (from) => `// Adopted from ${from} by \`vigiles init\` — body verbatim, standard\n` +
|
|
290
290
|
`// frontmatter mapped; no rules inferred. Review the diff, then \`compile\`.\n`;
|
|
291
291
|
/**
|
|
292
|
-
* Adopt an existing SKILL.md into
|
|
292
|
+
* Adopt an existing SKILL.md into an `experimental_skill()` spec. The body is carried verbatim
|
|
293
293
|
* (skills are freeform markdown — `##` headings stay in the body), so a clean
|
|
294
294
|
* skill round-trips below the integrity header.
|
|
295
295
|
*
|
|
@@ -325,7 +325,7 @@ function adoptSkill(markdown, dirName) {
|
|
|
325
325
|
const trimmedBody = body.trim();
|
|
326
326
|
if (trimmedBody)
|
|
327
327
|
lines.push(` body: ${tsTemplate(trimmedBody)},`);
|
|
328
|
-
const spec = (0, spec_js_1.
|
|
328
|
+
const spec = (0, spec_js_1.experimental_skill)({
|
|
329
329
|
name,
|
|
330
330
|
description,
|
|
331
331
|
...(argumentHint ? { argumentHint } : {}),
|
|
@@ -336,8 +336,8 @@ function adoptSkill(markdown, dirName) {
|
|
|
336
336
|
});
|
|
337
337
|
const source = SURFACE_HEADER(`${dirName}/SKILL.md`) +
|
|
338
338
|
unmappedNote("skill", unmappedKeys) +
|
|
339
|
-
`import {
|
|
340
|
-
`export default
|
|
339
|
+
`import { experimental_skill } from "vigiles/spec";\n\n` +
|
|
340
|
+
`export default experimental_skill({\n${lines.join("\n")}\n});\n`;
|
|
341
341
|
return { source, kind: "skill", spec, unmappedKeys };
|
|
342
342
|
}
|
|
343
343
|
/**
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* vigiles — Generator → SKILL.md compiler.
|
|
3
3
|
*
|
|
4
|
+
* PARKED (2026-08-16). Not part of the public API and not in `docs/skills.md`.
|
|
5
|
+
* It works — `vigiles compile` detects `genSkill(` and routes here (`cli.ts`),
|
|
6
|
+
* verified end-to-end — but `genSkill` / `act` / `checkpoint` / `finish` /
|
|
7
|
+
* `runSkill` are reachable from NO package subpath, so nobody outside this repo
|
|
8
|
+
* can author one. Parked rather than deleted because the code is sound and its
|
|
9
|
+
* tests pass; there is simply nothing to evaluate its usefulness against.
|
|
10
|
+
* Do not re-document it or give it an entry point without settling that first —
|
|
11
|
+
* see `docs/skills.md` §Status and the adoption-ladder work.
|
|
12
|
+
*
|
|
4
13
|
* A generator skill (`function* () { … yield act/gate/result … }`) has dynamic
|
|
5
14
|
* control flow, so it can't be rendered by *executing* it. Instead we parse its
|
|
6
15
|
* SOURCE with the TypeScript compiler API and render the structure to markdown:
|
|
@@ -2,6 +2,15 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* vigiles — Generator → SKILL.md compiler.
|
|
4
4
|
*
|
|
5
|
+
* PARKED (2026-08-16). Not part of the public API and not in `docs/skills.md`.
|
|
6
|
+
* It works — `vigiles compile` detects `genSkill(` and routes here (`cli.ts`),
|
|
7
|
+
* verified end-to-end — but `genSkill` / `act` / `checkpoint` / `finish` /
|
|
8
|
+
* `runSkill` are reachable from NO package subpath, so nobody outside this repo
|
|
9
|
+
* can author one. Parked rather than deleted because the code is sound and its
|
|
10
|
+
* tests pass; there is simply nothing to evaluate its usefulness against.
|
|
11
|
+
* Do not re-document it or give it an entry point without settling that first —
|
|
12
|
+
* see `docs/skills.md` §Status and the adoption-ladder work.
|
|
13
|
+
*
|
|
5
14
|
* A generator skill (`function* () { … yield act/gate/result … }`) has dynamic
|
|
6
15
|
* control flow, so it can't be rendered by *executing* it. Instead we parse its
|
|
7
16
|
* SOURCE with the TypeScript compiler API and render the structure to markdown:
|
package/dist/core/spec.d.ts
CHANGED
|
@@ -281,10 +281,10 @@ export declare function instructions(strings: TemplateStringsArray, ...values: I
|
|
|
281
281
|
* `<!-- vigiles:effect -->` markers. Independent of the `doc()` authoring
|
|
282
282
|
* surface — it does not block on it.
|
|
283
283
|
*
|
|
284
|
-
* @
|
|
284
|
+
* @experimental Experimental (parked P3) — NOT part of the frozen public surface;
|
|
285
285
|
* may change or be removed without a major bump pre-1.0.
|
|
286
286
|
*/
|
|
287
|
-
export declare function
|
|
287
|
+
export declare function experimental_effect(strings: TemplateStringsArray, ...values: InstructionFragment[]): EffectRegion;
|
|
288
288
|
/**
|
|
289
289
|
* The purity an author DECLARES for a skill/agent — the floor `compile`
|
|
290
290
|
* enforces against the tool contract (see `purityViolations` in
|
|
@@ -493,14 +493,20 @@ export type SkillSpecInput<P extends AuthoredPurity | undefined, V extends ToolV
|
|
|
493
493
|
* Define a SKILL.md specification.
|
|
494
494
|
*
|
|
495
495
|
* // skills/my-skill/SKILL.md.spec.ts
|
|
496
|
-
* export default
|
|
496
|
+
* export default experimental_skill({ name: "my-skill", description: "…" });
|
|
497
497
|
*
|
|
498
498
|
* Generic over a tool `Vocabulary` (default `OpenToolVocabulary` — no
|
|
499
|
-
* constraint), exactly like `agent()`: a vocabulary-bound `
|
|
500
|
-
* `vigiles/claude-code`) makes `purity: "pure"` + a side-effecting tool a
|
|
501
|
-
* error; the bare core
|
|
499
|
+
* constraint), exactly like `agent()`: a vocabulary-bound `experimental_skill`
|
|
500
|
+
* (e.g. `vigiles/claude-code`) makes `purity: "pure"` + a side-effecting tool a
|
|
501
|
+
* `tsc` error; the bare core one accepts any tools, as before.
|
|
502
|
+
*
|
|
503
|
+
* The name also disambiguates: `vigiles/testing` exports a `skill()` that is a
|
|
504
|
+
* DIFFERENT function — a `Check<Trace>` taking an id string, asking whether a
|
|
505
|
+
* skill fired. This one authors a skill; that one observes one.
|
|
506
|
+
*
|
|
507
|
+
* @experimental
|
|
502
508
|
*/
|
|
503
|
-
export declare function
|
|
509
|
+
export declare function experimental_skill<const P extends AuthoredPurity | undefined = undefined, V extends ToolVocabulary = OpenToolVocabulary>(spec: SkillSpecInput<P, V>): SkillSpec;
|
|
504
510
|
/**
|
|
505
511
|
* A subagent definition (compiles to `agents/<name>.md`). Unlike a skill —
|
|
506
512
|
* reference material the model reads on activation — a subagent is a *delegated
|
|
@@ -758,10 +764,10 @@ export type NeedsContract<N extends Shape> = N;
|
|
|
758
764
|
*
|
|
759
765
|
* needs({ plan: "string", files: "string[]" })
|
|
760
766
|
*
|
|
761
|
-
* @
|
|
767
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
762
768
|
* public API (pre-1.0); may change without a major bump.
|
|
763
769
|
*/
|
|
764
|
-
export declare function
|
|
770
|
+
export declare function experimental_needs<const N extends Shape>(shape: N): NeedsContract<N>;
|
|
765
771
|
/**
|
|
766
772
|
* A typed pipeline step: a `TypedAgentSpec` paired with the input `needs` it
|
|
767
773
|
* reads from the prior step's `ok`. `step()` builds one; `pipe` checks that the
|
|
@@ -782,10 +788,10 @@ export interface PipeStep<Needs extends Shape, Ok extends Shape, Err extends Sha
|
|
|
782
788
|
*
|
|
783
789
|
* pipeStep(implementer, needs({ plan: "string", files: "string[]" }))
|
|
784
790
|
*
|
|
785
|
-
* @
|
|
791
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
786
792
|
* public API (pre-1.0); may change without a major bump.
|
|
787
793
|
*/
|
|
788
|
-
export declare function
|
|
794
|
+
export declare function experimental_pipeStep<Needs extends Shape, Ok extends Shape, Err extends Shape>(a: TypedAgentSpec<Ok, Err>, needsContract?: Needs): PipeStep<Needs, Ok, Err>;
|
|
789
795
|
/**
|
|
790
796
|
* True iff `Producer` provides EVERY field `Consumer` needs, with matching
|
|
791
797
|
* field types. When satisfiable it is `true`; otherwise it collapses to a
|
|
@@ -844,10 +850,10 @@ export interface Pipeline<Ok extends Shape, Err extends Shape> {
|
|
|
844
850
|
* its `needs` must be empty (`needs({})` or omitted). Returns a `Pipeline`
|
|
845
851
|
* carrying that step's `ok`/`err` forward.
|
|
846
852
|
*
|
|
847
|
-
* @
|
|
853
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
848
854
|
* public API (pre-1.0); may change without a major bump.
|
|
849
855
|
*/
|
|
850
|
-
export declare function
|
|
856
|
+
export declare function experimental_start<Ok extends Shape, Err extends Shape>(first: PipeStep<Record<string, never>, Ok, Err> | TypedAgentSpec<Ok, Err>): Pipeline<Ok, Err>;
|
|
851
857
|
/**
|
|
852
858
|
* Append a step to a typed pipeline. The handoff is CHECKED: the constraint
|
|
853
859
|
* `Supplies<PriorOk, Needs>` must be `true`, else the `next` parameter's type
|
|
@@ -859,10 +865,10 @@ export declare function start<Ok extends Shape, Err extends Shape>(first: PipeSt
|
|
|
859
865
|
* exporting a function called `then` becomes a thenable, so `await import()` of
|
|
860
866
|
* any barrel re-exporting it would invoke it — a footgun the rename avoids.
|
|
861
867
|
*
|
|
862
|
-
* @
|
|
868
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
863
869
|
* public API (pre-1.0); may change without a major bump.
|
|
864
870
|
*/
|
|
865
|
-
export declare function
|
|
871
|
+
export declare function experimental_andThen<PriorOk extends Shape, PriorErr extends Shape, Needs extends Shape, Ok extends Shape, Err extends Shape>(prior: Pipeline<PriorOk, PriorErr>, next: Supplies<PriorOk, Needs> extends true ? PipeStep<Needs, Ok, Err> : {
|
|
866
872
|
readonly __HANDOFF_ERROR: Supplies<PriorOk, Needs>;
|
|
867
873
|
}): Pipeline<Ok, PriorErr | Err>;
|
|
868
874
|
/**
|
|
@@ -882,19 +888,19 @@ export declare function andThen<PriorOk extends Shape, PriorErr extends Shape, N
|
|
|
882
888
|
* pipeStep(reviewer, needs({ diff: "string" })),
|
|
883
889
|
* ) // ← won't compile if a handoff doesn't line up
|
|
884
890
|
*
|
|
885
|
-
* @
|
|
891
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
886
892
|
* public API (pre-1.0); may change without a major bump.
|
|
887
893
|
*/
|
|
888
|
-
export declare function
|
|
889
|
-
export declare function
|
|
894
|
+
export declare function experimental_pipe<A extends Shape, AE extends Shape>(a: TypedAgentSpec<A, AE>): Pipeline<A, AE>;
|
|
895
|
+
export declare function experimental_pipe<A extends Shape, AE extends Shape, BN extends Shape, B extends Shape, BE extends Shape>(a: TypedAgentSpec<A, AE>, b: Supplies<A, BN> extends true ? PipeStep<BN, B, BE> : {
|
|
890
896
|
readonly __HANDOFF_ERROR: Supplies<A, BN>;
|
|
891
897
|
}): Pipeline<B, AE | BE>;
|
|
892
|
-
export declare function
|
|
898
|
+
export declare function experimental_pipe<A extends Shape, AE extends Shape, BN extends Shape, B extends Shape, BE extends Shape, CN extends Shape, C extends Shape, CE extends Shape>(a: TypedAgentSpec<A, AE>, b: Supplies<A, BN> extends true ? PipeStep<BN, B, BE> : {
|
|
893
899
|
readonly __HANDOFF_ERROR: Supplies<A, BN>;
|
|
894
900
|
}, c: Supplies<B, CN> extends true ? PipeStep<CN, C, CE> : {
|
|
895
901
|
readonly __HANDOFF_ERROR: Supplies<B, CN>;
|
|
896
902
|
}): Pipeline<C, AE | BE | CE>;
|
|
897
|
-
export declare function
|
|
903
|
+
export declare function experimental_pipe<A extends Shape, AE extends Shape, BN extends Shape, B extends Shape, BE extends Shape, CN extends Shape, C extends Shape, CE extends Shape, DN extends Shape, D extends Shape, DE extends Shape>(a: TypedAgentSpec<A, AE>, b: Supplies<A, BN> extends true ? PipeStep<BN, B, BE> : {
|
|
898
904
|
readonly __HANDOFF_ERROR: Supplies<A, BN>;
|
|
899
905
|
}, c: Supplies<B, CN> extends true ? PipeStep<CN, C, CE> : {
|
|
900
906
|
readonly __HANDOFF_ERROR: Supplies<B, CN>;
|
package/dist/core/spec.js
CHANGED
|
@@ -21,21 +21,21 @@ exports.ref = ref;
|
|
|
21
21
|
exports.dir = dir;
|
|
22
22
|
exports.glob = glob;
|
|
23
23
|
exports.instructions = instructions;
|
|
24
|
-
exports.
|
|
24
|
+
exports.experimental_effect = experimental_effect;
|
|
25
25
|
exports.claude = claude;
|
|
26
26
|
exports.project = project;
|
|
27
27
|
exports.input = input;
|
|
28
28
|
exports.step = step;
|
|
29
|
-
exports.
|
|
29
|
+
exports.experimental_skill = experimental_skill;
|
|
30
30
|
exports.agent = agent;
|
|
31
31
|
exports.result = result;
|
|
32
32
|
exports.delegate = delegate;
|
|
33
33
|
exports.railway = railway;
|
|
34
|
-
exports.
|
|
35
|
-
exports.
|
|
36
|
-
exports.
|
|
37
|
-
exports.
|
|
38
|
-
exports.
|
|
34
|
+
exports.experimental_needs = experimental_needs;
|
|
35
|
+
exports.experimental_pipeStep = experimental_pipeStep;
|
|
36
|
+
exports.experimental_start = experimental_start;
|
|
37
|
+
exports.experimental_andThen = experimental_andThen;
|
|
38
|
+
exports.experimental_pipe = experimental_pipe;
|
|
39
39
|
exports.defineConfig = defineConfig;
|
|
40
40
|
// ---------------------------------------------------------------------------
|
|
41
41
|
// Template literal types for type-safe linter references
|
|
@@ -192,10 +192,10 @@ function instructions(strings, ...values) {
|
|
|
192
192
|
* `<!-- vigiles:effect -->` markers. Independent of the `doc()` authoring
|
|
193
193
|
* surface — it does not block on it.
|
|
194
194
|
*
|
|
195
|
-
* @
|
|
195
|
+
* @experimental Experimental (parked P3) — NOT part of the frozen public surface;
|
|
196
196
|
* may change or be removed without a major bump pre-1.0.
|
|
197
197
|
*/
|
|
198
|
-
function
|
|
198
|
+
function experimental_effect(strings, ...values) {
|
|
199
199
|
const body = [];
|
|
200
200
|
for (let i = 0; i < strings.length; i++) {
|
|
201
201
|
if (strings[i])
|
|
@@ -234,14 +234,20 @@ function step(instr, opts = {}) {
|
|
|
234
234
|
* Define a SKILL.md specification.
|
|
235
235
|
*
|
|
236
236
|
* // skills/my-skill/SKILL.md.spec.ts
|
|
237
|
-
* export default
|
|
237
|
+
* export default experimental_skill({ name: "my-skill", description: "…" });
|
|
238
238
|
*
|
|
239
239
|
* Generic over a tool `Vocabulary` (default `OpenToolVocabulary` — no
|
|
240
|
-
* constraint), exactly like `agent()`: a vocabulary-bound `
|
|
241
|
-
* `vigiles/claude-code`) makes `purity: "pure"` + a side-effecting tool a
|
|
242
|
-
* error; the bare core
|
|
240
|
+
* constraint), exactly like `agent()`: a vocabulary-bound `experimental_skill`
|
|
241
|
+
* (e.g. `vigiles/claude-code`) makes `purity: "pure"` + a side-effecting tool a
|
|
242
|
+
* `tsc` error; the bare core one accepts any tools, as before.
|
|
243
|
+
*
|
|
244
|
+
* The name also disambiguates: `vigiles/testing` exports a `skill()` that is a
|
|
245
|
+
* DIFFERENT function — a `Check<Trace>` taking an id string, asking whether a
|
|
246
|
+
* skill fired. This one authors a skill; that one observes one.
|
|
247
|
+
*
|
|
248
|
+
* @experimental
|
|
243
249
|
*/
|
|
244
|
-
function
|
|
250
|
+
function experimental_skill(spec) {
|
|
245
251
|
return { _specType: "skill", ...spec };
|
|
246
252
|
}
|
|
247
253
|
/**
|
|
@@ -333,10 +339,10 @@ function railway(spec) {
|
|
|
333
339
|
*
|
|
334
340
|
* needs({ plan: "string", files: "string[]" })
|
|
335
341
|
*
|
|
336
|
-
* @
|
|
342
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
337
343
|
* public API (pre-1.0); may change without a major bump.
|
|
338
344
|
*/
|
|
339
|
-
function
|
|
345
|
+
function experimental_needs(shape) {
|
|
340
346
|
return shape;
|
|
341
347
|
}
|
|
342
348
|
/**
|
|
@@ -346,10 +352,10 @@ function needs(shape) {
|
|
|
346
352
|
*
|
|
347
353
|
* pipeStep(implementer, needs({ plan: "string", files: "string[]" }))
|
|
348
354
|
*
|
|
349
|
-
* @
|
|
355
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
350
356
|
* public API (pre-1.0); may change without a major bump.
|
|
351
357
|
*/
|
|
352
|
-
function
|
|
358
|
+
function experimental_pipeStep(a, needsContract = {}) {
|
|
353
359
|
return { _step: "typed-delegate", agent: a, needs: needsContract };
|
|
354
360
|
}
|
|
355
361
|
/**
|
|
@@ -357,11 +363,13 @@ function pipeStep(a, needsContract = {}) {
|
|
|
357
363
|
* its `needs` must be empty (`needs({})` or omitted). Returns a `Pipeline`
|
|
358
364
|
* carrying that step's `ok`/`err` forward.
|
|
359
365
|
*
|
|
360
|
-
* @
|
|
366
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
361
367
|
* public API (pre-1.0); may change without a major bump.
|
|
362
368
|
*/
|
|
363
|
-
function
|
|
364
|
-
const step = "_step" in first
|
|
369
|
+
function experimental_start(first) {
|
|
370
|
+
const step = "_step" in first
|
|
371
|
+
? first
|
|
372
|
+
: experimental_pipeStep(first, {});
|
|
365
373
|
const out = step.agent.output;
|
|
366
374
|
return {
|
|
367
375
|
_specType: "pipeline",
|
|
@@ -385,10 +393,10 @@ function start(first) {
|
|
|
385
393
|
* exporting a function called `then` becomes a thenable, so `await import()` of
|
|
386
394
|
* any barrel re-exporting it would invoke it — a footgun the rename avoids.
|
|
387
395
|
*
|
|
388
|
-
* @
|
|
396
|
+
* @experimental Experimental typed-composition surface — NOT part of the frozen
|
|
389
397
|
* public API (pre-1.0); may change without a major bump.
|
|
390
398
|
*/
|
|
391
|
-
function
|
|
399
|
+
function experimental_andThen(prior, next) {
|
|
392
400
|
const real = next;
|
|
393
401
|
const out = real.agent.output;
|
|
394
402
|
const rw = railway({
|
|
@@ -403,12 +411,12 @@ function andThen(prior, next) {
|
|
|
403
411
|
railway: rw,
|
|
404
412
|
};
|
|
405
413
|
}
|
|
406
|
-
function
|
|
414
|
+
function experimental_pipe(first, ...rest) {
|
|
407
415
|
// The overloads above enforce each handoff at the type level; the runtime body
|
|
408
416
|
// is the same left fold of start/andThen, untyped (the checks already happened).
|
|
409
|
-
let pipeline =
|
|
417
|
+
let pipeline = experimental_start(first);
|
|
410
418
|
for (const s of rest) {
|
|
411
|
-
pipeline =
|
|
419
|
+
pipeline = experimental_andThen(pipeline, s);
|
|
412
420
|
}
|
|
413
421
|
return pipeline;
|
|
414
422
|
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `vigiles/eval` — the **paid** surface: every runtime export here can call a
|
|
3
|
+
* model, and therefore can spend money.
|
|
4
|
+
*
|
|
5
|
+
* That single property is the whole reason this subpath exists. The package used
|
|
6
|
+
* to split its testing API by TEST TIER (`vigiles/unit`, `vigiles/integration`,
|
|
7
|
+
* `vigiles/e2e`, `vigiles/testing`); it now splits on COST. Everything free is on
|
|
8
|
+
* the package root, [`vigiles`](./test.ts). Everything that bills is here. The
|
|
9
|
+
* name rhymes with the `vigiles eval` CLI verb.
|
|
10
|
+
*
|
|
11
|
+
* ## Why every runtime export is also named `paid_`
|
|
12
|
+
*
|
|
13
|
+
* The import path warns ONCE, at the top of the file. The name warns EVERY time,
|
|
14
|
+
* at the call site. Reading `await judged(trace, "did it refuse?")` on line 140,
|
|
15
|
+
* the import line is long out of view — `await paid_judged(...)` still says what
|
|
16
|
+
* it costs. This is not a new idiom in this package: `vigiles/experimental`
|
|
17
|
+
* already pairs a quarantined subpath with an `experimental_` name prefix for
|
|
18
|
+
* exactly this reason. The same device, applied to a second axis.
|
|
19
|
+
*
|
|
20
|
+
* ⚠️ **The prefix slightly OVERSTATES the cost, and that is a deliberate trade
|
|
21
|
+
* rather than an oversight.** `paid_judged` takes an injectable judge:
|
|
22
|
+
* `paid_judged(rubric, { judge: myFn })` calls no model and spends nothing — only
|
|
23
|
+
* the DEFAULT judge bills. `paid_measureTriggerRate` and friends likewise accept
|
|
24
|
+
* an injected `evalDriver`. The strictly accurate prefix would be `metered_`, but
|
|
25
|
+
* it reads a beat slower, and a warning that is not absorbed at a glance is not a
|
|
26
|
+
* warning. Clarity wins; the imprecision is recorded here and on each symbol
|
|
27
|
+
* rather than left for a reader to discover.
|
|
28
|
+
*
|
|
29
|
+
* ## Why the split had to be cost, not tier
|
|
30
|
+
*
|
|
31
|
+
* The old `vigiles/unit` docstring promised "no `claude`, no model, no
|
|
32
|
+
* bubblewrap, no network" — and re-exported `judged`, whose default judge is a
|
|
33
|
+
* real model call (`check.ts` resolves `opts.judge ?? ((o) => runJudge(o))`). A
|
|
34
|
+
* test importing only from that barrel could still spend money, and the one line
|
|
35
|
+
* a reader would have relied on to know otherwise was the false one. `judged`
|
|
36
|
+
* moved HERE and became `paid_judged`, which makes that class of defect
|
|
37
|
+
* unrepresentable rather than apologised for: on this path, billing is the
|
|
38
|
+
* advertised property, in the subpath and in every name.
|
|
39
|
+
*
|
|
40
|
+
* ## What is NOT here, and why
|
|
41
|
+
*
|
|
42
|
+
* The ~two dozen helpers that read an eval RESULT — `assertImproves`,
|
|
43
|
+
* `assertNoRegression`, `assertReliable`, `assertSignificant`, `assertTriggerRate`,
|
|
44
|
+
* `reliable`, `improvement`, `significantlyBeats`, `compareArms`, `diffReports`,
|
|
45
|
+
* `diffToJUnit`, `formatBaselineDiff`, `parseBaselineFile`, `readBaseline`,
|
|
46
|
+
* `toBaselineFile`, `writeBaseline`, `cost`, `latency`, `tokens`, `inputTokens`,
|
|
47
|
+
* `outputTokens`, `cacheTokens`, `formatEvalReport`, `formatCheckReport`,
|
|
48
|
+
* `formatTriggerRateReport`, `checkReportToJUnit`, `assertRates` — spend nothing,
|
|
49
|
+
* so they live on the free root barrel, unprefixed, even though their subject
|
|
50
|
+
* matter is evals.
|
|
51
|
+
*
|
|
52
|
+
* That is the one seam the split cannot make clean: the free helpers are typed
|
|
53
|
+
* over report shapes DEFINED here. The coupling is real and irreducible, so both
|
|
54
|
+
* barrels re-export the same report types. **The types are NOT prefixed** — the
|
|
55
|
+
* prefix is a warning about calling something, and a type is never called;
|
|
56
|
+
* `paid_EvalReport` would be noise on a symbol that cannot bill. Importing
|
|
57
|
+
* `vigiles/eval` for a type alone should never be necessary; importing it for a
|
|
58
|
+
* FUNCTION means you accepted a bill.
|
|
59
|
+
*
|
|
60
|
+
* Harness-specific eval drivers are not here either — `codexEvalDriver` /
|
|
61
|
+
* `codexEvalRunner` / `codexEvalAgentRunner` live on `vigiles/codex`, and
|
|
62
|
+
* `measureSelectionMatrix` / `assertNoCollision` on `vigiles/claude-code`, because
|
|
63
|
+
* those surfaces are chosen by harness, not by cost.
|
|
64
|
+
*/
|
|
65
|
+
/**
|
|
66
|
+
* Run an A/B eval across arms with a real model. Spends money.
|
|
67
|
+
*
|
|
68
|
+
* ⚠️ The `paid_` prefix overstates slightly: pass your own `evalDriver` and no
|
|
69
|
+
* model of ours is called. The DEFAULT path bills.
|
|
70
|
+
*/
|
|
71
|
+
export { runEval as paid_runEval } from "./eval.js";
|
|
72
|
+
/**
|
|
73
|
+
* Score checks over N trials of one task with a real model. Spends money.
|
|
74
|
+
*
|
|
75
|
+
* ⚠️ `paid_` overstates slightly: with an injected `evalDriver` this drives
|
|
76
|
+
* whatever you supply. The DEFAULT path bills.
|
|
77
|
+
*/
|
|
78
|
+
export { measure as paid_measure } from "./eval.js";
|
|
79
|
+
/**
|
|
80
|
+
* Score the same checks per arm (a hook/skill/rule on vs off) with a real model.
|
|
81
|
+
* Spends money.
|
|
82
|
+
*
|
|
83
|
+
* ⚠️ `paid_` overstates slightly: an injected `evalDriver` replaces the billed
|
|
84
|
+
* path. The DEFAULT path bills.
|
|
85
|
+
*/
|
|
86
|
+
export { measureArms as paid_measureArms } from "./eval.js";
|
|
87
|
+
/**
|
|
88
|
+
* Measure whether a skill's description actually FIRES (recall + precision) by
|
|
89
|
+
* running real prompts against a real model. Spends money.
|
|
90
|
+
*
|
|
91
|
+
* ⚠️ `paid_` overstates slightly: an injected `evalDriver` replaces the billed
|
|
92
|
+
* path, and `stubSkillBodies` makes the default path much cheaper without making
|
|
93
|
+
* it free. The DEFAULT path bills.
|
|
94
|
+
*/
|
|
95
|
+
export { measureTriggerRate as paid_measureTriggerRate } from "./eval.js";
|
|
96
|
+
/**
|
|
97
|
+
* The model-graded judge (harness-agnostic — grades text against a rubric).
|
|
98
|
+
* Shells out to the `claude` CLI synchronously; needs model auth. Spends money.
|
|
99
|
+
*
|
|
100
|
+
* ⚠️ Unlike the others this one has no injection seam — it always calls a model,
|
|
101
|
+
* so here `paid_` is exact.
|
|
102
|
+
*/
|
|
103
|
+
export { judge as paid_judge } from "./judge.js";
|
|
104
|
+
/**
|
|
105
|
+
* The one member of the declarative check vocabulary that bills: its default
|
|
106
|
+
* judge is a real model call. Every other `Check` is deterministic and lives
|
|
107
|
+
* unprefixed on the free root barrel.
|
|
108
|
+
*
|
|
109
|
+
* ⚠️ The `paid_` prefix overstates slightly, and this is the symbol it overstates
|
|
110
|
+
* most: `paid_judged(rubric, { judge: myFn })` calls YOUR function and spends
|
|
111
|
+
* nothing. Only `paid_judged(rubric)` — the default — bills.
|
|
112
|
+
*/
|
|
113
|
+
export { judged as paid_judged } from "./check.js";
|
|
114
|
+
/**
|
|
115
|
+
* The Claude-Code eval driver: the transport `paid_runEval` and friends use by
|
|
116
|
+
* default. Naming it here is how you pass it explicitly; using it spends money.
|
|
117
|
+
*/
|
|
118
|
+
export { claudeEvalDriver as paid_claudeEvalDriver } from "./eval.js";
|
|
119
|
+
export type { Check, CheckResult, JudgeFn } from "./check.js";
|
|
120
|
+
export type { Trace } from "./harness-test.js";
|
|
121
|
+
export type { EvalArm, EvalDriver, EvalSpec, EvalReport, EvalUsage, MeasureSpec, ArmsMeasureSpec, ArmReport, ArmUsage, ArmsCheckReport, CheckRate, CheckReport, MetricStat, Metrics, ModelOutputParser, ParsedModelRun, PromptDiversityIssue, PromptTriggerStat, RunContext, RunOut, SelectionTrialResult, TriggerRateReport, TriggerRateSpec, AgentRunArgs, AgentRunner, } from "./eval.js";
|
|
122
|
+
//# sourceMappingURL=eval-surface.d.ts.map
|