vigiles 15.2.0 → 15.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-code/agent-result.d.ts +10 -1
- package/dist/adapters/claude-code/agent-result.js +14 -2
- package/dist/eval.js +17 -1
- package/dist/experimental-emit.d.ts +163 -0
- package/dist/experimental-emit.js +243 -0
- package/dist/experimental.d.ts +9 -2
- package/dist/experimental.js +13 -3
- package/package.json +1 -1
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* third `malformed` track for a worker that didn't honor its contract (no block,
|
|
18
18
|
* bad JSON, or a shape that doesn't match the declared schema).
|
|
19
19
|
*/
|
|
20
|
-
import type { OutputContract } from "../../core/spec.js";
|
|
20
|
+
import type { OutputContract, OutputFieldType } from "../../core/spec.js";
|
|
21
21
|
/** The outcome of parsing a worker's result block. */
|
|
22
22
|
export type ParsedAgentResult<S = Record<string, unknown>, E = Record<string, unknown>> = {
|
|
23
23
|
readonly kind: "ok";
|
|
@@ -29,6 +29,15 @@ export type ParsedAgentResult<S = Record<string, unknown>, E = Record<string, un
|
|
|
29
29
|
readonly kind: "malformed";
|
|
30
30
|
readonly reason: string;
|
|
31
31
|
};
|
|
32
|
+
/**
|
|
33
|
+
* Validate a parsed object against a contract track; null when it conforms.
|
|
34
|
+
*
|
|
35
|
+
* Exported because the EXPERIMENTAL emit channel (`src/experimental-emit.ts`)
|
|
36
|
+
* validates the SAME `OutputContract` on a different delivery. One contract, two
|
|
37
|
+
* deliveries, one validator — a second copy would drift, and the two rails
|
|
38
|
+
* disagreeing about what satisfies a contract is the worst outcome available.
|
|
39
|
+
*/
|
|
40
|
+
export declare function shapeError(obj: Record<string, unknown>, shape: Readonly<Record<string, OutputFieldType>>): string | null;
|
|
32
41
|
/**
|
|
33
42
|
* Parse the last `vigiles:ok` / `vigiles:err` block from a worker's output.
|
|
34
43
|
*
|
|
@@ -19,10 +19,15 @@
|
|
|
19
19
|
* bad JSON, or a shape that doesn't match the declared schema).
|
|
20
20
|
*/
|
|
21
21
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.shapeError = shapeError;
|
|
22
23
|
exports.parseAgentResult = parseAgentResult;
|
|
23
24
|
// Capture every vigiles:ok / vigiles:err fenced block; the LAST one is the
|
|
24
25
|
// worker's final answer (earlier ones may be illustrative in its reasoning).
|
|
25
|
-
|
|
26
|
+
// The CLOSING fence must own its line: a ``` inside a JSON string value sits
|
|
27
|
+
// mid-line, so it no longer terminates the block (measured 2026-08-13 — a
|
|
28
|
+
// contract field carrying a code snippet made every such answer `malformed`
|
|
29
|
+
// even though the model's JSON was valid).
|
|
30
|
+
const BLOCK_RE = /^[ \t]*```vigiles:(ok|err)[ \t]*\r?\n([\s\S]*?)\r?\n^[ \t]*```[ \t]*$/gm;
|
|
26
31
|
/** Does a runtime value match a declared field type? */
|
|
27
32
|
function fieldMatches(value, type) {
|
|
28
33
|
switch (type) {
|
|
@@ -36,7 +41,14 @@ function fieldMatches(value, type) {
|
|
|
36
41
|
return Array.isArray(value) && value.every((v) => typeof v === "string");
|
|
37
42
|
}
|
|
38
43
|
}
|
|
39
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Validate a parsed object against a contract track; null when it conforms.
|
|
46
|
+
*
|
|
47
|
+
* Exported because the EXPERIMENTAL emit channel (`src/experimental-emit.ts`)
|
|
48
|
+
* validates the SAME `OutputContract` on a different delivery. One contract, two
|
|
49
|
+
* deliveries, one validator — a second copy would drift, and the two rails
|
|
50
|
+
* disagreeing about what satisfies a contract is the worst outcome available.
|
|
51
|
+
*/
|
|
40
52
|
function shapeError(obj, shape) {
|
|
41
53
|
for (const [field, type] of Object.entries(shape)) {
|
|
42
54
|
if (!(field in obj))
|
package/dist/eval.js
CHANGED
|
@@ -981,7 +981,23 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
|
|
|
981
981
|
}
|
|
982
982
|
// A signal in the captured streams that the model call was rate-limited /
|
|
983
983
|
// overloaded — worth a backoff + retry rather than counting as a real sample.
|
|
984
|
-
|
|
984
|
+
//
|
|
985
|
+
// 🔴 The separator is `[ -]?`, NOT `.?`, and that is load-bearing. Claude Code's
|
|
986
|
+
// stream-json emits an INFORMATIONAL `{"type":"rate_limit_event","rate_limit_info":
|
|
987
|
+
// {"status":"allowed",…}}` line on EVERY run (captured verbatim in
|
|
988
|
+
// examples/experimental-emit/records/rate-limit-event.json). `rate.?limit` matched
|
|
989
|
+
// `rate_limit_event`, because `.` matches `_` — so every trial looked rate-limited,
|
|
990
|
+
// every trial was retried `retries + 1` = 4 times, and only the LAST attempt's cost
|
|
991
|
+
// reached `maxCostUsd`. Measured 2026-08-13: 2 trials → 8 model runs, a budget cap
|
|
992
|
+
// of $0.60 crossed at roughly $3, and the run reported one trial. `[ -]?` cannot
|
|
993
|
+
// match `_`, so the telemetry line is no longer a match; a REAL limit still is,
|
|
994
|
+
// because the API reports it as `rate_limit_error` / 429 / "overloaded".
|
|
995
|
+
//
|
|
996
|
+
// Known boundary, stated rather than hidden: a `rate_limit_event` whose `status`
|
|
997
|
+
// is a rejection is no longer a match either. It never was one on its merits — the
|
|
998
|
+
// old pattern fired on the event's NAME regardless of status, so status-aware
|
|
999
|
+
// detection has never existed here.
|
|
1000
|
+
const RATE_LIMIT_RE = /rate[ -]?limit(?:ed)?\b|rate_limit_error|\b429\b|overloaded|too many requests/i;
|
|
985
1001
|
/** Whether a run's captured output looks like a rate-limit / overload. Pure. */
|
|
986
1002
|
function isRateLimited(out) {
|
|
987
1003
|
return RATE_LIMIT_RE.test(`${out.stderr ?? ""}\n${out.stdout}`);
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ⚠️ EXPERIMENTAL — the EMIT delivery for a typed result: the skill CALLS a tool
|
|
3
|
+
* carrying its outcome instead of ENDING its turn with a fenced block.
|
|
4
|
+
*
|
|
5
|
+
* ## Why this exists
|
|
6
|
+
*
|
|
7
|
+
* A typed `output` (an `OutputContract`) is valid only on a forked skill: compile
|
|
8
|
+
* hard-errors `output-without-fork` on the other 31, because an inline skill is
|
|
9
|
+
* spliced into the conversation, has no call→return boundary, and therefore has
|
|
10
|
+
* no return value to type (`research/spec-syntax-and-railway-scope.md`).
|
|
11
|
+
*
|
|
12
|
+
* A TOOL CALL needs no return boundary. The skill does not return the structure —
|
|
13
|
+
* it EMITS it, mid-conversation, and the call lands in `Trace.toolCalls`. So the
|
|
14
|
+
* objection that grounds the exclusion does not apply to this delivery. Same
|
|
15
|
+
* `OutputContract`; a different way of getting it out.
|
|
16
|
+
*
|
|
17
|
+
* ## What is UNPROVEN (why the `experimental_` prefix is on every runtime export)
|
|
18
|
+
*
|
|
19
|
+
* 1. **N=8, one skill, one model.** Measured 2026-08-13 against `paper-status`
|
|
20
|
+
* (unforked; `allowed-tools: Bash, Read, Grep, Glob`) on sonnet: 8 runs, 8
|
|
21
|
+
* emits, all on the `ok` track, all parsing against the contract, none
|
|
22
|
+
* repeated. That answers "does it land at all" and nothing about the rate —
|
|
23
|
+
* 8 of 8 bounds the true failure rate at about 31%, which is not evidence of
|
|
24
|
+
* reliability. Raw arguments + the free re-scorer:
|
|
25
|
+
* `examples/experimental-emit/`.
|
|
26
|
+
* 2. **Nobody depends on it.** Neither `compileSkill` nor `compileAgent` emits
|
|
27
|
+
* this instruction; the caller pastes `.instruction` into a skill body and
|
|
28
|
+
* serves `.tool` from their own MCP server by hand. There is no compile-time
|
|
29
|
+
* path, so no skill in any corpus is typed by it yet.
|
|
30
|
+
* 3. **The transport is not part of the contract.** MCP is how the tool reached
|
|
31
|
+
* the model in the measurement. Whether a plugin can hand the model a tool
|
|
32
|
+
* WITHOUT a separate server process is untested, and the answer changes what
|
|
33
|
+
* this surface should look like.
|
|
34
|
+
* 4. **The runtime does not enforce `required`.** Measured: Claude Code accepted a
|
|
35
|
+
* call omitting three declared-required fields (raw proof in `mine`,
|
|
36
|
+
* `vigiles/repro/output-contract-2026-08-13/mcp-arm/schema-probe-emitted.jsonl`).
|
|
37
|
+
* `inputSchema` is DESCRIPTION for the model, not a runtime gate — which is
|
|
38
|
+
* exactly why `experimental_parseEmitted` re-validates on the receiving side
|
|
39
|
+
* and why "the API validates it for you" must not be claimed.
|
|
40
|
+
*
|
|
41
|
+
* ## What would have to be true to drop the prefix
|
|
42
|
+
*
|
|
43
|
+
* - A rate, not an existence proof: ≥30 trials across ≥2 unforked skills and ≥2
|
|
44
|
+
* models, with the emit-landing rate reported and its failure modes named.
|
|
45
|
+
* - One consumer inside vigiles that compiles the instruction from the spec, so
|
|
46
|
+
* the tool name and the shape cannot drift apart by hand.
|
|
47
|
+
* - A measured answer to (3) — plugin-served tool vs external MCP server — since
|
|
48
|
+
* an in-process tool would remove the standing-up cost this surface assumes.
|
|
49
|
+
*
|
|
50
|
+
* Until then: not covered by the stability guarantee, may change or be removed
|
|
51
|
+
* without a major bump. See `docs/../STABILITY.md` and `src/experimental.ts`.
|
|
52
|
+
*
|
|
53
|
+
* @experimental
|
|
54
|
+
* @module
|
|
55
|
+
*/
|
|
56
|
+
import type { OutputContract } from "./core/spec.js";
|
|
57
|
+
import type { ToolCall } from "./core/harness-driver.js";
|
|
58
|
+
import { type ParsedAgentResult } from "./adapters/claude-code/agent-result.js";
|
|
59
|
+
/** A JSON-Schema fragment for one declared field. */
|
|
60
|
+
export type EmitFieldSchema = {
|
|
61
|
+
readonly type: "string";
|
|
62
|
+
} | {
|
|
63
|
+
readonly type: "number";
|
|
64
|
+
} | {
|
|
65
|
+
readonly type: "boolean";
|
|
66
|
+
} | {
|
|
67
|
+
readonly type: "array";
|
|
68
|
+
readonly items: {
|
|
69
|
+
readonly type: "string";
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
/** The `track` discriminator's schema — the only enum this surface emits. */
|
|
73
|
+
export interface EmitTrackSchema {
|
|
74
|
+
readonly type: "string";
|
|
75
|
+
readonly enum: readonly ["ok", "err"];
|
|
76
|
+
}
|
|
77
|
+
/** Anything that can sit under `properties`: a field, the discriminator, a track. */
|
|
78
|
+
export type EmitPropertySchema = EmitFieldSchema | EmitTrackSchema | EmitObjectSchema;
|
|
79
|
+
/** A JSON-Schema object node — one track's payload, or the whole argument. */
|
|
80
|
+
export interface EmitObjectSchema {
|
|
81
|
+
readonly type: "object";
|
|
82
|
+
readonly properties: Readonly<Record<string, EmitPropertySchema>>;
|
|
83
|
+
readonly required: readonly string[];
|
|
84
|
+
readonly additionalProperties: false;
|
|
85
|
+
}
|
|
86
|
+
/** An MCP tool definition, in the shape a `tools/list` response carries. */
|
|
87
|
+
export interface EmitToolDefinition {
|
|
88
|
+
readonly name: string;
|
|
89
|
+
readonly description: string;
|
|
90
|
+
readonly inputSchema: EmitObjectSchema;
|
|
91
|
+
}
|
|
92
|
+
/** What `experimental_emitTool` hands back: the tool, and the prose that asks for it. */
|
|
93
|
+
export interface ExperimentalEmitTool {
|
|
94
|
+
/** Serve this from your MCP server's `tools/list`. */
|
|
95
|
+
readonly tool: EmitToolDefinition;
|
|
96
|
+
/**
|
|
97
|
+
* Markdown fragment for the skill body. The SAME contract rendered for the
|
|
98
|
+
* model — kept next to the schema so the two cannot drift when hand-wired.
|
|
99
|
+
*/
|
|
100
|
+
readonly instruction: string;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* ⚠️ EXPERIMENTAL. Derive an emit TOOL from an `OutputContract` — the same
|
|
104
|
+
* contract the fork rail renders as a `vigiles:ok` / `vigiles:err` fenced block.
|
|
105
|
+
*
|
|
106
|
+
* const emit = experimental_emitTool(contract);
|
|
107
|
+
* // emit.tool → serve from your MCP server
|
|
108
|
+
* // emit.instruction → paste into the (unforked) skill's body
|
|
109
|
+
*
|
|
110
|
+
* The two tracks are NESTED (`{ track, ok? , err? }`), not flattened into one bag
|
|
111
|
+
* of fields. That is deliberate: a flat union cannot say which fields are required
|
|
112
|
+
* on which track, so "success fields mixed with error fields" would be a
|
|
113
|
+
* well-formed call. Nested, it is not expressible.
|
|
114
|
+
*
|
|
115
|
+
* 🔴 `required` in the returned schema is DESCRIPTION, not enforcement — measured,
|
|
116
|
+
* see the module header (4). Validate what arrives with
|
|
117
|
+
* `experimental_parseEmitted`.
|
|
118
|
+
*
|
|
119
|
+
* @experimental
|
|
120
|
+
*/
|
|
121
|
+
export declare function experimental_emitTool(contract: OutputContract, options?: {
|
|
122
|
+
readonly name?: string;
|
|
123
|
+
}): ExperimentalEmitTool;
|
|
124
|
+
/**
|
|
125
|
+
* ⚠️ EXPERIMENTAL. Read the emitted result out of a run's tool calls, validated
|
|
126
|
+
* against the contract. Pure — returns the same `ParsedAgentResult` vocabulary the
|
|
127
|
+
* fenced rail's `parseAgentResult` returns, so an eval `measure` can use it as a
|
|
128
|
+
* metric and the assertion below can wrap it, without one dual-purpose function.
|
|
129
|
+
*
|
|
130
|
+
* Accepts `Trace["toolCalls"]`, `SubagentTrace["toolCalls"]` or an eval
|
|
131
|
+
* `ctx.toolCalls`. Names are matched bare (`emit_result`) or MCP-prefixed
|
|
132
|
+
* (`mcp__<server>__emit_result`).
|
|
133
|
+
*
|
|
134
|
+
* Differs from the fenced rail in ONE deliberate way: **more than one call is
|
|
135
|
+
* `malformed`, not last-one-wins.** The fenced parser takes the LAST block because
|
|
136
|
+
* an earlier block may be illustrative reasoning; a tool call is an action, never
|
|
137
|
+
* illustrative, so "exactly once" is checkable here and is not checkable there.
|
|
138
|
+
*
|
|
139
|
+
* @experimental
|
|
140
|
+
*/
|
|
141
|
+
export declare function experimental_parseEmitted(toolCalls: readonly ToolCall[], contract: OutputContract, options?: {
|
|
142
|
+
readonly name?: string;
|
|
143
|
+
}): ParsedAgentResult;
|
|
144
|
+
/**
|
|
145
|
+
* ⚠️ EXPERIMENTAL. Assert the run emitted a SUCCESS result, and return its value —
|
|
146
|
+
* the emit-channel counterpart of `assertAgentOk`, for a skill that has no return
|
|
147
|
+
* value to assert on.
|
|
148
|
+
*
|
|
149
|
+
* Throws on a missing emit, a repeated emit, an error track, or a payload that
|
|
150
|
+
* does not match the contract. The failure message names every tool the run DID
|
|
151
|
+
* call, because "the skill never emitted" and "the skill emitted the wrong shape"
|
|
152
|
+
* are different bugs and the tool list separates them at a glance.
|
|
153
|
+
*
|
|
154
|
+
* The error track is reachable through `experimental_parseEmitted`; a matching
|
|
155
|
+
* `…EmittedErr` is deliberately NOT shipped while the surface is this young —
|
|
156
|
+
* three exports is the whole prototype.
|
|
157
|
+
*
|
|
158
|
+
* @experimental
|
|
159
|
+
*/
|
|
160
|
+
export declare function experimental_assertEmittedOk(toolCalls: readonly ToolCall[], contract: OutputContract, options?: {
|
|
161
|
+
readonly name?: string;
|
|
162
|
+
}): Record<string, unknown>;
|
|
163
|
+
//# sourceMappingURL=experimental-emit.d.ts.map
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ⚠️ EXPERIMENTAL — the EMIT delivery for a typed result: the skill CALLS a tool
|
|
4
|
+
* carrying its outcome instead of ENDING its turn with a fenced block.
|
|
5
|
+
*
|
|
6
|
+
* ## Why this exists
|
|
7
|
+
*
|
|
8
|
+
* A typed `output` (an `OutputContract`) is valid only on a forked skill: compile
|
|
9
|
+
* hard-errors `output-without-fork` on the other 31, because an inline skill is
|
|
10
|
+
* spliced into the conversation, has no call→return boundary, and therefore has
|
|
11
|
+
* no return value to type (`research/spec-syntax-and-railway-scope.md`).
|
|
12
|
+
*
|
|
13
|
+
* A TOOL CALL needs no return boundary. The skill does not return the structure —
|
|
14
|
+
* it EMITS it, mid-conversation, and the call lands in `Trace.toolCalls`. So the
|
|
15
|
+
* objection that grounds the exclusion does not apply to this delivery. Same
|
|
16
|
+
* `OutputContract`; a different way of getting it out.
|
|
17
|
+
*
|
|
18
|
+
* ## What is UNPROVEN (why the `experimental_` prefix is on every runtime export)
|
|
19
|
+
*
|
|
20
|
+
* 1. **N=8, one skill, one model.** Measured 2026-08-13 against `paper-status`
|
|
21
|
+
* (unforked; `allowed-tools: Bash, Read, Grep, Glob`) on sonnet: 8 runs, 8
|
|
22
|
+
* emits, all on the `ok` track, all parsing against the contract, none
|
|
23
|
+
* repeated. That answers "does it land at all" and nothing about the rate —
|
|
24
|
+
* 8 of 8 bounds the true failure rate at about 31%, which is not evidence of
|
|
25
|
+
* reliability. Raw arguments + the free re-scorer:
|
|
26
|
+
* `examples/experimental-emit/`.
|
|
27
|
+
* 2. **Nobody depends on it.** Neither `compileSkill` nor `compileAgent` emits
|
|
28
|
+
* this instruction; the caller pastes `.instruction` into a skill body and
|
|
29
|
+
* serves `.tool` from their own MCP server by hand. There is no compile-time
|
|
30
|
+
* path, so no skill in any corpus is typed by it yet.
|
|
31
|
+
* 3. **The transport is not part of the contract.** MCP is how the tool reached
|
|
32
|
+
* the model in the measurement. Whether a plugin can hand the model a tool
|
|
33
|
+
* WITHOUT a separate server process is untested, and the answer changes what
|
|
34
|
+
* this surface should look like.
|
|
35
|
+
* 4. **The runtime does not enforce `required`.** Measured: Claude Code accepted a
|
|
36
|
+
* call omitting three declared-required fields (raw proof in `mine`,
|
|
37
|
+
* `vigiles/repro/output-contract-2026-08-13/mcp-arm/schema-probe-emitted.jsonl`).
|
|
38
|
+
* `inputSchema` is DESCRIPTION for the model, not a runtime gate — which is
|
|
39
|
+
* exactly why `experimental_parseEmitted` re-validates on the receiving side
|
|
40
|
+
* and why "the API validates it for you" must not be claimed.
|
|
41
|
+
*
|
|
42
|
+
* ## What would have to be true to drop the prefix
|
|
43
|
+
*
|
|
44
|
+
* - A rate, not an existence proof: ≥30 trials across ≥2 unforked skills and ≥2
|
|
45
|
+
* models, with the emit-landing rate reported and its failure modes named.
|
|
46
|
+
* - One consumer inside vigiles that compiles the instruction from the spec, so
|
|
47
|
+
* the tool name and the shape cannot drift apart by hand.
|
|
48
|
+
* - A measured answer to (3) — plugin-served tool vs external MCP server — since
|
|
49
|
+
* an in-process tool would remove the standing-up cost this surface assumes.
|
|
50
|
+
*
|
|
51
|
+
* Until then: not covered by the stability guarantee, may change or be removed
|
|
52
|
+
* without a major bump. See `docs/../STABILITY.md` and `src/experimental.ts`.
|
|
53
|
+
*
|
|
54
|
+
* @experimental
|
|
55
|
+
* @module
|
|
56
|
+
*/
|
|
57
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
58
|
+
exports.experimental_emitTool = experimental_emitTool;
|
|
59
|
+
exports.experimental_parseEmitted = experimental_parseEmitted;
|
|
60
|
+
exports.experimental_assertEmittedOk = experimental_assertEmittedOk;
|
|
61
|
+
const agent_result_js_1 = require("./adapters/claude-code/agent-result.js");
|
|
62
|
+
/** The default tool name, when `options.name` is not given. */
|
|
63
|
+
const DEFAULT_EMIT_TOOL = "emit_result";
|
|
64
|
+
function fieldSchema(type) {
|
|
65
|
+
switch (type) {
|
|
66
|
+
case "string":
|
|
67
|
+
return { type: "string" };
|
|
68
|
+
case "number":
|
|
69
|
+
return { type: "number" };
|
|
70
|
+
case "boolean":
|
|
71
|
+
return { type: "boolean" };
|
|
72
|
+
case "string[]":
|
|
73
|
+
return { type: "array", items: { type: "string" } };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function trackSchema(shape) {
|
|
77
|
+
const properties = {};
|
|
78
|
+
for (const [field, type] of Object.entries(shape)) {
|
|
79
|
+
properties[field] = fieldSchema(type);
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
type: "object",
|
|
83
|
+
properties,
|
|
84
|
+
required: Object.keys(shape),
|
|
85
|
+
additionalProperties: false,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
/** Render a declared shape the way the fenced contract renders it, for the prose half. */
|
|
89
|
+
function renderShape(shape) {
|
|
90
|
+
const fields = Object.entries(shape)
|
|
91
|
+
.map(([k, t]) => `"${k}": ${t}`)
|
|
92
|
+
.join(", ");
|
|
93
|
+
return fields ? `{ ${fields} }` : "{}";
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* ⚠️ EXPERIMENTAL. Derive an emit TOOL from an `OutputContract` — the same
|
|
97
|
+
* contract the fork rail renders as a `vigiles:ok` / `vigiles:err` fenced block.
|
|
98
|
+
*
|
|
99
|
+
* const emit = experimental_emitTool(contract);
|
|
100
|
+
* // emit.tool → serve from your MCP server
|
|
101
|
+
* // emit.instruction → paste into the (unforked) skill's body
|
|
102
|
+
*
|
|
103
|
+
* The two tracks are NESTED (`{ track, ok? , err? }`), not flattened into one bag
|
|
104
|
+
* of fields. That is deliberate: a flat union cannot say which fields are required
|
|
105
|
+
* on which track, so "success fields mixed with error fields" would be a
|
|
106
|
+
* well-formed call. Nested, it is not expressible.
|
|
107
|
+
*
|
|
108
|
+
* 🔴 `required` in the returned schema is DESCRIPTION, not enforcement — measured,
|
|
109
|
+
* see the module header (4). Validate what arrives with
|
|
110
|
+
* `experimental_parseEmitted`.
|
|
111
|
+
*
|
|
112
|
+
* @experimental
|
|
113
|
+
*/
|
|
114
|
+
function experimental_emitTool(contract, options = {}) {
|
|
115
|
+
const name = options.name ?? DEFAULT_EMIT_TOOL;
|
|
116
|
+
const tool = {
|
|
117
|
+
name,
|
|
118
|
+
description: "Emit this task's structured result. Call this exactly once. Set " +
|
|
119
|
+
'`track` to "ok" and fill `ok` on success, or "err" and fill `err` on ' +
|
|
120
|
+
"failure. Do not call it twice and do not fill both tracks.",
|
|
121
|
+
inputSchema: {
|
|
122
|
+
type: "object",
|
|
123
|
+
properties: {
|
|
124
|
+
track: { type: "string", enum: ["ok", "err"] },
|
|
125
|
+
ok: trackSchema(contract.ok),
|
|
126
|
+
err: trackSchema(contract.err),
|
|
127
|
+
},
|
|
128
|
+
required: ["track"],
|
|
129
|
+
additionalProperties: false,
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
const instruction = [
|
|
133
|
+
"## Output contract",
|
|
134
|
+
"",
|
|
135
|
+
`Emit your result by calling the \`${name}\` tool exactly once, at the point`,
|
|
136
|
+
"you have the answer. Do not print it as a code block; the call IS the result.",
|
|
137
|
+
"",
|
|
138
|
+
`On success: \`track: "ok"\`, with \`ok\` =`,
|
|
139
|
+
"",
|
|
140
|
+
"```json",
|
|
141
|
+
renderShape(contract.ok),
|
|
142
|
+
"```",
|
|
143
|
+
"",
|
|
144
|
+
`On failure: \`track: "err"\`, with \`err\` =`,
|
|
145
|
+
"",
|
|
146
|
+
"```json",
|
|
147
|
+
renderShape(contract.err),
|
|
148
|
+
"```",
|
|
149
|
+
].join("\n");
|
|
150
|
+
return { tool, instruction };
|
|
151
|
+
}
|
|
152
|
+
/** Does this observed tool name refer to `name` (bare, or MCP-prefixed)? */
|
|
153
|
+
function isEmitCall(observed, name) {
|
|
154
|
+
return observed === name || observed.endsWith(`__${name}`);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* ⚠️ EXPERIMENTAL. Read the emitted result out of a run's tool calls, validated
|
|
158
|
+
* against the contract. Pure — returns the same `ParsedAgentResult` vocabulary the
|
|
159
|
+
* fenced rail's `parseAgentResult` returns, so an eval `measure` can use it as a
|
|
160
|
+
* metric and the assertion below can wrap it, without one dual-purpose function.
|
|
161
|
+
*
|
|
162
|
+
* Accepts `Trace["toolCalls"]`, `SubagentTrace["toolCalls"]` or an eval
|
|
163
|
+
* `ctx.toolCalls`. Names are matched bare (`emit_result`) or MCP-prefixed
|
|
164
|
+
* (`mcp__<server>__emit_result`).
|
|
165
|
+
*
|
|
166
|
+
* Differs from the fenced rail in ONE deliberate way: **more than one call is
|
|
167
|
+
* `malformed`, not last-one-wins.** The fenced parser takes the LAST block because
|
|
168
|
+
* an earlier block may be illustrative reasoning; a tool call is an action, never
|
|
169
|
+
* illustrative, so "exactly once" is checkable here and is not checkable there.
|
|
170
|
+
*
|
|
171
|
+
* @experimental
|
|
172
|
+
*/
|
|
173
|
+
function experimental_parseEmitted(toolCalls, contract, options = {}) {
|
|
174
|
+
const name = options.name ?? DEFAULT_EMIT_TOOL;
|
|
175
|
+
const calls = toolCalls.filter((c) => isEmitCall(c.name, name));
|
|
176
|
+
if (calls.length === 0) {
|
|
177
|
+
return { kind: "malformed", reason: `no \`${name}\` tool call in the run` };
|
|
178
|
+
}
|
|
179
|
+
if (calls.length > 1) {
|
|
180
|
+
return {
|
|
181
|
+
kind: "malformed",
|
|
182
|
+
reason: `\`${name}\` was called ${String(calls.length)} times; the contract is exactly once`,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
const input = calls[0].input;
|
|
186
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
187
|
+
return {
|
|
188
|
+
kind: "malformed",
|
|
189
|
+
reason: `\`${name}\` call carried no object argument`,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const args = input;
|
|
193
|
+
const track = args.track;
|
|
194
|
+
if (track !== "ok" && track !== "err") {
|
|
195
|
+
return {
|
|
196
|
+
kind: "malformed",
|
|
197
|
+
reason: `\`${name}\` call has no \`track\` of "ok" or "err"`,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
const payload = args[track];
|
|
201
|
+
if (typeof payload !== "object" ||
|
|
202
|
+
payload === null ||
|
|
203
|
+
Array.isArray(payload)) {
|
|
204
|
+
return {
|
|
205
|
+
kind: "malformed",
|
|
206
|
+
reason: `\`${name}\` call declared track "${track}" but carried no \`${track}\` object`,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
const obj = payload;
|
|
210
|
+
const bad = (0, agent_result_js_1.shapeError)(obj, track === "ok" ? contract.ok : contract.err);
|
|
211
|
+
if (bad)
|
|
212
|
+
return { kind: "malformed", reason: `${track} payload: ${bad}` };
|
|
213
|
+
return track === "ok"
|
|
214
|
+
? { kind: "ok", value: obj }
|
|
215
|
+
: { kind: "err", error: obj };
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* ⚠️ EXPERIMENTAL. Assert the run emitted a SUCCESS result, and return its value —
|
|
219
|
+
* the emit-channel counterpart of `assertAgentOk`, for a skill that has no return
|
|
220
|
+
* value to assert on.
|
|
221
|
+
*
|
|
222
|
+
* Throws on a missing emit, a repeated emit, an error track, or a payload that
|
|
223
|
+
* does not match the contract. The failure message names every tool the run DID
|
|
224
|
+
* call, because "the skill never emitted" and "the skill emitted the wrong shape"
|
|
225
|
+
* are different bugs and the tool list separates them at a glance.
|
|
226
|
+
*
|
|
227
|
+
* The error track is reachable through `experimental_parseEmitted`; a matching
|
|
228
|
+
* `…EmittedErr` is deliberately NOT shipped while the surface is this young —
|
|
229
|
+
* three exports is the whole prototype.
|
|
230
|
+
*
|
|
231
|
+
* @experimental
|
|
232
|
+
*/
|
|
233
|
+
function experimental_assertEmittedOk(toolCalls, contract, options = {}) {
|
|
234
|
+
const r = experimental_parseEmitted(toolCalls, contract, options);
|
|
235
|
+
if (r.kind === "ok")
|
|
236
|
+
return r.value;
|
|
237
|
+
const why = r.kind === "err"
|
|
238
|
+
? `it emitted an error result: ${JSON.stringify(r.error)}`
|
|
239
|
+
: r.reason;
|
|
240
|
+
const observed = toolCalls.map((c) => c.name).join(", ") || "none";
|
|
241
|
+
throw new Error(`expected an emitted success result, but ${why} (tools called: ${observed})`);
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=experimental-emit.js.map
|
package/dist/experimental.d.ts
CHANGED
|
@@ -10,8 +10,14 @@
|
|
|
10
10
|
* NOT covered by the stability guarantee (STABILITY.md): the shape may change or
|
|
11
11
|
* be removed WITHOUT a major-version bump. Do not depend on it in production.
|
|
12
12
|
*
|
|
13
|
-
* Current contents
|
|
14
|
-
*
|
|
13
|
+
* Current contents:
|
|
14
|
+
* - the R3 disposable-service tier (real side-effect testing; see
|
|
15
|
+
* docs/measuring-skills.md § Experimental and src/services.ts);
|
|
16
|
+
* - the EMIT delivery for a typed result (`src/experimental-emit.ts`) — a skill
|
|
17
|
+
* that CALLS a tool with its outcome instead of ending its turn with a fenced
|
|
18
|
+
* block, which is how an UNFORKED skill can carry an `OutputContract` at all.
|
|
19
|
+
* Read that module's header before using it: it lists, by number, what is
|
|
20
|
+
* unproven and what would have to be true to drop the prefix.
|
|
15
21
|
*
|
|
16
22
|
* ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
|
|
17
23
|
* the ONLY isolation vigiles provides — it does not confine the skill's filesystem
|
|
@@ -24,4 +30,5 @@
|
|
|
24
30
|
*/
|
|
25
31
|
export { experimental_startServices, experimental_withServices, type ServiceSpec, type ServiceReady, type ServiceReset, type ServiceHandle, type ServiceSession, type ContainerRuntime, } from "./services.js";
|
|
26
32
|
export { experimental_dockerRuntime, makeDockerRuntime, type DockerExec, type NetProbe, } from "./services-docker.js";
|
|
33
|
+
export { experimental_emitTool, experimental_parseEmitted, experimental_assertEmittedOk, type EmitFieldSchema, type EmitObjectSchema, type EmitPropertySchema, type EmitTrackSchema, type EmitToolDefinition, type ExperimentalEmitTool, } from "./experimental-emit.js";
|
|
27
34
|
//# sourceMappingURL=experimental.d.ts.map
|
package/dist/experimental.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = void 0;
|
|
3
|
+
exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = void 0;
|
|
4
4
|
/**
|
|
5
5
|
* `vigiles/experimental` — ⚠️ EXPERIMENTAL, UNSTABLE public surface.
|
|
6
6
|
*
|
|
@@ -13,8 +13,14 @@ exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experim
|
|
|
13
13
|
* NOT covered by the stability guarantee (STABILITY.md): the shape may change or
|
|
14
14
|
* be removed WITHOUT a major-version bump. Do not depend on it in production.
|
|
15
15
|
*
|
|
16
|
-
* Current contents
|
|
17
|
-
*
|
|
16
|
+
* Current contents:
|
|
17
|
+
* - the R3 disposable-service tier (real side-effect testing; see
|
|
18
|
+
* docs/measuring-skills.md § Experimental and src/services.ts);
|
|
19
|
+
* - the EMIT delivery for a typed result (`src/experimental-emit.ts`) — a skill
|
|
20
|
+
* that CALLS a tool with its outcome instead of ending its turn with a fenced
|
|
21
|
+
* block, which is how an UNFORKED skill can carry an `OutputContract` at all.
|
|
22
|
+
* Read that module's header before using it: it lists, by number, what is
|
|
23
|
+
* unproven and what would have to be true to drop the prefix.
|
|
18
24
|
*
|
|
19
25
|
* ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
|
|
20
26
|
* the ONLY isolation vigiles provides — it does not confine the skill's filesystem
|
|
@@ -31,4 +37,8 @@ Object.defineProperty(exports, "experimental_withServices", { enumerable: true,
|
|
|
31
37
|
var services_docker_js_1 = require("./services-docker.js");
|
|
32
38
|
Object.defineProperty(exports, "experimental_dockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.experimental_dockerRuntime; } });
|
|
33
39
|
Object.defineProperty(exports, "makeDockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.makeDockerRuntime; } });
|
|
40
|
+
var experimental_emit_js_1 = require("./experimental-emit.js");
|
|
41
|
+
Object.defineProperty(exports, "experimental_emitTool", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_emitTool; } });
|
|
42
|
+
Object.defineProperty(exports, "experimental_parseEmitted", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_parseEmitted; } });
|
|
43
|
+
Object.defineProperty(exports, "experimental_assertEmittedOk", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_assertEmittedOk; } });
|
|
34
44
|
//# sourceMappingURL=experimental.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vigiles",
|
|
3
|
-
"version": "15.2.
|
|
3
|
+
"version": "15.2.1",
|
|
4
4
|
"description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"claude-code",
|