vigiles 15.2.0 → 15.3.0

package/dist/adapters/claude-code/agent-result.d.ts

@@ -17,7 +17,7 @@
  * third `malformed` track for a worker that didn't honor its contract (no block,
  * bad JSON, or a shape that doesn't match the declared schema).
  */
-import type { OutputContract } from "../../core/spec.js";
+import type { OutputContract, OutputFieldType } from "../../core/spec.js";
 /** The outcome of parsing a worker's result block. */
 export type ParsedAgentResult<S = Record<string, unknown>, E = Record<string, unknown>> = {
     readonly kind: "ok";
@@ -29,6 +29,15 @@ export type ParsedAgentResult<S = Record<string, unknown>, E = Record<string, un
     readonly kind: "malformed";
     readonly reason: string;
 };
+/**
+ * Validate a parsed object against a contract track; null when it conforms.
+ *
+ * Exported because the EXPERIMENTAL emit channel (`src/experimental-emit.ts`)
+ * validates the SAME `OutputContract` on a different delivery. One contract, two
+ * deliveries, one validator — a second copy would drift, and the two rails
+ * disagreeing about what satisfies a contract is the worst outcome available.
+ */
+export declare function shapeError(obj: Record<string, unknown>, shape: Readonly<Record<string, OutputFieldType>>): string | null;
 /**
  * Parse the last `vigiles:ok` / `vigiles:err` block from a worker's output.
  *

package/dist/adapters/claude-code/agent-result.js

@@ -19,10 +19,15 @@
  * bad JSON, or a shape that doesn't match the declared schema).
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.shapeError = shapeError;
 exports.parseAgentResult = parseAgentResult;
 // Capture every vigiles:ok / vigiles:err fenced block; the LAST one is the
 // worker's final answer (earlier ones may be illustrative in its reasoning).
-const BLOCK_RE = /```vigiles:(ok|err)[ \t]*\r?\n([\s\S]*?)```/g;
+// The CLOSING fence must own its line: a ``` inside a JSON string value sits
+// mid-line, so it no longer terminates the block (measured 2026-08-13 — a
+// contract field carrying a code snippet made every such answer `malformed`
+// even though the model's JSON was valid).
+const BLOCK_RE = /^[ \t]*```vigiles:(ok|err)[ \t]*\r?\n([\s\S]*?)\r?\n^[ \t]*```[ \t]*$/gm;
 /** Does a runtime value match a declared field type? */
 function fieldMatches(value, type) {
     switch (type) {
@@ -36,7 +41,14 @@ function fieldMatches(value, type) {
             return Array.isArray(value) && value.every((v) => typeof v === "string");
     }
 }
-/** Validate a parsed object against a contract track; null when it conforms. */
+/**
+ * Validate a parsed object against a contract track; null when it conforms.
+ *
+ * Exported because the EXPERIMENTAL emit channel (`src/experimental-emit.ts`)
+ * validates the SAME `OutputContract` on a different delivery. One contract, two
+ * deliveries, one validator — a second copy would drift, and the two rails
+ * disagreeing about what satisfies a contract is the worst outcome available.
+ */
 function shapeError(obj, shape) {
     for (const [field, type] of Object.entries(shape)) {
         if (!(field in obj))

package/dist/eval.js

@@ -981,7 +981,23 @@ async function executeTrial(spec, arm, trialIndex, runner, cfg) {
 }
 // A signal in the captured streams that the model call was rate-limited /
 // overloaded — worth a backoff + retry rather than counting as a real sample.
-const RATE_LIMIT_RE = /rate.?limit|\b429\b|overloaded|too many requests/i;
+//
+// 🔴 The separator is `[ -]?`, NOT `.?`, and that is load-bearing. Claude Code's
+// stream-json emits an INFORMATIONAL `{"type":"rate_limit_event","rate_limit_info":
+// {"status":"allowed",…}}` line on EVERY run (captured verbatim in
+// examples/experimental-emit/records/rate-limit-event.json). `rate.?limit` matched
+// `rate_limit_event`, because `.` matches `_` — so every trial looked rate-limited,
+// every trial was retried `retries + 1` = 4 times, and only the LAST attempt's cost
+// reached `maxCostUsd`. Measured 2026-08-13: 2 trials → 8 model runs, a budget cap
+// of $0.60 crossed at roughly $3, and the run reported one trial. `[ -]?` cannot
+// match `_`, so the telemetry line is no longer a match; a REAL limit still is,
+// because the API reports it as `rate_limit_error` / 429 / "overloaded".
+//
+// Known boundary, stated rather than hidden: a `rate_limit_event` whose `status`
+// is a rejection is no longer a match either. It never was one on its merits — the
+// old pattern fired on the event's NAME regardless of status, so status-aware
+// detection has never existed here.
+const RATE_LIMIT_RE = /rate[ -]?limit(?:ed)?\b|rate_limit_error|\b429\b|overloaded|too many requests/i;
 /** Whether a run's captured output looks like a rate-limit / overload. Pure. */
 function isRateLimited(out) {
     return RATE_LIMIT_RE.test(`${out.stderr ?? ""}\n${out.stdout}`);

package/dist/experimental-emit.d.ts

@@ -0,0 +1,163 @@
+/**
+ * ⚠️ EXPERIMENTAL — the EMIT delivery for a typed result: the skill CALLS a tool
+ * carrying its outcome instead of ENDING its turn with a fenced block.
+ *
+ * ## Why this exists
+ *
+ * A typed `output` (an `OutputContract`) is valid only on a forked skill: compile
+ * hard-errors `output-without-fork` on the other 31, because an inline skill is
+ * spliced into the conversation, has no call→return boundary, and therefore has
+ * no return value to type (`research/spec-syntax-and-railway-scope.md`).
+ *
+ * A TOOL CALL needs no return boundary. The skill does not return the structure —
+ * it EMITS it, mid-conversation, and the call lands in `Trace.toolCalls`. So the
+ * objection that grounds the exclusion does not apply to this delivery. Same
+ * `OutputContract`; a different way of getting it out.
+ *
+ * ## What is UNPROVEN (why the `experimental_` prefix is on every runtime export)
+ *
+ *  1. **N=8, one skill, one model.** Measured 2026-08-13 against `paper-status`
+ *     (unforked; `allowed-tools: Bash, Read, Grep, Glob`) on sonnet: 8 runs, 8
+ *     emits, all on the `ok` track, all parsing against the contract, none
+ *     repeated. That answers "does it land at all" and nothing about the rate —
+ *     8 of 8 bounds the true failure rate at about 31%, which is not evidence of
+ *     reliability. Raw arguments + the free re-scorer:
+ *     `examples/experimental-emit/`.
+ *  2. **Nobody depends on it.** Neither `compileSkill` nor `compileAgent` emits
+ *     this instruction; the caller pastes `.instruction` into a skill body and
+ *     serves `.tool` from their own MCP server by hand. There is no compile-time
+ *     path, so no skill in any corpus is typed by it yet.
+ *  3. **The transport is not part of the contract.** MCP is how the tool reached
+ *     the model in the measurement. Whether a plugin can hand the model a tool
+ *     WITHOUT a separate server process is untested, and the answer changes what
+ *     this surface should look like.
+ *  4. **The runtime does not enforce `required`.** Measured: Claude Code accepted a
+ *     call omitting three declared-required fields (raw proof in `mine`,
+ *     `vigiles/repro/output-contract-2026-08-13/mcp-arm/schema-probe-emitted.jsonl`).
+ *     `inputSchema` is DESCRIPTION for the model, not a runtime gate — which is
+ *     exactly why `experimental_parseEmitted` re-validates on the receiving side
+ *     and why "the API validates it for you" must not be claimed.
+ *
+ * ## What would have to be true to drop the prefix
+ *
+ *  - A rate, not an existence proof: ≥30 trials across ≥2 unforked skills and ≥2
+ *    models, with the emit-landing rate reported and its failure modes named.
+ *  - One consumer inside vigiles that compiles the instruction from the spec, so
+ *    the tool name and the shape cannot drift apart by hand.
+ *  - A measured answer to (3) — plugin-served tool vs external MCP server — since
+ *    an in-process tool would remove the standing-up cost this surface assumes.
+ *
+ * Until then: not covered by the stability guarantee, may change or be removed
+ * without a major bump. See `docs/../STABILITY.md` and `src/experimental.ts`.
+ *
+ * @experimental
+ * @module
+ */
+import type { OutputContract } from "./core/spec.js";
+import type { ToolCall } from "./core/harness-driver.js";
+import { type ParsedAgentResult } from "./adapters/claude-code/agent-result.js";
+/** A JSON-Schema fragment for one declared field. */
+export type EmitFieldSchema = {
+    readonly type: "string";
+} | {
+    readonly type: "number";
+} | {
+    readonly type: "boolean";
+} | {
+    readonly type: "array";
+    readonly items: {
+        readonly type: "string";
+    };
+};
+/** The `track` discriminator's schema — the only enum this surface emits. */
+export interface EmitTrackSchema {
+    readonly type: "string";
+    readonly enum: readonly ["ok", "err"];
+}
+/** Anything that can sit under `properties`: a field, the discriminator, a track. */
+export type EmitPropertySchema = EmitFieldSchema | EmitTrackSchema | EmitObjectSchema;
+/** A JSON-Schema object node — one track's payload, or the whole argument. */
+export interface EmitObjectSchema {
+    readonly type: "object";
+    readonly properties: Readonly<Record<string, EmitPropertySchema>>;
+    readonly required: readonly string[];
+    readonly additionalProperties: false;
+}
+/** An MCP tool definition, in the shape a `tools/list` response carries. */
+export interface EmitToolDefinition {
+    readonly name: string;
+    readonly description: string;
+    readonly inputSchema: EmitObjectSchema;
+}
+/** What `experimental_emitTool` hands back: the tool, and the prose that asks for it. */
+export interface ExperimentalEmitTool {
+    /** Serve this from your MCP server's `tools/list`. */
+    readonly tool: EmitToolDefinition;
+    /**
+     * Markdown fragment for the skill body. The SAME contract rendered for the
+     * model — kept next to the schema so the two cannot drift when hand-wired.
+     */
+    readonly instruction: string;
+}
+/**
+ * ⚠️ EXPERIMENTAL. Derive an emit TOOL from an `OutputContract` — the same
+ * contract the fork rail renders as a `vigiles:ok` / `vigiles:err` fenced block.
+ *
+ *   const emit = experimental_emitTool(contract);
+ *   // emit.tool        → serve from your MCP server
+ *   // emit.instruction → paste into the (unforked) skill's body
+ *
+ * The two tracks are NESTED (`{ track, ok? , err? }`), not flattened into one bag
+ * of fields. That is deliberate: a flat union cannot say which fields are required
+ * on which track, so "success fields mixed with error fields" would be a
+ * well-formed call. Nested, it is not expressible.
+ *
+ * 🔴 `required` in the returned schema is DESCRIPTION, not enforcement — measured,
+ * see the module header (4). Validate what arrives with
+ * `experimental_parseEmitted`.
+ *
+ * @experimental
+ */
+export declare function experimental_emitTool(contract: OutputContract, options?: {
+    readonly name?: string;
+}): ExperimentalEmitTool;
+/**
+ * ⚠️ EXPERIMENTAL. Read the emitted result out of a run's tool calls, validated
+ * against the contract. Pure — returns the same `ParsedAgentResult` vocabulary the
+ * fenced rail's `parseAgentResult` returns, so an eval `measure` can use it as a
+ * metric and the assertion below can wrap it, without one dual-purpose function.
+ *
+ * Accepts `Trace["toolCalls"]`, `SubagentTrace["toolCalls"]` or an eval
+ * `ctx.toolCalls`. Names are matched bare (`emit_result`) or MCP-prefixed
+ * (`mcp__<server>__emit_result`).
+ *
+ * Differs from the fenced rail in ONE deliberate way: **more than one call is
+ * `malformed`, not last-one-wins.** The fenced parser takes the LAST block because
+ * an earlier block may be illustrative reasoning; a tool call is an action, never
+ * illustrative, so "exactly once" is checkable here and is not checkable there.
+ *
+ * @experimental
+ */
+export declare function experimental_parseEmitted(toolCalls: readonly ToolCall[], contract: OutputContract, options?: {
+    readonly name?: string;
+}): ParsedAgentResult;
+/**
+ * ⚠️ EXPERIMENTAL. Assert the run emitted a SUCCESS result, and return its value —
+ * the emit-channel counterpart of `assertAgentOk`, for a skill that has no return
+ * value to assert on.
+ *
+ * Throws on a missing emit, a repeated emit, an error track, or a payload that
+ * does not match the contract. The failure message names every tool the run DID
+ * call, because "the skill never emitted" and "the skill emitted the wrong shape"
+ * are different bugs and the tool list separates them at a glance.
+ *
+ * The error track is reachable through `experimental_parseEmitted`; a matching
+ * `…EmittedErr` is deliberately NOT shipped while the surface is this young —
+ * three exports is the whole prototype.
+ *
+ * @experimental
+ */
+export declare function experimental_assertEmittedOk(toolCalls: readonly ToolCall[], contract: OutputContract, options?: {
+    readonly name?: string;
+}): Record<string, unknown>;
+//# sourceMappingURL=experimental-emit.d.ts.map

package/dist/experimental-emit.js

@@ -0,0 +1,260 @@
+"use strict";
+/**
+ * ⚠️ EXPERIMENTAL — the EMIT delivery for a typed result: the skill CALLS a tool
+ * carrying its outcome instead of ENDING its turn with a fenced block.
+ *
+ * ## Why this exists
+ *
+ * A typed `output` (an `OutputContract`) is valid only on a forked skill: compile
+ * hard-errors `output-without-fork` on the other 31, because an inline skill is
+ * spliced into the conversation, has no call→return boundary, and therefore has
+ * no return value to type (`research/spec-syntax-and-railway-scope.md`).
+ *
+ * A TOOL CALL needs no return boundary. The skill does not return the structure —
+ * it EMITS it, mid-conversation, and the call lands in `Trace.toolCalls`. So the
+ * objection that grounds the exclusion does not apply to this delivery. Same
+ * `OutputContract`; a different way of getting it out.
+ *
+ * ## What is UNPROVEN (why the `experimental_` prefix is on every runtime export)
+ *
+ *  1. **N=8, one skill, one model.** Measured 2026-08-13 against `paper-status`
+ *     (unforked; `allowed-tools: Bash, Read, Grep, Glob`) on sonnet: 8 runs, 8
+ *     emits, all on the `ok` track, all parsing against the contract, none
+ *     repeated. That answers "does it land at all" and nothing about the rate —
+ *     8 of 8 bounds the true failure rate at about 31%, which is not evidence of
+ *     reliability. Raw arguments + the free re-scorer:
+ *     `examples/experimental-emit/`.
+ *  2. **Nobody depends on it.** Neither `compileSkill` nor `compileAgent` emits
+ *     this instruction; the caller pastes `.instruction` into a skill body and
+ *     serves `.tool` from their own MCP server by hand. There is no compile-time
+ *     path, so no skill in any corpus is typed by it yet.
+ *  3. **The transport is not part of the contract.** MCP is how the tool reached
+ *     the model in the measurement. Whether a plugin can hand the model a tool
+ *     WITHOUT a separate server process is untested, and the answer changes what
+ *     this surface should look like.
+ *  4. **The runtime does not enforce `required`.** Measured: Claude Code accepted a
+ *     call omitting three declared-required fields (raw proof in `mine`,
+ *     `vigiles/repro/output-contract-2026-08-13/mcp-arm/schema-probe-emitted.jsonl`).
+ *     `inputSchema` is DESCRIPTION for the model, not a runtime gate — which is
+ *     exactly why `experimental_parseEmitted` re-validates on the receiving side
+ *     and why "the API validates it for you" must not be claimed.
+ *
+ * ## What would have to be true to drop the prefix
+ *
+ *  - A rate, not an existence proof: ≥30 trials across ≥2 unforked skills and ≥2
+ *    models, with the emit-landing rate reported and its failure modes named.
+ *  - One consumer inside vigiles that compiles the instruction from the spec, so
+ *    the tool name and the shape cannot drift apart by hand.
+ *  - A measured answer to (3) — plugin-served tool vs external MCP server — since
+ *    an in-process tool would remove the standing-up cost this surface assumes.
+ *
+ * Until then: not covered by the stability guarantee, may change or be removed
+ * without a major bump. See `docs/../STABILITY.md` and `src/experimental.ts`.
+ *
+ * @experimental
+ * @module
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.experimental_emitTool = experimental_emitTool;
+exports.experimental_parseEmitted = experimental_parseEmitted;
+exports.experimental_assertEmittedOk = experimental_assertEmittedOk;
+const agent_result_js_1 = require("./adapters/claude-code/agent-result.js");
+/** The default tool name, when `options.name` is not given. */
+const DEFAULT_EMIT_TOOL = "emit_result";
+function fieldSchema(type) {
+    switch (type) {
+        case "string":
+            return { type: "string" };
+        case "number":
+            return { type: "number" };
+        case "boolean":
+            return { type: "boolean" };
+        case "string[]":
+            return { type: "array", items: { type: "string" } };
+        default:
+            // 🔴 Unreachable from TypeScript, reachable from JavaScript — and the one
+            // shipped example of this API (examples/experimental-emit/run-emit.mjs) is
+            // .mjs, so this is the path a real author takes.
+            //
+            // Without this arm the switch fell through to `undefined`, and every later
+            // step read that as a field: `"actions" in properties` is TRUE for an
+            // undefined value, so nothing noticed; `JSON.stringify` then DROPPED the
+            // key while `required` kept it and `additionalProperties: false` forbade
+            // it. The served schema demanded a property it also banned — unsatisfiable,
+            // silent, and contradicted by `.instruction`, which still asked the model
+            // to send it. Throwing here makes the contradiction impossible to construct
+            // instead of merely unlikely.
+            throw new TypeError(`experimental_emitTool: unsupported field type ${JSON.stringify(type)}. ` +
+                `Supported: "string", "number", "boolean", "string[]". ` +
+                `Nested objects and enums are outside this surface's ceiling — flatten the field, ` +
+                `or use the fenced fork rail if the shape cannot be flattened.`);
+    }
+}
+function trackSchema(shape) {
+    const properties = {};
+    for (const [field, type] of Object.entries(shape)) {
+        properties[field] = fieldSchema(type);
+    }
+    return {
+        type: "object",
+        properties,
+        required: Object.keys(shape),
+        additionalProperties: false,
+    };
+}
+/** Render a declared shape the way the fenced contract renders it, for the prose half. */
+function renderShape(shape) {
+    const fields = Object.entries(shape)
+        .map(([k, t]) => `"${k}": ${t}`)
+        .join(", ");
+    return fields ? `{ ${fields} }` : "{}";
+}
+/**
+ * ⚠️ EXPERIMENTAL. Derive an emit TOOL from an `OutputContract` — the same
+ * contract the fork rail renders as a `vigiles:ok` / `vigiles:err` fenced block.
+ *
+ *   const emit = experimental_emitTool(contract);
+ *   // emit.tool        → serve from your MCP server
+ *   // emit.instruction → paste into the (unforked) skill's body
+ *
+ * The two tracks are NESTED (`{ track, ok? , err? }`), not flattened into one bag
+ * of fields. That is deliberate: a flat union cannot say which fields are required
+ * on which track, so "success fields mixed with error fields" would be a
+ * well-formed call. Nested, it is not expressible.
+ *
+ * 🔴 `required` in the returned schema is DESCRIPTION, not enforcement — measured,
+ * see the module header (4). Validate what arrives with
+ * `experimental_parseEmitted`.
+ *
+ * @experimental
+ */
+function experimental_emitTool(contract, options = {}) {
+    const name = options.name ?? DEFAULT_EMIT_TOOL;
+    const tool = {
+        name,
+        description: "Emit this task's structured result. Call this exactly once. Set " +
+            '`track` to "ok" and fill `ok` on success, or "err" and fill `err` on ' +
+            "failure. Do not call it twice and do not fill both tracks.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                track: { type: "string", enum: ["ok", "err"] },
+                ok: trackSchema(contract.ok),
+                err: trackSchema(contract.err),
+            },
+            required: ["track"],
+            additionalProperties: false,
+        },
+    };
+    const instruction = [
+        "## Output contract",
+        "",
+        `Emit your result by calling the \`${name}\` tool exactly once, at the point`,
+        "you have the answer. Do not print it as a code block; the call IS the result.",
+        "",
+        `On success: \`track: "ok"\`, with \`ok\` =`,
+        "",
+        "```json",
+        renderShape(contract.ok),
+        "```",
+        "",
+        `On failure: \`track: "err"\`, with \`err\` =`,
+        "",
+        "```json",
+        renderShape(contract.err),
+        "```",
+    ].join("\n");
+    return { tool, instruction };
+}
+/** Does this observed tool name refer to `name` (bare, or MCP-prefixed)? */
+function isEmitCall(observed, name) {
+    return observed === name || observed.endsWith(`__${name}`);
+}
+/**
+ * ⚠️ EXPERIMENTAL. Read the emitted result out of a run's tool calls, validated
+ * against the contract. Pure — returns the same `ParsedAgentResult` vocabulary the
+ * fenced rail's `parseAgentResult` returns, so an eval `measure` can use it as a
+ * metric and the assertion below can wrap it, without one dual-purpose function.
+ *
+ * Accepts `Trace["toolCalls"]`, `SubagentTrace["toolCalls"]` or an eval
+ * `ctx.toolCalls`. Names are matched bare (`emit_result`) or MCP-prefixed
+ * (`mcp__<server>__emit_result`).
+ *
+ * Differs from the fenced rail in ONE deliberate way: **more than one call is
+ * `malformed`, not last-one-wins.** The fenced parser takes the LAST block because
+ * an earlier block may be illustrative reasoning; a tool call is an action, never
+ * illustrative, so "exactly once" is checkable here and is not checkable there.
+ *
+ * @experimental
+ */
+function experimental_parseEmitted(toolCalls, contract, options = {}) {
+    const name = options.name ?? DEFAULT_EMIT_TOOL;
+    const calls = toolCalls.filter((c) => isEmitCall(c.name, name));
+    if (calls.length === 0) {
+        return { kind: "malformed", reason: `no \`${name}\` tool call in the run` };
+    }
+    if (calls.length > 1) {
+        return {
+            kind: "malformed",
+            reason: `\`${name}\` was called ${String(calls.length)} times; the contract is exactly once`,
+        };
+    }
+    const input = calls[0].input;
+    if (typeof input !== "object" || input === null || Array.isArray(input)) {
+        return {
+            kind: "malformed",
+            reason: `\`${name}\` call carried no object argument`,
+        };
+    }
+    const args = input;
+    const track = args.track;
+    if (track !== "ok" && track !== "err") {
+        return {
+            kind: "malformed",
+            reason: `\`${name}\` call has no \`track\` of "ok" or "err"`,
+        };
+    }
+    const payload = args[track];
+    if (typeof payload !== "object" ||
+        payload === null ||
+        Array.isArray(payload)) {
+        return {
+            kind: "malformed",
+            reason: `\`${name}\` call declared track "${track}" but carried no \`${track}\` object`,
+        };
+    }
+    const obj = payload;
+    const bad = (0, agent_result_js_1.shapeError)(obj, track === "ok" ? contract.ok : contract.err);
+    if (bad)
+        return { kind: "malformed", reason: `${track} payload: ${bad}` };
+    return track === "ok"
+        ? { kind: "ok", value: obj }
+        : { kind: "err", error: obj };
+}
+/**
+ * ⚠️ EXPERIMENTAL. Assert the run emitted a SUCCESS result, and return its value —
+ * the emit-channel counterpart of `assertAgentOk`, for a skill that has no return
+ * value to assert on.
+ *
+ * Throws on a missing emit, a repeated emit, an error track, or a payload that
+ * does not match the contract. The failure message names every tool the run DID
+ * call, because "the skill never emitted" and "the skill emitted the wrong shape"
+ * are different bugs and the tool list separates them at a glance.
+ *
+ * The error track is reachable through `experimental_parseEmitted`; a matching
+ * `…EmittedErr` is deliberately NOT shipped while the surface is this young —
+ * three exports is the whole prototype.
+ *
+ * @experimental
+ */
+function experimental_assertEmittedOk(toolCalls, contract, options = {}) {
+    const r = experimental_parseEmitted(toolCalls, contract, options);
+    if (r.kind === "ok")
+        return r.value;
+    const why = r.kind === "err"
+        ? `it emitted an error result: ${JSON.stringify(r.error)}`
+        : r.reason;
+    const observed = toolCalls.map((c) => c.name).join(", ") || "none";
+    throw new Error(`expected an emitted success result, but ${why} (tools called: ${observed})`);
+}
+//# sourceMappingURL=experimental-emit.js.map

package/dist/experimental.d.ts

@@ -10,8 +10,14 @@
  * NOT covered by the stability guarantee (STABILITY.md): the shape may change or
  * be removed WITHOUT a major-version bump. Do not depend on it in production.
  *
- * Current contents — the R3 disposable-service tier (real side-effect testing;
- * see docs/measuring-skills.md § Experimental and src/services.ts).
+ * Current contents:
+ *  - the R3 disposable-service tier (real side-effect testing; see
+ *    docs/measuring-skills.md § Experimental and src/services.ts);
+ *  - the EMIT delivery for a typed result (`src/experimental-emit.ts`) — a skill
+ *    that CALLS a tool with its outcome instead of ending its turn with a fenced
+ *    block, which is how an UNFORKED skill can carry an `OutputContract` at all.
+ *    Read that module's header before using it: it lists, by number, what is
+ *    unproven and what would have to be true to drop the prefix.
  *
  * ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
  * the ONLY isolation vigiles provides — it does not confine the skill's filesystem
@@ -24,4 +30,5 @@
  */
 export { experimental_startServices, experimental_withServices, type ServiceSpec, type ServiceReady, type ServiceReset, type ServiceHandle, type ServiceSession, type ContainerRuntime, } from "./services.js";
 export { experimental_dockerRuntime, makeDockerRuntime, type DockerExec, type NetProbe, } from "./services-docker.js";
+export { experimental_emitTool, experimental_parseEmitted, experimental_assertEmittedOk, type EmitFieldSchema, type EmitObjectSchema, type EmitPropertySchema, type EmitTrackSchema, type EmitToolDefinition, type ExperimentalEmitTool, } from "./experimental-emit.js";
 //# sourceMappingURL=experimental.d.ts.map

package/dist/experimental.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = void 0;
+exports.experimental_assertEmittedOk = exports.experimental_parseEmitted = exports.experimental_emitTool = exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = void 0;
 /**
  * `vigiles/experimental` — ⚠️ EXPERIMENTAL, UNSTABLE public surface.
  *
@@ -13,8 +13,14 @@ exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experim
  * NOT covered by the stability guarantee (STABILITY.md): the shape may change or
  * be removed WITHOUT a major-version bump. Do not depend on it in production.
  *
- * Current contents — the R3 disposable-service tier (real side-effect testing;
- * see docs/measuring-skills.md § Experimental and src/services.ts).
+ * Current contents:
+ *  - the R3 disposable-service tier (real side-effect testing; see
+ *    docs/measuring-skills.md § Experimental and src/services.ts);
+ *  - the EMIT delivery for a typed result (`src/experimental-emit.ts`) — a skill
+ *    that CALLS a tool with its outcome instead of ending its turn with a fenced
+ *    block, which is how an UNFORKED skill can carry an `OutputContract` at all.
+ *    Read that module's header before using it: it lists, by number, what is
+ *    unproven and what would have to be true to drop the prefix.
  *
  * ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
  * the ONLY isolation vigiles provides — it does not confine the skill's filesystem
@@ -31,4 +37,8 @@ Object.defineProperty(exports, "experimental_withServices", { enumerable: true,
 var services_docker_js_1 = require("./services-docker.js");
 Object.defineProperty(exports, "experimental_dockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.experimental_dockerRuntime; } });
 Object.defineProperty(exports, "makeDockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.makeDockerRuntime; } });
+var experimental_emit_js_1 = require("./experimental-emit.js");
+Object.defineProperty(exports, "experimental_emitTool", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_emitTool; } });
+Object.defineProperty(exports, "experimental_parseEmitted", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_parseEmitted; } });
+Object.defineProperty(exports, "experimental_assertEmittedOk", { enumerable: true, get: function () { return experimental_emit_js_1.experimental_assertEmittedOk; } });
 //# sourceMappingURL=experimental.js.map

package/dist/mutations.d.ts

@@ -0,0 +1,96 @@
+/** One edit: the file, the exact substring to find, and what replaces it. */
+export type MutationEdit = readonly [
+    file: string,
+    find: string,
+    replace: string
+];
+/** One planted defect and the assertion that must catch it. */
+export interface MutationCase {
+    /** Short id, printed in the report. */
+    readonly name: string;
+    /** What the mutation takes away, in words — the column a reader scans. */
+    readonly disables: string;
+    /**
+     * The edits that plant it. A LIST, because some defects are only expressible as more than one:
+     * removing a guard AND the vocabulary check that would otherwise throw for an unrelated reason.
+     * Every edit must match its `find` EXACTLY ONCE, or the case is reported `not-applied`.
+     */
+    readonly edits: readonly MutationEdit[];
+    /** The test file that must go red. Its absence is refused, loudly, before anything is touched. */
+    readonly test: string;
+    /** A substring the test's complaint must contain, so a kill by a NEIGHBOUR is not counted. */
+    readonly expect: string;
+}
+/**
+ * - `killed` — the test went red AND printed `expect`. The only outcome that counts as proof.
+ * - `wrong-assertion` — red, but not with this case's message: two defects share one assertion and
+ *   neither is really watched.
+ * - `survived` — the test stayed green. The assertion is vacuous, or absent.
+ * - `unjudgeable` — the test was ALREADY red before the run, and `expect` never printed, so "red"
+ *   carries no information about this mutation. Not a pass and not a failure of the assertion.
+ * - `not-applied` — the edit did not land: `find` matched zero or several times, or the
+ *   replacement equalled the original.
+ */
+export type MutationVerdict = "killed" | "wrong-assertion" | "survived" | "unjudgeable" | "not-applied";
+/** What one case did. */
+export interface MutationOutcome {
+    readonly name: string;
+    readonly disables: string;
+    readonly verdict: MutationVerdict;
+    /** Human-readable specifics — which file, how many matches, whether a retry was needed. */
+    readonly detail: string;
+}
+/** What a whole run did. */
+export interface MutationReport {
+    readonly outcomes: readonly MutationOutcome[];
+    /** Cases with verdict `killed`. */
+    readonly killed: number;
+    /** Test files that were red BEFORE any mutation, so they can testify about nothing. */
+    readonly alreadyRed: readonly string[];
+    /**
+     * Whether every test that was green before the run is green again after it. `false` means the
+     * restore failed and the working tree is not what it was — the one outcome worth interrupting for.
+     */
+    readonly restored: boolean;
+}
+export interface RunMutationsOptions {
+    /** Working directory for the test runs; every path in `edits` and `test` is absolute. */
+    readonly cwd: string;
+    readonly cases: readonly MutationCase[];
+    /**
+     * Extra environment for each test run, merged over `process.env`.
+     * Use it to hand the test the same variables its normal runner would.
+     */
+    readonly env?: NodeJS.ProcessEnv;
+}
+/**
+ * Plant each case's defect, run the test that owns it, restore, and report what happened.
+ *
+ * ```ts
+ * const report = runMutations({
+ *   cwd: repoRoot,
+ *   cases: [{
+ *     name: "year",
+ *     disables: "the year comparison",
+ *     edits: [[checker, "rec.year !== ourYear", "false"]],
+ *     test: harness,
+ *     expect: "a wrong year was not reported",
+ *   }],
+ * });
+ * console.log(formatMutationReport(report));
+ * process.exit(report.killed === report.outcomes.length && report.restored ? 0 : 1);
+ * ```
+ *
+ * 🔴 THIS REWRITES THE FILES NAMED IN `edits` AND RESTORES THEM. The restore runs in a `finally`
+ * AND on SIGINT/SIGTERM, because a run killed midway would otherwise leave a neutered checker on
+ * disk — and the next run would then measure an already-broken repo and call it healthy. Commit
+ * before running. (Do not reach for `git stash` to clear the tree: the stash is repository-global,
+ * so a second agent working in another worktree of the same repo will pop your entries.)
+ *
+ * @throws if `cases` is empty, or if any named test file does not exist — both BEFORE any file is
+ * touched, so the message arrives with a clean working tree.
+ */
+export declare function runMutations(o: RunMutationsOptions): MutationReport;
+/** Render a report the way the CLI-style runners in this package render theirs. */
+export declare function formatMutationReport(report: MutationReport): string;
+//# sourceMappingURL=mutations.d.ts.map

package/dist/mutations.js

@@ -0,0 +1,237 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runMutations = runMutations;
+exports.formatMutationReport = formatMutationReport;
+/**
+ * `runMutations` — prove a test can FAIL, by breaking the thing it watches.
+ *
+ * A green test says the checker passed. It does not say the test would notice if the check were
+ * deleted: an assertion can be vacuous, a fixture can be wrong in the same direction as the code,
+ * and both look exactly like a pass. The only way to tell is to plant a defect and require the
+ * test to go red — with the message that NAMES that defect, not merely with something red.
+ *
+ * ## Why this is in vigiles rather than in a repo's own scripts
+ *
+ * It arrived as ten hand-written copies of one driver in a dogfooding repo (1799 lines), and the
+ * copies had DRIFTED. Counted against the committed versions on 2026-08-14:
+ *
+ *   - the NO-OP guard (a replacement equal to the original leaves a green test proving nothing)
+ *     was in 4 of 10;
+ *   - the RETRY (a non-kill re-run once before it is believed) was in 1 of 10;
+ *   - the strict rule — killed by its OWN named assertion, not merely by something going red —
+ *     was in 1 of 10.
+ *
+ * Every one of those was written AFTER it caught something, in whichever copy happened to catch
+ * it, and never travelled to the other nine. Two copies also named a test file that a refactor had
+ * deleted; the runner they used exits 0 on a path matching nothing, so those cases reported
+ * SURVIVED on every run for three days. That is the argument for one engine: not less code, but
+ * one place where each of those becomes impossible again.
+ *
+ * ## What this is NOT
+ *
+ * Not Stryker/mutmut/PIT. Those GENERATE mutants from operators (flip a `<`, drop a `return`) over
+ * production code and score a suite by kill ratio. Here the mutations are HAND-AUTHORED and each
+ * one names the assertion that must catch it, because the subject is usually not general-purpose
+ * code — it is a checker, a hook, an instruction file — where "flip an operator" produces mostly
+ * unreachable nonsense and a kill ratio measures nothing. The generated-operator approach is the
+ * better tool when it applies; reach for it there.
+ *
+ * ## The contract, in one sentence
+ *
+ * Plant one defect, run the test that owns it, and require the test to fail with the message that
+ * names it — anything else is reported as a finding, never as a pass.
+ *
+ * @module
+ */
+const node_fs_1 = require("node:fs");
+const node_child_process_1 = require("node:child_process");
+const node_path_1 = require("node:path");
+const run_scripts_js_1 = require("./adapters/claude-code/run-scripts.js");
+function runTest(file, o) {
+    const argv = (0, run_scripts_js_1.interpreterArgs)(file, (0, run_scripts_js_1.detectNodeCaps)(o.cwd));
+    const r = (0, node_child_process_1.spawnSync)("node", argv, {
+        cwd: o.cwd,
+        encoding: "utf8",
+        env: { ...process.env, ...o.env },
+    });
+    return { failed: r.status !== 0, out: (r.stdout ?? "") + (r.stderr ?? "") };
+}
+/**
+ * Plant each case's defect, run the test that owns it, restore, and report what happened.
+ *
+ * ```ts
+ * const report = runMutations({
+ *   cwd: repoRoot,
+ *   cases: [{
+ *     name: "year",
+ *     disables: "the year comparison",
+ *     edits: [[checker, "rec.year !== ourYear", "false"]],
+ *     test: harness,
+ *     expect: "a wrong year was not reported",
+ *   }],
+ * });
+ * console.log(formatMutationReport(report));
+ * process.exit(report.killed === report.outcomes.length && report.restored ? 0 : 1);
+ * ```
+ *
+ * 🔴 THIS REWRITES THE FILES NAMED IN `edits` AND RESTORES THEM. The restore runs in a `finally`
+ * AND on SIGINT/SIGTERM, because a run killed midway would otherwise leave a neutered checker on
+ * disk — and the next run would then measure an already-broken repo and call it healthy. Commit
+ * before running. (Do not reach for `git stash` to clear the tree: the stash is repository-global,
+ * so a second agent working in another worktree of the same repo will pop your entries.)
+ *
+ * @throws if `cases` is empty, or if any named test file does not exist — both BEFORE any file is
+ * touched, so the message arrives with a clean working tree.
+ */
+function runMutations(o) {
+    if (o.cases.length === 0) {
+        throw new Error("runMutations: no cases. A mutation run with nothing to run reports success, which is the exact claim this API exists to make impossible.");
+    }
+    const tests = [...new Set(o.cases.map((c) => c.test))];
+    // A test path that resolves to nothing is the defect that hid for three days in the corpus this
+    // came from: the runner exits 0 on "no files matched", so every case naming it reported SURVIVED
+    // and sent the reader hunting for an assertion that was never reached.
+    const missing = tests.filter((t) => !(0, node_fs_1.existsSync)(t));
+    if (missing.length > 0) {
+        throw new Error(`runMutations: test file(s) do not exist, so no case naming them could ever be judged:\n  ${missing.join("\n  ")}`);
+    }
+    // Baselined BEFORE anything is touched. Without this, a test that is red on purpose (an open
+    // finding it is meant to report) makes every run announce "the restore failed" about a working
+    // restore, and makes every case against it look killed.
+    const alreadyRed = tests.filter((t) => runTest(t, o).failed);
+    const redBefore = new Set(alreadyRed);
+    const targets = [...new Set(o.cases.flatMap((c) => c.edits.map(([f]) => f)))];
+    const originals = new Map(targets.map((f) => [f, (0, node_fs_1.readFileSync)(f, "utf8")]));
+    const restore = () => {
+        for (const [f, text] of originals)
+            (0, node_fs_1.writeFileSync)(f, text);
+    };
+    const onSignal = (sig) => {
+        restore();
+        process.exit(sig === "SIGINT" ? 130 : 143);
+    };
+    process.on("SIGINT", onSignal);
+    process.on("SIGTERM", onSignal);
+    const outcomes = [];
+    try {
+        for (const c of o.cases) {
+            const applied = apply(c);
+            if (applied) {
+                outcomes.push({
+                    name: c.name,
+                    disables: c.disables,
+                    verdict: "not-applied",
+                    detail: applied,
+                });
+                restore();
+                continue;
+            }
+            // A non-kill is retried ONCE before it is believed. Observed on a real corpus: a row killed
+            // as named came back as a neighbour's assertion in a later full run and reproduced as a clean
+            // kill when replayed alone. Reporting a flake as a survivor sends the next reader to rewrite
+            // a working assertion, which is worse than one extra run.
+            let judged = judge(c, o, redBefore);
+            if (judged.verdict !== "killed") {
+                const second = judge(c, o, redBefore);
+                if (second.verdict === "killed")
+                    judged = {
+                        verdict: "killed",
+                        detail: "killed on retry (the first run was a flake)",
+                    };
+                else
+                    judged = second;
+            }
+            outcomes.push({ name: c.name, disables: c.disables, ...judged });
+            restore();
+        }
+    }
+    finally {
+        restore();
+        process.off("SIGINT", onSignal);
+        process.off("SIGTERM", onSignal);
+    }
+    // Only a test that was GREEN before can testify about the restore.
+    const restored = tests
+        .filter((t) => !redBefore.has(t))
+        .every((t) => !runTest(t, o).failed);
+    return {
+        outcomes,
+        killed: outcomes.filter((r) => r.verdict === "killed").length,
+        alreadyRed,
+        restored,
+    };
+}
+/** Plants one case's edits. Returns a reason string when it could not be planted, else null. */
+function apply(c) {
+    for (const [file, find, replace] of c.edits) {
+        const src = (0, node_fs_1.readFileSync)(file, "utf8");
+        const n = src.split(find).length - 1;
+        if (n !== 1) {
+            return n === 0
+                ? `"find" matched nothing in ${(0, node_path_1.basename)(file)} — the source moved under the case`
+                : `"find" matched ${String(n)} times in ${(0, node_path_1.basename)(file)}; an edit must be unambiguous`;
+        }
+        const next = src.replace(find, replace);
+        // The mutation-that-does-not-mutate, caught against the BYTES rather than the intent: a
+        // replacement equal to the original leaves a green test that reads exactly like a kill.
+        if (next === src)
+            return `the replacement equals the original in ${(0, node_path_1.basename)(file)}`;
+        (0, node_fs_1.writeFileSync)(file, next);
+    }
+    return null;
+}
+function judge(c, o, redBefore) {
+    const { failed, out } = runTest(c.test, o);
+    if (!failed)
+        return {
+            verdict: "survived",
+            detail: `${(0, node_path_1.basename)(c.test)} stayed green with the defect planted`,
+        };
+    if (out.includes(c.expect))
+        return { verdict: "killed", detail: `named by "${c.expect}"` };
+    // When the test was ALREADY red, "red" carries no information — but the MESSAGE still does,
+    // because a runner that aborts at the first failure never reaches a later assertion. Absent it,
+    // the two causes are indistinguishable, and calling it a wrong assertion would be a guess.
+    return redBefore.has(c.test)
+        ? {
+            verdict: "unjudgeable",
+            detail: `${(0, node_path_1.basename)(c.test)} was red before the run and "${c.expect}" never printed`,
+        }
+        : {
+            verdict: "wrong-assertion",
+            detail: `red, but "${c.expect}" never printed — a neighbour's assertion caught it`,
+        };
+}
+/** Render a report the way the CLI-style runners in this package render theirs. */
+function formatMutationReport(report) {
+    const lines = [];
+    if (report.alreadyRed.length > 0) {
+        lines.push(`ℹ️  already red before any mutation: ${report.alreadyRed.map((t) => (0, node_path_1.basename)(t)).join(", ")} — excluded from the restore check; a case naming one is reported unjudgeable.`, "");
+    }
+    const width = Math.max(...report.outcomes.map((r) => r.name.length));
+    const mark = {
+        killed: "✓ killed",
+        "wrong-assertion": "🔴 wrong assertion",
+        survived: "🔴 SURVIVED",
+        unjudgeable: "🔴 unjudgeable",
+        "not-applied": "🔴 not applied",
+    };
+    for (const r of report.outcomes) {
+        lines.push(`${r.name.padEnd(width)}  ${mark[r.verdict].padEnd(20)} ${r.disables}`);
+        if (r.verdict !== "killed")
+            lines.push(`${" ".repeat(width + 2)}  ${r.detail}`);
+    }
+    lines.push("");
+    lines.push(report.restored
+        ? "restored: every test that was green before the run is green again"
+        : "🔴 RESTORE FAILED — a test that was green before the run is red now. The working tree is not what it was.");
+    const bad = report.outcomes.length - report.killed;
+    lines.push(bad === 0
+        ? `✓ all ${String(report.outcomes.length)} mutations killed, each at its own assertion`
+        : `🔴 ${String(bad)} of ${String(report.outcomes.length)} not killed as named: ${report.outcomes
+            .filter((r) => r.verdict !== "killed")
+            .map((r) => r.name)
+            .join(", ")}`);
+    return lines.join("\n");
+}
+//# sourceMappingURL=mutations.js.map

package/dist/testing.d.ts

@@ -11,6 +11,7 @@
  * `research/adapter-api-design.md`.
  */
 export { recordCheck } from "./check-count.js";
+export { runMutations, formatMutationReport, type MutationCase, type MutationEdit, type MutationOutcome, type MutationReport, type MutationVerdict, type RunMutationsOptions, } from "./mutations.js";
 export { runScript } from "./run-script.js";
 export type { RunScriptOptions, ScriptRunResult } from "./run-script.js";
 export { runHook, propertyHook, fileToolEvents } from "./run-hook.js";

package/dist/testing.js

@@ -30,7 +30,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.runHarness = exports.runHarnessTest = exports.judge = exports.hookFired = exports.loadHook = exports.stubSkillBody = exports.parseClaudeRun = exports.claudeEvalDriver = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.measureTriggerRate = exports.measureArms = exports.measure = exports.runEval = exports.fileToolEvents = exports.propertyHook = exports.runHook = exports.runScript = exports.recordCheck = void 0;
+exports.formatContainment = exports.compareContainment = exports.skillContract = exports.mustNotInclude = exports.mustInclude = exports.commandsIn = exports.runHarness = exports.runHarnessTest = exports.judge = exports.hookFired = exports.loadHook = exports.stubSkillBody = exports.parseClaudeRun = exports.claudeEvalDriver = exports.formatTriggerRateReport = exports.formatEvalReport = exports.formatCheckReport = exports.checkReportToJUnit = exports.checkPromptDiversity = exports.assertPromptDiversity = exports.assertRates = exports.measureTriggerRate = exports.measureArms = exports.measure = exports.runEval = exports.fileToolEvents = exports.propertyHook = exports.runHook = exports.runScript = exports.formatMutationReport = exports.runMutations = exports.recordCheck = void 0;
 // --- reporting: how much did this script actually do? ---
 // `vigiles test` can otherwise see only an exit code, so a file that runs NOTHING
 // prints the same `✓` as one that ran and passed (measured 2026-08-08 on a file
@@ -39,6 +39,14 @@ exports.formatContainment = exports.compareContainment = exports.skillContract =
 // vitest's `expect` — so those are visible to the runner too. See check-count.ts.
 var check_count_js_1 = require("./check-count.js");
 Object.defineProperty(exports, "recordCheck", { enumerable: true, get: function () { return check_count_js_1.recordCheck; } });
+// --- the tier above the tiers: is a passing test PROVING anything? ---
+// Every tier below reports that a check passed. None can tell a watched assertion
+// from a vacuous one — both print `✓`. `runMutations` plants a defect, runs the
+// test that owns it, and requires the test to fail with the message that NAMES
+// it, so "green" stops being the strongest claim a suite can make about itself.
+var mutations_js_1 = require("./mutations.js");
+Object.defineProperty(exports, "runMutations", { enumerable: true, get: function () { return mutations_js_1.runMutations; } });
+Object.defineProperty(exports, "formatMutationReport", { enumerable: true, get: function () { return mutations_js_1.formatMutationReport; } });
 // --- unit tier: runScript (the primitive) + runHook (it, plus a decision) ---
 // `runScript` runs any program and reports what it DID (exit, both streams,
 // writes, egress). `runHook` is that plus the hook protocol: event to stdin,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "15.2.0",
+  "version": "15.3.0",
   "description": "Audit, test and measure the harness your AI agent runs on — grade your CLAUDE.md / AGENTS.md, skills, subagents and hooks, run them against a scripted model, and measure whether they actually fire.",
   "keywords": [
     "claude-code",