vigiles 12.3.0 → 12.4.0

package/README.md

@@ -174,7 +174,15 @@ A hook that blocks nothing, a skill that hijacks unrelated prompts, context that
 
 ### 📊 Eval — does a skill help, or just cost more?
 
-_"65% fewer tokens." Says who?_ vigiles A/Bs the claim on real coding tasks and reports the token bill, whether it hit its target, and whether the code still works. promptfoo and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max subscription. Evals run locally; a committed lock file — like a `package-lock` — records the result, so CI catches stale numbers without calling the model again. (On Claude Code today; Codex eval support is landing.)
+_"Caveman Mode cuts 65% of your tokens." Says who?_ vigiles A/Bs the claim on real coding tasks and hands you three numbers: the **token bill**, whether it hit its **target**, and whether your code still **works**.
+
+```text
+caveman vs verbose · haiku · $0 on your subscription
+  output tokens   762 → 842   (+11% — the "saving" reversed)
+  correctness     1.0 → 1.0   (the fact survived)
+```
+
+Point it at any harness change that claims a number — does a compression skill pay for itself, is a subagent worth its cost, which model is cheapest here. promptfoo and DeepEval bill **per token, every run**; vigiles runs on your own Claude Pro/Max subscription, so you measure on every change, not once. A committed lock file (like `package-lock`) keeps CI honest without re-calling the model. (Claude Code today; Codex landing.)
 **[Measure a skill →](docs/measuring-skills.md)**
 
 ## Quick start

package/dist/experimental.d.ts

@@ -0,0 +1,27 @@
+/**
+ * `vigiles/experimental` — ⚠️ EXPERIMENTAL, UNSTABLE public surface.
+ *
+ * Everything re-exported here is a DRAFT. It is deliberately quarantined behind
+ * the `experimental` subpath (and the `experimental_` name prefix on runtime
+ * exports) so the import itself signals the risk at the call site:
+ *
+ *   import { experimental_startServices } from "vigiles/experimental";
+ *
+ * NOT covered by the stability guarantee (STABILITY.md): the shape may change or
+ * be removed WITHOUT a major-version bump. Do not depend on it in production.
+ *
+ * Current contents — the R3 disposable-service tier (real side-effect testing;
+ * see docs/measuring-skills.md § Experimental and src/services.ts).
+ *
+ * ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
+ * the ONLY isolation vigiles provides — it does not confine the skill's filesystem
+ * or network. Run it in a disposable environment with NO production access and
+ * keep real credentials out of the run. See the SAFETY note in src/services.ts and
+ * docs/measuring-skills.md § Experimental.
+ *
+ * @experimental
+ * @module vigiles/experimental
+ */
+export { experimental_startServices, experimental_withServices, type ServiceSpec, type ServiceReady, type ServiceReset, type ServiceHandle, type ServiceSession, type ContainerRuntime, } from "./services.js";
+export { experimental_dockerRuntime, makeDockerRuntime, type DockerExec, type NetProbe, } from "./services-docker.js";
+//# sourceMappingURL=experimental.d.ts.map

package/dist/experimental.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeDockerRuntime = exports.experimental_dockerRuntime = exports.experimental_withServices = exports.experimental_startServices = void 0;
+/**
+ * `vigiles/experimental` — ⚠️ EXPERIMENTAL, UNSTABLE public surface.
+ *
+ * Everything re-exported here is a DRAFT. It is deliberately quarantined behind
+ * the `experimental` subpath (and the `experimental_` name prefix on runtime
+ * exports) so the import itself signals the risk at the call site:
+ *
+ *   import { experimental_startServices } from "vigiles/experimental";
+ *
+ * NOT covered by the stability guarantee (STABILITY.md): the shape may change or
+ * be removed WITHOUT a major-version bump. Do not depend on it in production.
+ *
+ * Current contents — the R3 disposable-service tier (real side-effect testing;
+ * see docs/measuring-skills.md § Experimental and src/services.ts).
+ *
+ * ⚠️ SAFETY: R3 runs a model-driven skill FOR REAL. The disposable container is
+ * the ONLY isolation vigiles provides — it does not confine the skill's filesystem
+ * or network. Run it in a disposable environment with NO production access and
+ * keep real credentials out of the run. See the SAFETY note in src/services.ts and
+ * docs/measuring-skills.md § Experimental.
+ *
+ * @experimental
+ * @module vigiles/experimental
+ */
+var services_js_1 = require("./services.js");
+Object.defineProperty(exports, "experimental_startServices", { enumerable: true, get: function () { return services_js_1.experimental_startServices; } });
+Object.defineProperty(exports, "experimental_withServices", { enumerable: true, get: function () { return services_js_1.experimental_withServices; } });
+var services_docker_js_1 = require("./services-docker.js");
+Object.defineProperty(exports, "experimental_dockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.experimental_dockerRuntime; } });
+Object.defineProperty(exports, "makeDockerRuntime", { enumerable: true, get: function () { return services_docker_js_1.makeDockerRuntime; } });
+//# sourceMappingURL=experimental.js.map

package/dist/services-docker.d.ts

@@ -0,0 +1,50 @@
+import type { ContainerRuntime, ServiceSpec } from "./services.js";
+/** A synchronous docker CLI call — the injected real-IO seam. */
+export type DockerExec = (args: readonly string[]) => {
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+};
+/** A TCP readiness probe against a published host port — the second IO seam. */
+export type NetProbe = (port: number) => Promise<boolean>;
+/**
+ * A docker-safe, collision-free container name for a declared service. Sanitises
+ * the service name to docker's `[a-zA-Z0-9_.-]` charset and appends the process
+ * id + a monotonic counter so parallel/repeated runs never clash. Pure given the
+ * injected suffix, so it's unit-tested.
+ */
+export declare function containerNameFor(name: string, suffix: string): string;
+/** The `docker run -d …` argv for a service. Pure. */
+export declare function dockerRunArgs(spec: ServiceSpec, containerName: string): string[];
+/** The `docker exec <c> sh -c <cmd>` argv. Pure. */
+export declare function dockerExecArgs(containerName: string, command: string): string[];
+/** The port to publish — a 0-or-1 element list (v0 is single-port). Pure. */
+export declare function publishedPorts(spec: ServiceSpec): number[];
+/**
+ * Parse the published host port out of `docker port <c> <containerPort>` output,
+ * e.g. `0.0.0.0:49153` or `127.0.0.1:49153\n[::]:49153` → `49153`. Returns
+ * `undefined` when nothing is published — parse-don't-validate: the caller gets
+ * "a port or nothing", never a magic `0` to special-case. Pure.
+ */
+export declare function parseDockerPort(output: string): number | undefined;
+/**
+ * Build a Docker-backed {@link ContainerRuntime}. The two real-IO seams default
+ * to the real docker CLI + a real TCP probe, but both are injectable so the
+ * lifecycle is unit-testable with fakes.
+ *
+ * @experimental
+ */
+export declare function makeDockerRuntime(deps?: {
+    exec?: DockerExec;
+    netProbe?: NetProbe;
+    sleep?: (ms: number) => Promise<void>;
+    /** Readiness deadline; injectable so a test can cover the timeout path. */
+    readyTimeoutMs?: number;
+}): ContainerRuntime;
+/**
+ * The default Docker-backed runtime — pass to `experimental_startServices`.
+ *
+ * @experimental
+ */
+export declare const experimental_dockerRuntime: ContainerRuntime;
+//# sourceMappingURL=services-docker.d.ts.map

package/dist/services-docker.js

@@ -0,0 +1,234 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.experimental_dockerRuntime = void 0;
+exports.containerNameFor = containerNameFor;
+exports.dockerRunArgs = dockerRunArgs;
+exports.dockerExecArgs = dockerExecArgs;
+exports.publishedPorts = publishedPorts;
+exports.parseDockerPort = parseDockerPort;
+exports.makeDockerRuntime = makeDockerRuntime;
+/**
+ * vigiles — a Docker-backed {@link ContainerRuntime} for the R3 disposable-service
+ * tier (⚠️ EXPERIMENTAL / UNSTABLE — see src/services.ts and `vigiles/experimental`).
+ *
+ * This is the v0 backend the R3 build spec (research/r3-disposable-services.md)
+ * scopes: `docker run` a throwaway service, wait for it to be ready, run its seed,
+ * hand back a {@link ServiceHandle} whose `exec` shells into the container, and
+ * `docker rm -f` it on teardown.
+ *
+ * The design mirrors `src/sandbox.ts` / `src/egress.ts`: the COMMAND BUILDERS and
+ * output parsers are pure (unit-tested), and the two real-IO seams — the
+ * synchronous docker CLI (`exec`) and the TCP readiness probe (`netProbe`) — are
+ * INJECTED, so the orchestration is fully testable with fakes and only the
+ * end-to-end integration test needs a live daemon (it skips when absent).
+ *
+ * @experimental
+ * @module vigiles/experimental (docker backend)
+ */
+const node_child_process_1 = require("node:child_process");
+const node_net_1 = require("node:net");
+const hash_js_1 = require("./core/hash.js");
+/** How long to wait for a service to become ready before giving up. */
+const DEFAULT_READY_TIMEOUT_MS = 30_000;
+/** Delay between readiness polls. */
+const READY_POLL_MS = 250;
+let containerCounter = 0;
+/**
+ * A docker-safe, collision-free container name for a declared service. Sanitises
+ * the service name to docker's `[a-zA-Z0-9_.-]` charset and appends the process
+ * id + a monotonic counter so parallel/repeated runs never clash. Pure given the
+ * injected suffix, so it's unit-tested.
+ */
+function containerNameFor(name, suffix) {
+    const safe = name.replace(/[^a-zA-Z0-9_.-]/g, "-").replace(/^[-.]+/, "") || "svc";
+    return `vigiles-${safe}-${suffix}`;
+}
+/** The `docker run -d …` argv for a service. Pure. */
+function dockerRunArgs(spec, containerName) {
+    const args = ["run", "-d", "--rm", "--name", containerName];
+    for (const [k, v] of Object.entries(spec.env ?? {})) {
+        args.push("-e", `${k}=${v}`);
+    }
+    for (const port of publishedPorts(spec)) {
+        // Publish to an ephemeral loopback host port; `docker port` discovers it.
+        args.push("-p", `127.0.0.1::${port}`);
+    }
+    args.push(spec.image);
+    return args;
+}
+/** The `docker exec <c> sh -c <cmd>` argv. Pure. */
+function dockerExecArgs(containerName, command) {
+    return ["exec", containerName, "sh", "-c", command];
+}
+/** The port to publish — a 0-or-1 element list (v0 is single-port). Pure. */
+function publishedPorts(spec) {
+    return spec.port === undefined ? [] : [spec.port];
+}
+/**
+ * Parse the published host port out of `docker port <c> <containerPort>` output,
+ * e.g. `0.0.0.0:49153` or `127.0.0.1:49153\n[::]:49153` → `49153`. Returns
+ * `undefined` when nothing is published — parse-don't-validate: the caller gets
+ * "a port or nothing", never a magic `0` to special-case. Pure.
+ */
+function parseDockerPort(output) {
+    for (const line of output.split("\n")) {
+        const m = /:(\d+)\s*$/.exec(line.trim());
+        if (m)
+            return Number(m[1]);
+    }
+    return undefined;
+}
+/* v8 ignore start -- the real-daemon IO seams: they spawn the real `docker` CLI /
+   open a real socket / sleep, and are exercised by the gated integration test
+   (skipped without a daemon) + the real-socket unit test — not the coverage gate,
+   which the pure builders + the fake-injected orchestration below carry. Same
+   pattern as sandbox.ts's `runSandboxed`. */
+/** Default real docker CLI seam (synchronous). */
+const realDockerExec = (args) => {
+    const r = (0, node_child_process_1.spawnSync)("docker", args, { encoding: "utf-8" });
+    return {
+        stdout: r.stdout ?? "",
+        stderr: r.stderr ?? "",
+        code: r.status ?? 1,
+    };
+};
+/** Default TCP probe: resolve true if a connection to 127.0.0.1:port succeeds. */
+const realNetProbe = (port) => new Promise((resolve) => {
+    const sock = (0, node_net_1.connect)({ host: "127.0.0.1", port }, () => {
+        sock.destroy();
+        resolve(true);
+    });
+    sock.on("error", () => {
+        sock.destroy();
+        resolve(false);
+    });
+    sock.setTimeout(1000, () => {
+        sock.destroy();
+        resolve(false);
+    });
+});
+const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Parse the authoring union into a tagged {@link ReadyProbe} (exec > log > tcp). */
+function parseReady(ready) {
+    if ("exec" in ready)
+        return { kind: "exec", command: ready.exec };
+    if ("log" in ready)
+        return { kind: "log", pattern: ready.log };
+    return { kind: "tcp", containerPort: ready.tcp }; // carry the DECLARED port
+}
+/** One readiness attempt for a parsed {@link ReadyProbe} — exhaustive by `kind`. */
+function probeReady(ctx, probe) {
+    switch (probe.kind) {
+        case "exec":
+            return Promise.resolve(ctx.exec(dockerExecArgs(ctx.containerName, probe.command)).code === 0);
+        case "log": {
+            const r = ctx.exec(["logs", ctx.containerName]);
+            return Promise.resolve(probe.pattern.test(r.stdout + r.stderr));
+        }
+        case "tcp": {
+            // Resolve the DECLARED tcp port's host mapping (it must be the published
+            // port) — the value isn't discarded, so a wrong port surfaces as an error.
+            const hostPort = parseDockerPort(ctx.exec(["port", ctx.containerName, String(probe.containerPort)])
+                .stdout);
+            if (hostPort === undefined) {
+                throw new Error(`readiness { tcp: ${probe.containerPort} } — container port ${probe.containerPort} is not published for "${ctx.containerName}"`);
+            }
+            return ctx.netProbe(hostPort);
+        }
+        /* v8 ignore next 2 -- exhaustiveness guard, unreachable given ReadyProbe */
+        default:
+            return (0, hash_js_1.assertNever)(probe);
+    }
+}
+/** Poll {@link probeReady} until it passes or the deadline elapses. */
+async function waitReady(ctx, ready, timeoutMs) {
+    if (!ready)
+        return;
+    const probe = parseReady(ready); // parse ONCE, then poll
+    const started = numericNow();
+    for (;;) {
+        if (await probeReady(ctx, probe))
+            return;
+        if (numericNow() - started >= timeoutMs) {
+            throw new Error(`service "${ctx.containerName}" did not become ready within ${timeoutMs}ms`);
+        }
+        await ctx.sleep(READY_POLL_MS);
+    }
+}
+// Isolated so the deadline math is injectable-free but still centralised.
+function numericNow() {
+    return Date.now();
+}
+/**
+ * Build a Docker-backed {@link ContainerRuntime}. The two real-IO seams default
+ * to the real docker CLI + a real TCP probe, but both are injectable so the
+ * lifecycle is unit-testable with fakes.
+ *
+ * @experimental
+ */
+function makeDockerRuntime(deps = {}) {
+    const exec = deps.exec ?? realDockerExec;
+    const netProbe = deps.netProbe ?? realNetProbe;
+    const sleep = deps.sleep ?? delay;
+    const readyTimeoutMs = deps.readyTimeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
+    return {
+        name: "docker",
+        available() {
+            return exec(["info"]).code === 0;
+        },
+        async start(name, spec) {
+            const containerName = containerNameFor(name, `${process.pid}-${containerCounter++}`);
+            const run = exec(dockerRunArgs(spec, containerName));
+            if (run.code !== 0) {
+                throw new Error(`docker run failed for service "${name}" (${spec.image}): ${run.stderr.trim()}`);
+            }
+            const primary = publishedPorts(spec)[0];
+            const hostPort = primary === undefined
+                ? undefined
+                : parseDockerPort(exec(["port", containerName, String(primary)]).stdout);
+            const handle = {
+                host: "127.0.0.1",
+                port: hostPort, // undefined when the service exposes no port
+                // no `url`: the docker backend can't form a generic scheme — build your
+                // own from host + port. (Omitted, not `""`.)
+                exec(command) {
+                    const r = exec(dockerExecArgs(containerName, command));
+                    return { stdout: r.stdout, stderr: r.stderr, code: r.code };
+                },
+            };
+            try {
+                await waitReady({ exec, netProbe, sleep, containerName }, spec.ready, readyTimeoutMs);
+                if (spec.seed) {
+                    const s = exec(dockerExecArgs(containerName, spec.seed));
+                    if (s.code !== 0) {
+                        throw new Error(`seed failed for service "${name}": ${s.stderr.trim()}`);
+                    }
+                }
+            }
+            catch (err) {
+                exec(["rm", "-f", containerName]);
+                throw err;
+            }
+            // Carry the container name for stop() via a non-enumerable back-reference.
+            containerNames.set(handle, containerName);
+            return handle;
+        },
+        async stop(handle) {
+            const containerName = containerNames.get(handle);
+            if (containerName) {
+                exec(["rm", "-f", containerName]);
+                containerNames.delete(handle);
+            }
+            return Promise.resolve();
+        },
+    };
+}
+/** Maps a returned handle → its container name, so stop() needs no public field. */
+const containerNames = new WeakMap();
+/**
+ * The default Docker-backed runtime — pass to `experimental_startServices`.
+ *
+ * @experimental
+ */
+exports.experimental_dockerRuntime = makeDockerRuntime();
+//# sourceMappingURL=services-docker.js.map

package/dist/services.d.ts

@@ -0,0 +1,209 @@
+/**
+ * vigiles — R3 disposable-service tier (⚠️ EXPERIMENTAL / UNSTABLE).
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
+ *   EXPERIMENTAL: this surface is a DRAFT. Import it from `vigiles/experimental`,
+ *   NOT from a stable subpath. It is NOT covered by the stability guarantee and
+ *   may change shape or be removed WITHOUT a major-version bump. Do not build a
+ *   production workflow on it yet. See docs/measuring-skills.md § Experimental.
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * WHAT THIS IS. The free tiers (`runHook`, `measureTriggerRate`, `runEval`)
+ * cover a skill's OUTPUT and its side-effect SAFETY (did it call / not call a
+ * tool, write / not write a file) with no container. What they do NOT do is let
+ * a skill actually PERFORM a side effect against a real service and verify the
+ * resulting state — apply a migration to a real Postgres and check the row
+ * landed. That is the R3 rung (see research/eval-coverage-and-isolation.md): the
+ * REAL system's semantics are the thing under test, so it can't be faked.
+ *
+ * THE POSTURE. vigiles COMPOSES with a throwaway container; it does not reinvent
+ * the sandbox. A {@link ServiceSpec} declares a disposable service; the injected
+ * {@link ContainerRuntime} port starts it, and it is force-removed on teardown.
+ *
+ * ⚠️ SAFETY — the isolation is the DISPOSABLE CONTAINER, nothing more. A skill is
+ * model-driven, so the MODEL chooses the actions: it can do anything the run
+ * ENVIRONMENT allows. vigiles creates and destroys the container; it does NOT, in
+ * this tier, confine the skill's filesystem or block its network. So an R3 run is
+ * only as safe as the environment you run it in:
+ *   - run it in a DISPOSABLE environment — a CI job, a throwaway container/VM, or
+ *     a dev box with NO production access;
+ *   - point the task at the disposable service's connection string ONLY;
+ *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
+ *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     cleared env) to scrub them so the model has no real keys to misuse.
+ * Treat it like running an untrusted script. A future increment adds an egress
+ * wall (the skill reaches only the model + the service); until then that job is
+ * the operator's. See docs/measuring-skills.md § Experimental.
+ *
+ * WHAT SHIPS TODAY vs LATER. Today: the TYPES, the {@link ContainerRuntime} port,
+ * the pure {@link experimental_startServices} / {@link experimental_withServices}
+ * orchestration, and a Docker backend (`src/services-docker.ts`). Deferred: the
+ * `runEval` / `measureArms` `services` option + `ctx.service(name)`, per-trial
+ * reset via an eval-loop hook, and the egress wall. Requires Docker (Linux-first).
+ * It is an explicit opt-in and NEVER part of `vigiles audit` (audit stays
+ * side-effect-free).
+ *
+ * @experimental
+ * @module vigiles/experimental (services)
+ */
+/**
+ * How a service signals it is ready to accept work — polled by the
+ * {@link ContainerRuntime} before {@link ContainerRuntime.start} resolves, so a
+ * consumer never races a half-booted service.
+ *
+ * @experimental
+ */
+export type ServiceReady = {
+    readonly tcp: number;
+} | {
+    readonly log: RegExp;
+} | {
+    readonly exec: string;
+};
+/**
+ * When a fresh container is provisioned. `"per-trial"` (the DEFAULT) gives each
+ * eval trial a clean service so trial N can never observe trial N-1's writes —
+ * the repeatability property that makes a side-effecting eval trustworthy.
+ * `"per-arm"` shares one container across an arm's trials: faster, but only sound
+ * when the task is idempotent or reads only.
+ *
+ * @experimental
+ */
+export type ServiceReset = "per-trial" | "per-arm";
+/**
+ * A disposable service the agent's task may act against — declared per arm/spec
+ * on the (future) `services` eval option.
+ *
+ * @experimental
+ */
+export interface ServiceSpec {
+    /** Container image, e.g. `"postgres:16"` or `"redis:7"`. */
+    readonly image: string;
+    /** Environment for the container (e.g. `POSTGRES_PASSWORD`). */
+    readonly env?: Readonly<Record<string, string>>;
+    /**
+     * The port to publish + reach — the one `ServiceHandle.port` reports and
+     * `endpoints` lists. v0 supports a SINGLE port (the common disposable-service
+     * case: a DB, a cache); multi-port publishing is deferred rather than half-done
+     * (a declared-but-unexposed secondary port would be a misleading state).
+     */
+    readonly port?: number;
+    /** Readiness gate — start() resolves only once this passes. */
+    readonly ready?: ServiceReady;
+    /**
+     * A one-shot command run INSIDE the fresh container after it is ready and
+     * before the agent runs — e.g. `"psql -U postgres -d app -f schema.sql"`. Its
+     * output is recorded once; it is NOT model-synthesized.
+     */
+    readonly seed?: string;
+    /** Provisioning cadence — {@link ServiceReset}. Defaults to `"per-trial"`. */
+    readonly reset?: ServiceReset;
+}
+/**
+ * A started service, handed to `measure((ctx) => …)` as `ctx.service(name)` so a
+ * check can inspect the REAL resulting state.
+ *
+ * NOTE: {@link ServiceHandle.exec} is SYNCHRONOUS on purpose — the eval `measure`
+ * callback is synchronous (it blocks via `spawnSync`, like `judge()`), so state
+ * inspection must be sync too. `exec` is the ONLY inspection primitive by design:
+ * it shells the service's own CLI (`psql`, `redis-cli`, …) so vigiles never takes
+ * a per-service client dependency.
+ *
+ * @experimental
+ */
+export interface ServiceHandle {
+    /** Host the agent reaches the service on (from inside the run environment). */
+    readonly host: string;
+    /** The published host port — `undefined` for a service that exposes none. */
+    readonly port?: number;
+    /**
+     * A connection URL when the runtime can form one; omitted otherwise (build your
+     * own from `host` + `port` — the scheme differs per service, so no generic URL).
+     */
+    readonly url?: string;
+    /** Run a command inside the container and read its result (synchronous). */
+    exec(command: string): {
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly code: number;
+    };
+}
+/**
+ * A live set of started services + the teardown that disposes them. Returned by
+ * {@link experimental_startServices}; the caller wires `endpoints` into the run's
+ * egress allowlist and maps `handles` into the agent's env / into `measure`.
+ *
+ * @experimental
+ */
+export interface ServiceSession {
+    /** The started services, keyed by the name declared in the `services` map. */
+    readonly handles: Readonly<Record<string, ServiceHandle>>;
+    /**
+     * `host:port` endpoints for the started services — where to point your task.
+     * Intended to ALSO pin an egress allowlist (composing with `src/egress.ts`),
+     * but that wall is a FUTURE hardening: the eval tier does NOT apply it today, so
+     * right now these just identify the services. Do not read them as a network
+     * confinement guarantee — see the SAFETY note in this module's header.
+     */
+    readonly endpoints: readonly string[];
+    /** Stop + remove every started container. Always await this in a `finally`. */
+    teardown(): Promise<void>;
+}
+/**
+ * The container-lifecycle PORT — the seam a concrete backend implements (raw
+ * `docker` is the planned v0 default; podman / testcontainers / the macOS
+ * os-isolation backend swap in by injection, never a conditional). Kept
+ * harness-agnostic and dependency-free at the core, exactly like the five
+ * harness ports: the abstraction exists so a second backend is a NEW OBJECT, not
+ * an edit threaded through the orchestrator.
+ *
+ * `start` is responsible for HONOURING {@link ServiceSpec.ready} + running
+ * {@link ServiceSpec.seed}, so it resolves only with a ready, seeded service.
+ *
+ * @experimental
+ */
+export interface ContainerRuntime {
+    /** Backend name for diagnostics, e.g. `"docker"`. */
+    readonly name: string;
+    /** Whether this backend can actually run here (e.g. `docker info` succeeds). */
+    available(): boolean | Promise<boolean>;
+    /** Provision + ready + seed one service, returning its handle. */
+    start(name: string, spec: ServiceSpec): Promise<ServiceHandle>;
+    /** Stop + remove a started service. */
+    stop(handle: ServiceHandle): Promise<void>;
+}
+/**
+ * Start every declared service against an injected {@link ContainerRuntime},
+ * returning a {@link ServiceSession}. Pure orchestration over the port — no
+ * `docker` is imported here, so this is fully unit-testable with a fake runtime
+ * (the `decideSandbox`-is-pure / `runSandboxed`-is-real split). If any service
+ * fails to start, the ones already up are torn down before the error propagates,
+ * so a partial failure never leaks a container.
+ *
+ * A Docker-backed `ContainerRuntime` and the `runEval` / `measureArms` wiring are
+ * the next increment — until then this is the primitive you compose by hand:
+ * start → pin egress to `session.endpoints` → run → read `session.handles` in
+ * `measure` → `await session.teardown()`.
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+export declare function experimental_startServices(services: Readonly<Record<string, ServiceSpec>>, runtime: ContainerRuntime): Promise<ServiceSession>;
+/**
+ * Run `fn` with the declared services up, disposing them afterwards — even if
+ * `fn` throws. The scope-guard form of {@link experimental_startServices}: it
+ * removes the manual `try/finally` so a `measureArms` / `measure` call can be
+ * wrapped in one line and its containers are always cleaned up.
+ *
+ * ⚠️ Read the SAFETY note in this module's header first — the container is the
+ * only isolation; keep real credentials out of the run.
+ *
+ * LIFECYCLE NOTE (honest): the services live for the WHOLE `fn` — i.e. per-RUN,
+ * not per-trial. An eval that mutates service state across trials should make its
+ * task self-contained (e.g. `drop … if exists; create; migrate`) or run
+ * `trials: 1`. True per-trial reset needs an eval-loop hook and is the next
+ * increment (research/r3-disposable-services.md).
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+export declare function experimental_withServices<T>(services: Readonly<Record<string, ServiceSpec>>, runtime: ContainerRuntime, fn: (session: ServiceSession) => Promise<T>): Promise<T>;
+//# sourceMappingURL=services.d.ts.map

package/dist/services.js

@@ -0,0 +1,118 @@
+"use strict";
+/**
+ * vigiles — R3 disposable-service tier (⚠️ EXPERIMENTAL / UNSTABLE).
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
+ *   EXPERIMENTAL: this surface is a DRAFT. Import it from `vigiles/experimental`,
+ *   NOT from a stable subpath. It is NOT covered by the stability guarantee and
+ *   may change shape or be removed WITHOUT a major-version bump. Do not build a
+ *   production workflow on it yet. See docs/measuring-skills.md § Experimental.
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * WHAT THIS IS. The free tiers (`runHook`, `measureTriggerRate`, `runEval`)
+ * cover a skill's OUTPUT and its side-effect SAFETY (did it call / not call a
+ * tool, write / not write a file) with no container. What they do NOT do is let
+ * a skill actually PERFORM a side effect against a real service and verify the
+ * resulting state — apply a migration to a real Postgres and check the row
+ * landed. That is the R3 rung (see research/eval-coverage-and-isolation.md): the
+ * REAL system's semantics are the thing under test, so it can't be faked.
+ *
+ * THE POSTURE. vigiles COMPOSES with a throwaway container; it does not reinvent
+ * the sandbox. A {@link ServiceSpec} declares a disposable service; the injected
+ * {@link ContainerRuntime} port starts it, and it is force-removed on teardown.
+ *
+ * ⚠️ SAFETY — the isolation is the DISPOSABLE CONTAINER, nothing more. A skill is
+ * model-driven, so the MODEL chooses the actions: it can do anything the run
+ * ENVIRONMENT allows. vigiles creates and destroys the container; it does NOT, in
+ * this tier, confine the skill's filesystem or block its network. So an R3 run is
+ * only as safe as the environment you run it in:
+ *   - run it in a DISPOSABLE environment — a CI job, a throwaway container/VM, or
+ *     a dev box with NO production access;
+ *   - point the task at the disposable service's connection string ONLY;
+ *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
+ *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     cleared env) to scrub them so the model has no real keys to misuse.
+ * Treat it like running an untrusted script. A future increment adds an egress
+ * wall (the skill reaches only the model + the service); until then that job is
+ * the operator's. See docs/measuring-skills.md § Experimental.
+ *
+ * WHAT SHIPS TODAY vs LATER. Today: the TYPES, the {@link ContainerRuntime} port,
+ * the pure {@link experimental_startServices} / {@link experimental_withServices}
+ * orchestration, and a Docker backend (`src/services-docker.ts`). Deferred: the
+ * `runEval` / `measureArms` `services` option + `ctx.service(name)`, per-trial
+ * reset via an eval-loop hook, and the egress wall. Requires Docker (Linux-first).
+ * It is an explicit opt-in and NEVER part of `vigiles audit` (audit stays
+ * side-effect-free).
+ *
+ * @experimental
+ * @module vigiles/experimental (services)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.experimental_startServices = experimental_startServices;
+exports.experimental_withServices = experimental_withServices;
+/**
+ * Start every declared service against an injected {@link ContainerRuntime},
+ * returning a {@link ServiceSession}. Pure orchestration over the port — no
+ * `docker` is imported here, so this is fully unit-testable with a fake runtime
+ * (the `decideSandbox`-is-pure / `runSandboxed`-is-real split). If any service
+ * fails to start, the ones already up are torn down before the error propagates,
+ * so a partial failure never leaks a container.
+ *
+ * A Docker-backed `ContainerRuntime` and the `runEval` / `measureArms` wiring are
+ * the next increment — until then this is the primitive you compose by hand:
+ * start → pin egress to `session.endpoints` → run → read `session.handles` in
+ * `measure` → `await session.teardown()`.
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+async function experimental_startServices(services, runtime) {
+    const handles = {};
+    const endpoints = [];
+    const started = [];
+    const stopAll = async () => {
+        await Promise.allSettled(started.map((h) => runtime.stop(h)));
+    };
+    try {
+        for (const [name, spec] of Object.entries(services)) {
+            const handle = await runtime.start(name, spec);
+            handles[name] = handle;
+            started.push(handle);
+            // Only a service that published a port has a reachable endpoint.
+            if (handle.port !== undefined) {
+                endpoints.push(`${handle.host}:${handle.port}`);
+            }
+        }
+    }
+    catch (err) {
+        await stopAll();
+        throw err;
+    }
+    return { handles, endpoints, teardown: stopAll };
+}
+/**
+ * Run `fn` with the declared services up, disposing them afterwards — even if
+ * `fn` throws. The scope-guard form of {@link experimental_startServices}: it
+ * removes the manual `try/finally` so a `measureArms` / `measure` call can be
+ * wrapped in one line and its containers are always cleaned up.
+ *
+ * ⚠️ Read the SAFETY note in this module's header first — the container is the
+ * only isolation; keep real credentials out of the run.
+ *
+ * LIFECYCLE NOTE (honest): the services live for the WHOLE `fn` — i.e. per-RUN,
+ * not per-trial. An eval that mutates service state across trials should make its
+ * task self-contained (e.g. `drop … if exists; create; migrate`) or run
+ * `trials: 1`. True per-trial reset needs an eval-loop hook and is the next
+ * increment (research/r3-disposable-services.md).
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+async function experimental_withServices(services, runtime, fn) {
+    const session = await experimental_startServices(services, runtime);
+    try {
+        return await fn(session);
+    }
+    finally {
+        await session.teardown();
+    }
+}
+//# sourceMappingURL=services.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "12.3.0",
+  "version": "12.4.0",
   "description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
   "keywords": [
     "claude-code",
@@ -44,6 +44,7 @@
     "./claude-code": "./dist/claude-code.js",
     "./codex": "./dist/codex.js",
     "./adapter": "./dist/adapter.js",
+    "./experimental": "./dist/experimental.js",
     "./vitest": {
       "types": "./dist/vitest.d.mts",
       "default": "./dist/vitest.mjs"