vigiles 12.2.0 → 12.4.0

package/dist/services.js

@@ -0,0 +1,118 @@
+"use strict";
+/**
+ * vigiles — R3 disposable-service tier (⚠️ EXPERIMENTAL / UNSTABLE).
+ *
+ * ─────────────────────────────────────────────────────────────────────────────
+ *   EXPERIMENTAL: this surface is a DRAFT. Import it from `vigiles/experimental`,
+ *   NOT from a stable subpath. It is NOT covered by the stability guarantee and
+ *   may change shape or be removed WITHOUT a major-version bump. Do not build a
+ *   production workflow on it yet. See docs/measuring-skills.md § Experimental.
+ * ─────────────────────────────────────────────────────────────────────────────
+ *
+ * WHAT THIS IS. The free tiers (`runHook`, `measureTriggerRate`, `runEval`)
+ * cover a skill's OUTPUT and its side-effect SAFETY (did it call / not call a
+ * tool, write / not write a file) with no container. What they do NOT do is let
+ * a skill actually PERFORM a side effect against a real service and verify the
+ * resulting state — apply a migration to a real Postgres and check the row
+ * landed. That is the R3 rung (see research/eval-coverage-and-isolation.md): the
+ * REAL system's semantics are the thing under test, so it can't be faked.
+ *
+ * THE POSTURE. vigiles COMPOSES with a throwaway container; it does not reinvent
+ * the sandbox. A {@link ServiceSpec} declares a disposable service; the injected
+ * {@link ContainerRuntime} port starts it, and it is force-removed on teardown.
+ *
+ * ⚠️ SAFETY — the isolation is the DISPOSABLE CONTAINER, nothing more. A skill is
+ * model-driven, so the MODEL chooses the actions: it can do anything the run
+ * ENVIRONMENT allows. vigiles creates and destroys the container; it does NOT, in
+ * this tier, confine the skill's filesystem or block its network. So an R3 run is
+ * only as safe as the environment you run it in:
+ *   - run it in a DISPOSABLE environment — a CI job, a throwaway container/VM, or
+ *     a dev box with NO production access;
+ *   - point the task at the disposable service's connection string ONLY;
+ *   - keep real credentials OUT of the run (prod `DATABASE_URL`, cloud keys,
+ *     `~/.ssh`) — pair it with the eval tier's `ephemeralEnv` (throwaway HOME +
+ *     cleared env) to scrub them so the model has no real keys to misuse.
+ * Treat it like running an untrusted script. A future increment adds an egress
+ * wall (the skill reaches only the model + the service); until then that job is
+ * the operator's. See docs/measuring-skills.md § Experimental.
+ *
+ * WHAT SHIPS TODAY vs LATER. Today: the TYPES, the {@link ContainerRuntime} port,
+ * the pure {@link experimental_startServices} / {@link experimental_withServices}
+ * orchestration, and a Docker backend (`src/services-docker.ts`). Deferred: the
+ * `runEval` / `measureArms` `services` option + `ctx.service(name)`, per-trial
+ * reset via an eval-loop hook, and the egress wall. Requires Docker (Linux-first).
+ * It is an explicit opt-in and NEVER part of `vigiles audit` (audit stays
+ * side-effect-free).
+ *
+ * @experimental
+ * @module vigiles/experimental (services)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.experimental_startServices = experimental_startServices;
+exports.experimental_withServices = experimental_withServices;
+/**
+ * Start every declared service against an injected {@link ContainerRuntime},
+ * returning a {@link ServiceSession}. Pure orchestration over the port — no
+ * `docker` is imported here, so this is fully unit-testable with a fake runtime
+ * (the `decideSandbox`-is-pure / `runSandboxed`-is-real split). If any service
+ * fails to start, the ones already up are torn down before the error propagates,
+ * so a partial failure never leaks a container.
+ *
+ * A Docker-backed `ContainerRuntime` and the `runEval` / `measureArms` wiring are
+ * the next increment — until then this is the primitive you compose by hand:
+ * start → pin egress to `session.endpoints` → run → read `session.handles` in
+ * `measure` → `await session.teardown()`.
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+async function experimental_startServices(services, runtime) {
+    const handles = {};
+    const endpoints = [];
+    const started = [];
+    const stopAll = async () => {
+        await Promise.allSettled(started.map((h) => runtime.stop(h)));
+    };
+    try {
+        for (const [name, spec] of Object.entries(services)) {
+            const handle = await runtime.start(name, spec);
+            handles[name] = handle;
+            started.push(handle);
+            // Only a service that published a port has a reachable endpoint.
+            if (handle.port !== undefined) {
+                endpoints.push(`${handle.host}:${handle.port}`);
+            }
+        }
+    }
+    catch (err) {
+        await stopAll();
+        throw err;
+    }
+    return { handles, endpoints, teardown: stopAll };
+}
+/**
+ * Run `fn` with the declared services up, disposing them afterwards — even if
+ * `fn` throws. The scope-guard form of {@link experimental_startServices}: it
+ * removes the manual `try/finally` so a `measureArms` / `measure` call can be
+ * wrapped in one line and its containers are always cleaned up.
+ *
+ * ⚠️ Read the SAFETY note in this module's header first — the container is the
+ * only isolation; keep real credentials out of the run.
+ *
+ * LIFECYCLE NOTE (honest): the services live for the WHOLE `fn` — i.e. per-RUN,
+ * not per-trial. An eval that mutates service state across trials should make its
+ * task self-contained (e.g. `drop … if exists; create; migrate`) or run
+ * `trials: 1`. True per-trial reset needs an eval-loop hook and is the next
+ * increment (research/r3-disposable-services.md).
+ *
+ * @experimental — surface may change without a major-version bump.
+ */
+async function experimental_withServices(services, runtime, fn) {
+    const session = await experimental_startServices(services, runtime);
+    try {
+        return await fn(session);
+    }
+    finally {
+        await session.teardown();
+    }
+}
+//# sourceMappingURL=services.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vigiles",
-  "version": "12.2.0",
+  "version": "12.4.0",
   "description": "Lint & test the harness your AI agent runs on — verify the references in your CLAUDE.md / AGENTS.md and test that your hooks and skills actually work.",
   "keywords": [
     "claude-code",
@@ -44,6 +44,7 @@
     "./claude-code": "./dist/claude-code.js",
     "./codex": "./dist/codex.js",
     "./adapter": "./dist/adapter.js",
+    "./experimental": "./dist/experimental.js",
     "./vitest": {
       "types": "./dist/vitest.d.mts",
       "default": "./dist/vitest.mjs"
@@ -79,7 +80,7 @@
     "test:e2e": "npm run build && vitest run --project e2e",
     "test:cli-e2e": "bash test/e2e/run.sh",
     "test:harness": "npm run build && node dist/cli.js test",
-    "test:eval": "npm run build && node dist/cli.js eval",
+    "test:eval": "npm run build && node dist/cli.js eval --all",
     "test:vitest": "npm run build && vitest run --project runners",
     "test:jest": "npm run build && jest",
     "test:types": "npm run build && tsc --noEmit -p test/types/tsconfig.json",