agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/bench/_codeexec.py

@@ -0,0 +1,213 @@
+"""Code-tests verifier — run held-out tests against candidate code in isolation.
+
+This is the coding-modality verifier for the bench harness. It implements the
+trustworthiness rule every serious coding benchmark uses: **the oracle is held
+out** — the held-out checks live in a separate file the candidate code never
+imports or sees, and they are executed by a harness-written runner, not by the
+candidate.
+
+Sandboxes:
+  * ``subprocess`` (default) — a fresh interpreter in a throwaway tempdir, with a
+    scrubbed environment (no harness secrets) and a hard wall-clock timeout. This
+    is the sandbox used by the credential-free release gate, which only ever runs
+    trusted, shipped reference code. It is **not** a security boundary against
+    deliberately hostile code (no real filesystem/network isolation); for
+    untrusted agent output use the Docker lane (bench step 15E).
+  * ``docker`` — per-task container isolation with a no-network default
+    (bench step 15E).
+
+The convention for a checks file: it defines one or more ``check_*`` callables
+that import the candidate module (``import solution``) and ``assert`` the
+expected behaviour. The harness discovers them, runs each, and reports per-check
+pass/fail — so a candidate that no-ops, prints a fake "success" message, returns
+wrong answers, or fails to define the entrypoint is failed deterministically.
+
+THREAT-MODEL NOTE: the runner and the candidate share one process, so the
+deterministic-failure guarantee covers *accidental* gaming, not an *adversarial*
+candidate. A candidate that knows this runner's protocol could, during its import
+body (which runs before ``check_*``), print a forged ``{"results": ...}`` line and
+exit 0, or read the checks file to reflect expected values. Hardening that
+(process/UID separation of the oracle + an authenticated out-of-band verdict
+channel — the inject-tests-after-the-agent-finishes topology) is tracked separate
+work; until then do not treat a passing score from an untrusted adversarial
+candidate as authoritative. The release gate is unaffected: it runs only trusted
+shipped reference code.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from ..live._runner import scrubbed_lane_env
+
+#: Entry module the candidate code is written to (checks ``import`` this name).
+ENTRY_MODULE = "solution"
+_CHECKS_MODULE = "bench_checks"
+_DEFAULT_TIMEOUT_S = 10.0
+
+SUPPORTED_LANGUAGES = ("python",)
+
+# The harness-written runner: discovers ``check_*`` callables in the checks
+# module, runs each, and emits a single JSON line of per-check results to stdout.
+# The candidate (``solution.py``) is imported only by the checks module — never
+# by this runner directly — so the oracle stays out of the candidate's reach.
+_PYTHON_RUNNER = """\
+import importlib, json, sys, traceback
+results = {}
+fatal = None
+try:
+    checks = importlib.import_module("%(checks)s")
+except Exception:
+    fatal = "checks_import_failed: " + traceback.format_exc(limit=2).strip().replace(chr(10), " | ")
+    print(json.dumps({"results": {}, "fatal": fatal}))
+    sys.exit(1)
+names = sorted(n for n in dir(checks) if n.startswith("check_") and callable(getattr(checks, n)))
+if not names:
+    print(json.dumps({"results": {}, "fatal": "no check_* callables found"}))
+    sys.exit(1)
+for name in names:
+    try:
+        getattr(checks, name)()
+        results[name] = True
+    except Exception:
+        results[name] = False
+print(json.dumps({"results": results, "fatal": None}))
+sys.exit(0 if results and all(results.values()) else 1)
+"""
+
+
+def _tail(text: str, limit: int = 2000) -> str:
+    text = text or ""
+    return text[-limit:]
+
+
+def _empty_result(explanation: str, raw: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "result": {
+            "scalar": 0.0,
+            "components": {"checks_passed": 0.0, "checks_total": 0.0},
+            "pass_fail": {},
+            "explanation": explanation,
+        },
+        "raw": raw,
+    }
+
+
+def run_code_tests(
+    candidate_code: str,
+    checks_code: str,
+    *,
+    language: str = "python",
+    timeout_s: float = _DEFAULT_TIMEOUT_S,
+    sandbox: str = "subprocess",
+) -> dict[str, Any]:
+    """Run ``checks_code`` (the held-out oracle) against ``candidate_code``.
+
+    Returns ``{"result": <unified Result>, "raw": <execution evidence>}``. The
+    unified ``Result`` carries a scalar (fraction of checks passed), components
+    (passed/total), per-check ``pass_fail`` booleans, and an explanation.
+    """
+
+    if language not in SUPPORTED_LANGUAGES:
+        return _empty_result(
+            f"unsupported language {language!r}; supported: {SUPPORTED_LANGUAGES}",
+            {"sandbox": sandbox, "language": language, "infra_error": True},
+        )
+    if sandbox == "docker":
+        # The Docker lane is opt-in; never silently fall back to a weaker sandbox
+        # (that would mislabel isolation).
+        from ._docker import run_code_tests_docker  # local import: optional lane
+
+        return run_code_tests_docker(
+            candidate_code, checks_code, language=language, timeout_s=timeout_s
+        )
+    if sandbox != "subprocess":
+        return _empty_result(
+            f"unknown sandbox {sandbox!r}; expected 'subprocess' or 'docker'",
+            {"sandbox": sandbox, "language": language, "infra_error": True},
+        )
+
+    return _run_subprocess(candidate_code, checks_code, timeout_s=timeout_s)
+
+
+def _run_subprocess(
+    candidate_code: str, checks_code: str, *, timeout_s: float
+) -> dict[str, Any]:
+    with tempfile.TemporaryDirectory(prefix="agent-learn-bench-") as tmp:
+        root = Path(tmp)
+        (root / f"{ENTRY_MODULE}.py").write_text(candidate_code, encoding="utf-8")
+        (root / f"{_CHECKS_MODULE}.py").write_text(checks_code, encoding="utf-8")
+        (root / "_runner.py").write_text(
+            _PYTHON_RUNNER % {"checks": _CHECKS_MODULE}, encoding="utf-8"
+        )
+
+        raw: dict[str, Any] = {
+            "sandbox": "subprocess",
+            "language": "python",
+            "timed_out": False,
+            "exit_code": None,
+        }
+        try:
+            proc = subprocess.run(
+                [sys.executable, "_runner.py"],
+                cwd=str(root),
+                env=scrubbed_lane_env(()),  # no harness secrets cross into the run
+                capture_output=True,
+                text=True,
+                timeout=timeout_s,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raw["timed_out"] = True
+            raw["stdout_tail"] = _tail(exc.stdout if isinstance(exc.stdout, str) else "")
+            raw["stderr_tail"] = _tail(exc.stderr if isinstance(exc.stderr, str) else "")
+            return _empty_result(f"timed out after {timeout_s}s", raw)
+
+        raw["exit_code"] = proc.returncode
+        raw["stdout_tail"] = _tail(proc.stdout)
+        raw["stderr_tail"] = _tail(proc.stderr)
+
+        parsed = _parse_runner_stdout(proc.stdout)
+        if parsed is None:
+            return _empty_result(
+                f"runner produced no parseable result (exit {proc.returncode})", raw
+            )
+        fatal = parsed.get("fatal")
+        results = {str(k): bool(v) for k, v in (parsed.get("results") or {}).items()}
+        if fatal:
+            return _empty_result(str(fatal), raw)
+        if not results:
+            return _empty_result("no checks executed", raw)
+
+        total = len(results)
+        passed = sum(1 for v in results.values() if v)
+        return {
+            "result": {
+                "scalar": round(passed / total, 6),
+                "components": {
+                    "checks_passed": float(passed),
+                    "checks_total": float(total),
+                },
+                "pass_fail": results,
+                "explanation": f"{passed}/{total} checks passed",
+            },
+            "raw": raw,
+        }
+
+
+def _parse_runner_stdout(stdout: str) -> dict[str, Any] | None:
+    # The runner prints exactly one JSON line; tolerate trailing candidate prints
+    # by scanning for the last decodable JSON object.
+    for line in reversed((stdout or "").splitlines()):
+        line = line.strip()
+        if not line.startswith("{"):
+            continue
+        try:
+            return json.loads(line)
+        except json.JSONDecodeError:
+            continue
+    return None

fi/alk/bench/_coding.py

@@ -0,0 +1,215 @@
+"""Coding-modality bench suite + the artifact-in runner.
+
+A coding suite is a bench-native shape (``agent-learning.bench-suite.v1``): each
+task carries an ``instruction``, a held-out ``checks`` oracle (executed against
+the candidate, never imported by it), a ``reference_solution`` (the gold, used by
+the release gate to prove the verifier accepts a correct answer), and optional
+``guards``. This is deliberately distinct from the objective-anchored task
+dataset: coding's verdict is "do the held-out tests pass", not a weighted-metric
+mean. The two unify at the Result level, not the suite level — exactly the shape
+the prior-art survey found (task specs are modality-specific; the Task<->Verifier
+coupling and the unified Result are the invariant).
+
+``artifact_in`` control mode scores a *submitted artifact* (candidate code) with
+no live agent — the analogue of patch-scoring harnesses. The agent that produced
+the artifact is out of scope here; only the held-out oracle decides the verdict.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+from ._codeexec import run_code_tests
+from ._grader import GRADING_COMMAND, run_command_graded
+
+BENCH_SUITE_KIND = "agent-learning.bench-suite.v1"
+
+# A task is graded one of two ways:
+#  * "checks" (default, convenience tier): held-out check_* functions import the
+#    candidate in-process. Trusted / accidental-gaming only.
+#  * "command" (hardened tier): the candidate produces files/output, a held-out
+#    grader runs AFTER and emits the verdict via exit code + reward file. Robust
+#    against forge + oracle-read; multi-language. See _grader.py.
+_CHECKS_FIELDS = ("id", "instruction", "checks", "reference_solution")
+_COMMAND_FIELDS = ("id", "instruction", "grader_cmd", "grader_files", "reference_files")
+
+
+class CodingSuiteError(ValueError):
+    """Raised for a malformed coding bench suite."""
+
+
+def is_bench_suite(obj: Any) -> bool:
+    return isinstance(obj, Mapping) and obj.get("kind") == BENCH_SUITE_KIND
+
+
+def _task_grading(task: Mapping[str, Any]) -> str:
+    """The grading mode of a task: 'command' (hardened) or 'checks' (convenience)."""
+
+    mode = task.get("grading")
+    if mode in (GRADING_COMMAND, "checks"):
+        return str(mode)
+    # infer: a grader_cmd ⇒ command-graded; otherwise the legacy checks tier.
+    return GRADING_COMMAND if task.get("grader_cmd") else "checks"
+
+
+def load_coding_suite(obj: Mapping[str, Any] | str | Path) -> dict[str, Any]:
+    """Load + validate a coding bench suite (from a path or an in-memory mapping)."""
+
+    if isinstance(obj, (str, Path)):
+        data: Mapping[str, Any] = json.loads(Path(obj).expanduser().read_text("utf-8"))
+    else:
+        data = obj
+    if not is_bench_suite(data):
+        raise CodingSuiteError(f"not a {BENCH_SUITE_KIND} suite")
+    tasks = data.get("tasks")
+    if not isinstance(tasks, list) or not tasks:
+        raise CodingSuiteError("coding suite has no tasks")
+    seen: set[str] = set()
+    for i, task in enumerate(tasks):
+        if not isinstance(task, Mapping):
+            raise CodingSuiteError(f"task #{i} is not an object")
+        required = _COMMAND_FIELDS if _task_grading(task) == GRADING_COMMAND else _CHECKS_FIELDS
+        for field in required:
+            if not task.get(field):
+                raise CodingSuiteError(
+                    f"task #{i} ({_task_grading(task)}-graded) missing required field {field!r}"
+                )
+        tid = str(task["id"])
+        if tid in seen:
+            raise CodingSuiteError(f"duplicate task id {tid!r}")
+        seen.add(tid)
+        # Every task must declare at least one guard against reward hacking; the
+        # held-out oracle is the primary defence, but the suite must say so.
+        guards = task.get("guards") or {}
+        if int(guards.get("min_guard_count", 0)) < 1:
+            raise CodingSuiteError(
+                f"task {tid!r} must declare guards.min_guard_count >= 1 "
+                "(the held-out-oracle anti-gaming contract)"
+            )
+    return dict(data)
+
+
+def _coding_row(
+    task: Mapping[str, Any],
+    verdict_obj: Mapping[str, Any],
+    *,
+    evidence_class: str,
+    sandbox: str,
+) -> dict[str, Any]:
+    result = dict(verdict_obj["result"])
+    scalar = result.get("scalar")
+    # All-or-nothing: a coding task is resolved only if EVERY held-out check passes.
+    verdict = "pass" if scalar is not None and float(scalar) >= 1.0 else "fail"
+    return {
+        "task_id": str(task["id"]),
+        "modality": "coding",
+        "world_kind": "code_exec",
+        "control_mode": "artifact_in",
+        "result": result,
+        "verdict": verdict,
+        # The candidate code really executed; honest execution_class is executable.
+        "execution_class": "executable",
+        "evidence_class": evidence_class,
+        # executable + any evidence class is never an overclaim (it really ran).
+        "overclaim": False,
+        "sandbox": sandbox,
+        "raw": verdict_obj.get("raw", {}),
+    }
+
+
+def _void_row(task: Mapping[str, Any], reason: str, *, evidence_class: str) -> dict[str, Any]:
+    return {
+        "task_id": str(task["id"]),
+        "modality": "coding",
+        "world_kind": "code_exec",
+        "control_mode": "artifact_in",
+        "result": {
+            "scalar": None,
+            "components": {},
+            "pass_fail": {},
+            "explanation": reason,
+        },
+        "verdict": "void",
+        "execution_class": "executable",
+        "evidence_class": evidence_class,
+        "overclaim": False,
+        "error": reason,
+    }
+
+
+def run_coding_artifact_in(
+    suite: Mapping[str, Any],
+    submission: Mapping[str, Any],
+    *,
+    sandbox: str = "subprocess",
+    evidence_class: str = "captured_fixture",
+    max_tasks: int | None = None,
+    default_timeout_s: float = 10.0,
+) -> list[dict[str, Any]]:
+    """Score each task's submitted artifact against its held-out oracle.
+
+    ``submission`` maps ``task_id -> candidate``. For ``checks``-graded tasks the
+    candidate is a source string; for ``command``-graded (hardened) tasks it is a
+    ``{path: content}`` file map. A task with no submission is recorded ``void``
+    (never silently passed); an infra failure is recorded ``void`` too. Pass the
+    :func:`reference_submission` to verify the suite itself (what the gate does).
+    """
+
+    language = str(suite.get("language", "python"))
+    # Honesty: a Docker run executes untrusted candidate code under real
+    # isolation -> that is a genuine LIVE event, never a fixture/local class.
+    # Force at least live_lane (honor an explicit live_stressed); never downgrade.
+    if sandbox == "docker" and evidence_class not in ("live_lane", "live_stressed"):
+        evidence_class = "live_lane"
+    rows: list[dict[str, Any]] = []
+    tasks = suite["tasks"]
+    if max_tasks is not None:
+        tasks = tasks[: max(0, int(max_tasks))]
+    for task in tasks:
+        tid = str(task["id"])
+        candidate = submission.get(tid)
+        if candidate is None:
+            rows.append(_void_row(task, "no submission provided", evidence_class=evidence_class))
+            continue
+        timeout_s = float(task.get("timeout_s", default_timeout_s))
+        if _task_grading(task) == GRADING_COMMAND:
+            files = candidate if isinstance(candidate, Mapping) else {"solution": str(candidate)}
+            verdict_obj = run_command_graded(
+                task, {str(k): str(v) for k, v in files.items()},
+                sandbox=sandbox, timeout_s=timeout_s,
+            )
+        else:
+            verdict_obj = run_code_tests(
+                str(candidate), str(task["checks"]),
+                language=language, timeout_s=timeout_s, sandbox=sandbox,
+            )
+        # An infra/config failure (no Docker daemon, image pull failure, bad
+        # sandbox/language) means the lane never ran — record it as VOID, never as
+        # a real "fail". Conflating "the daemon was missing" with "the agent was
+        # wrong" would silently report a correct agent at 0%.
+        if (verdict_obj.get("raw") or {}).get("infra_error"):
+            reason = (verdict_obj.get("result") or {}).get("explanation") or "infrastructure error"
+            rows.append(_void_row(task, f"infra: {reason}", evidence_class=evidence_class))
+            continue
+        rows.append(
+            _coding_row(task, verdict_obj, evidence_class=evidence_class, sandbox=sandbox)
+        )
+    return rows
+
+
+def reference_submission(suite: Mapping[str, Any]) -> dict[str, Any]:
+    """The gold submission: every task id -> its reference.
+
+    ``checks``-graded tasks map to a ``reference_solution`` string; ``command``-
+    graded tasks map to a ``reference_files`` ``{path: content}`` map.
+    """
+
+    out: dict[str, Any] = {}
+    for t in suite["tasks"]:
+        if _task_grading(t) == GRADING_COMMAND:
+            out[str(t["id"])] = {str(k): str(v) for k, v in (t["reference_files"] or {}).items()}
+        else:
+            out[str(t["id"])] = str(t["reference_solution"])
+    return out

fi/alk/bench/_docker.py

@@ -0,0 +1,237 @@
+"""Docker code-exec lane — run held-out checks against untrusted candidate code
+in a per-task, network-isolated, resource-capped, ephemeral container.
+
+This is the harder-isolated sibling of the subprocess verifier in ``_codeexec``.
+For **untrusted agent output** it adds real OS-level isolation the subprocess lane
+cannot: no network, no host writes, dropped capabilities + no-new-privileges, a
+nosuid tmpfs, capped CPU/memory/PIDs, killed + removed after the run.
+
+Honesty: a Docker run of untrusted candidate code is a genuine **live** event,
+so :func:`fi.alk.bench._coding.run_coding_artifact_in` stamps these rows
+``evidence_class=live_lane`` (never ``captured_fixture``) — see that module.
+
+This lane is **opt-in** (``sandbox="docker"``) and is NEVER a release-gate
+prerequisite: the credential-free ``bench_contract_readiness`` gate runs the
+subprocess lane on trusted shipped code so it works anywhere with no Docker. The
+import is lazy (only resolved when ``sandbox="docker"`` is requested), so the kit
+imports fine on a machine with no Docker.
+
+KNOWN LIMITATION (oracle hold-out): hold-out here is only *structural* — the
+checks file is not part of the candidate's source, but it is materialised into the
+same container the candidate runs in, and the candidate's module body executes
+(at import) before the ``check_*`` functions. A deliberately adversarial candidate
+could therefore read the checks file at runtime and reflect the expected values,
+or print a forged result line. This lane defends against *accidental* gaming
+(no-op / fake-success / wrong answer all fail) and gives strong OS isolation, but
+it is NOT yet a hardened defence against a candidate that actively attacks the
+harness protocol. Closing that requires process/UID separation of the oracle from
+the candidate (the inject-tests-only-after-the-agent-finishes topology), tracked
+as the live-agent-in-container step. Do not treat a passing score from an
+untrusted, adversarial candidate as authoritative until that lands.
+"""
+
+from __future__ import annotations
+
+import base64
+import shutil
+import subprocess
+import uuid
+from typing import Any
+
+from ._codeexec import (
+    SUPPORTED_LANGUAGES,
+    _empty_result,
+    _parse_runner_stdout,
+    _tail,
+)
+
+# In-container bootstrap: materialise the candidate + held-out checks from base64
+# into the writable tmpfs (no host bind-mount — works identically on macOS Docker
+# Desktop and Linux), then run each check_* in isolation and emit one JSON line.
+# Doubled braces are literal dict syntax preserved through ``str.format``.
+_DOCKER_BOOTSTRAP = (
+    "import base64,importlib,json,sys,traceback\n"
+    "open('/tmp/solution.py','wb').write(base64.b64decode('{cand_b64}'))\n"
+    "open('/tmp/bench_checks.py','wb').write(base64.b64decode('{checks_b64}'))\n"
+    "sys.path.insert(0,'/tmp')\n"
+    "results={{}}\n"
+    "try:\n"
+    "    checks=importlib.import_module('bench_checks')\n"
+    "except Exception:\n"
+    "    print(json.dumps({{'results':{{}},'fatal':'checks_import_failed: '"
+    "+traceback.format_exc(limit=2).strip().replace(chr(10),' | ')}}));sys.exit(1)\n"
+    "names=sorted(n for n in dir(checks) if n.startswith('check_') and callable(getattr(checks,n)))\n"
+    "if not names:\n"
+    "    print(json.dumps({{'results':{{}},'fatal':'no check_* callables found'}}));sys.exit(1)\n"
+    "for name in names:\n"
+    "    try:\n"
+    "        getattr(checks,name)();results[name]=True\n"
+    "    except Exception:\n"
+    "        results[name]=False\n"
+    "print(json.dumps({{'results':results,'fatal':None}}))\n"
+    "sys.exit(0 if results and all(results.values()) else 1)\n"
+)
+
+# Default base image. Production should pin by digest (image@sha256:...) for
+# determinism; the tag default keeps the example/proof portable.
+DEFAULT_IMAGE = "python:3.11-slim"
+
+_DEFAULT_MEMORY = "256m"
+_DEFAULT_CPUS = "1.0"
+_DEFAULT_PIDS = 128
+
+
+def docker_available() -> bool:
+    """True if a working Docker daemon is reachable (cheap, no pull)."""
+
+    if shutil.which("docker") is None:
+        return False
+    try:
+        proc = subprocess.run(
+            ["docker", "info", "--format", "{{.ServerVersion}}"],
+            capture_output=True,
+            text=True,
+            timeout=15,
+        )
+    except Exception:
+        return False
+    return proc.returncode == 0
+
+
+def _build_docker_argv(
+    name: str, image: str, memory: str, cpus: str, bootstrap: str
+) -> list[str]:
+    """Build the hardened ``docker run`` argv (pure; unit-testable without a daemon).
+
+    Defense-in-depth for the untrusted lane: no network, read-only rootfs, a
+    non-root user, ALL capabilities dropped (the bounding set too — uid 65534 only
+    clears effective/permitted, but the base image ships setuid-root binaries that
+    could otherwise re-escalate), no new privileges, a nosuid size-capped tmpfs as
+    the only writable surface, and PID/memory/CPU caps. Per-task, ephemeral
+    (``--rm``); args passed as a list (never a shell).
+    """
+
+    return [
+        "docker", "run", "--rm",
+        "--name", name,
+        "--network", "none",            # the real win the subprocess lane can't give
+        "--memory", memory,
+        "--cpus", cpus,
+        "--pids-limit", str(_DEFAULT_PIDS),
+        "--user", "65534:65534",        # nobody: no in-container root
+        "--cap-drop", "ALL",            # drop the bounding set (block setuid re-escalation)
+        "--security-opt", "no-new-privileges",
+        "--read-only",                   # rootfs read-only; only the tmpfs is writable
+        "--tmpfs", "/tmp:size=16m,nosuid",  # nosec B108 — container-internal path, not a host temp
+        "--entrypoint", "python",
+        image,
+        "-B", "-c", bootstrap,           # -B: no .pyc writes under the read-only fs
+    ]
+
+
+def run_code_tests_docker(
+    candidate_code: str,
+    checks_code: str,
+    *,
+    language: str = "python",
+    timeout_s: float = 10.0,
+    image: str = DEFAULT_IMAGE,
+    memory: str = _DEFAULT_MEMORY,
+    cpus: str = _DEFAULT_CPUS,
+) -> dict[str, Any]:
+    """Run ``checks_code`` against ``candidate_code`` inside an isolated container.
+
+    Returns the same ``{"result", "raw"}`` shape as the subprocess verifier; the
+    ``raw`` block records ``sandbox="docker"``, the image, and isolation flags.
+    Never raises for an unavailable daemon or a hostile candidate — both surface
+    as an honest failing Result.
+    """
+
+    if language not in SUPPORTED_LANGUAGES:
+        return _empty_result(
+            f"unsupported language {language!r}; supported: {SUPPORTED_LANGUAGES}",
+            {"sandbox": "docker", "language": language, "infra_error": True},
+        )
+    if not docker_available():
+        return _empty_result(
+            "docker unavailable (no daemon / not installed)",
+            {"sandbox": "docker", "language": language,
+             "docker_available": False, "infra_error": True},
+        )
+
+    name = f"agent-learn-bench-{uuid.uuid4().hex[:12]}"
+    raw: dict[str, Any] = {
+        "sandbox": "docker",
+        "language": language,
+        "image": image,
+        "network": "none",
+        "memory": memory,
+        "cpus": cpus,
+        "cap_drop": "all",
+        "no_new_privileges": True,
+        "container": name,
+        "timed_out": False,
+        "exit_code": None,
+    }
+
+    bootstrap = _DOCKER_BOOTSTRAP.format(
+        cand_b64=base64.b64encode(candidate_code.encode("utf-8")).decode("ascii"),
+        checks_b64=base64.b64encode(checks_code.encode("utf-8")).decode("ascii"),
+    )
+    argv = _build_docker_argv(name, image, memory, cpus, bootstrap)
+    try:
+        proc = subprocess.run(
+            argv, capture_output=True, text=True, timeout=timeout_s + 20.0
+        )
+    except subprocess.TimeoutExpired as exc:
+        raw["timed_out"] = True
+        raw["stdout_tail"] = _tail(exc.stdout if isinstance(exc.stdout, str) else "")
+        raw["stderr_tail"] = _tail(exc.stderr if isinstance(exc.stderr, str) else "")
+        _force_kill(name)
+        return _empty_result(f"timed out after {timeout_s}s (container killed)", raw)
+
+    raw["exit_code"] = proc.returncode
+    raw["stdout_tail"] = _tail(proc.stdout)
+    raw["stderr_tail"] = _tail(proc.stderr)
+
+    # A daemon/image error (e.g. image not pulled) is infra, not an agent fail.
+    if proc.returncode not in (0, 1) and "{" not in (proc.stdout or ""):
+        raw["infra_error"] = True
+        return _empty_result(
+            f"docker run failed (exit {proc.returncode}): {_tail(proc.stderr, 300)}",
+            raw,
+        )
+
+    parsed = _parse_runner_stdout(proc.stdout)
+    if parsed is None:
+        return _empty_result(
+            f"runner produced no parseable result (exit {proc.returncode})", raw
+        )
+    fatal = parsed.get("fatal")
+    results = {str(k): bool(v) for k, v in (parsed.get("results") or {}).items()}
+    if fatal:
+        return _empty_result(str(fatal), raw)
+    if not results:
+        return _empty_result("no checks executed", raw)
+
+    total = len(results)
+    passed = sum(1 for v in results.values() if v)
+    return {
+        "result": {
+            "scalar": round(passed / total, 6),
+            "components": {
+                "checks_passed": float(passed),
+                "checks_total": float(total),
+            },
+            "pass_fail": results,
+            "explanation": f"{passed}/{total} checks passed",
+        },
+        "raw": raw,
+    }
+
+
+def _force_kill(name: str) -> None:
+    try:
+        subprocess.run(["docker", "kill", name], capture_output=True, timeout=15)
+    except Exception:  # nosec B110 — best-effort cleanup; a kill failure must never propagate
+        pass