metrik-bench 0.1.0__tar.gz

metrik_bench-0.1.0/.gitignore

@@ -0,0 +1,89 @@
+# Metrik — .gitignore
+#
+# NOTE: the previous version of this file contained `docs/*`, `.gitignore`, and `.claude`,
+# which excluded the entire documentation tree — i.e. every Phase 0/1 deliverable — from
+# version control. Replaced deliberately. docs/ is the project's primary output right now.
+
+# --- Build output that must never be committed ---
+# ADR 016: diagram sources live in docs/diagrams/*.mmd; rendered images are build output.
+docs/static/diagrams/
+# ADR 015: generated reference docs are committed on purpose (drift detection), so they
+# are NOT ignored here — see docs/reference/.
+
+# --- Python ---
+__pycache__/
+*.py[cod]
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+.venv/
+venv/
+env/
+
+# Tooling caches
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.hypothesis/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+.benchmarks/
+.tox/
+.nox/
+
+# uv (ADR 013) — uv.lock IS committed; the cache is not.
+.uv/
+
+# --- Node / web (ADR 009) ---
+node_modules/
+.next/
+out/
+.turbo/
+*.tsbuildinfo
+.pnpm-store/
+
+# --- Tauri (ADR 010) ---
+apps/desktop/src-tauri/target/
+apps/desktop/src-tauri/gen/
+
+# --- Docs site (ADR 015) ---
+docs/.docusaurus/
+docs/build/
+
+# --- Metrik runtime data ---
+# The artifact store and blob store (ADR 006). Content-addressed, machine-generated,
+# gigabytes. Never in git.
+.metrik/
+*.metrik-store/
+# Local config may hold API keys (ADR 018).
+metrik.local.toml
+.env
+.env.*
+!.env.example
+
+# Model weights and checkpoints — always too large, often license-encumbered (plan.md §5.7).
+*.safetensors
+*.gguf
+*.onnx
+*.pt
+*.pth
+*.bin
+# ...except the tiny-random fixtures, which are committed on purpose (ADR 017).
+!tests/fixtures/**
+
+# --- Editors and OS ---
+.vscode/
+!.vscode/extensions.json
+.idea/
+*.swp
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# --- Local agent/tooling settings ---
+.claude/settings.local.json

metrik_bench-0.1.0/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: metrik-bench
+Version: 0.1.0
+Summary: Metrik Benchmark Lab — sandboxed evaluation, or a refusal. Never an unsandboxed run.
+Project-URL: Homepage, https://github.com/Asmodeus14/Metrik
+Project-URL: Repository, https://github.com/Asmodeus14/Metrik
+Project-URL: Issues, https://github.com/Asmodeus14/Metrik/issues
+Author: The Metrik Authors
+License-Expression: Apache-2.0
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: metrik-sdk==0.1.0
+Description-Content-Type: text/markdown
+
+# metrik-bench
+
+Sandboxed evaluation — or a refusal. Never an unsandboxed run.
+
+```console
+$ metrik bench --check          # what isolation is available here, and what is not
+$ metrik bench <model> --task humaneval
+```
+
+## Why this package is unusual
+
+HumanEval and MBPP ask a model to write code and then **run it**. A benchmark runner that
+`exec()`s model output on your machine is a remote-code-execution primitive with extra steps.
+
+[ADR 019](../../docs/planning/adr/019-benchmark-sandbox.md) answers that by refusing rather
+than degrading, and records the clause as **not revisitable**. There is no in-process fallback,
+no `--i-know-what-im-doing` flag, and no configuration that produces one.
+
+That is enforced in two places, deliberately:
+
+- **The type.** `BenchResult` cannot be constructed for a code-executing task without a sandbox
+  attestation, and `isolation: "none"` fails validation. A runner that finds no container
+  runtime can emit nothing at all.
+- **The runner.** It checks for a runtime *before* generating anything, so the refusal arrives
+  in a second rather than after an hour of generation.
+
+## What a refusal looks like
+
+```
+error: humaneval executes model-generated code and no container runtime is available
+  -> Install Docker Desktop and start it, then re-run. `metrik bench --check` shows
+     what was probed. Tasks that do not execute code (mmlu) run without a container.
+```
+
+Note the last sentence: MMLU is multiple choice and needs no container. Requiring one for
+everything would push people to disable the check they actually need.

metrik_bench-0.1.0/README.md

@@ -0,0 +1,36 @@
+# metrik-bench
+
+Sandboxed evaluation — or a refusal. Never an unsandboxed run.
+
+```console
+$ metrik bench --check          # what isolation is available here, and what is not
+$ metrik bench <model> --task humaneval
+```
+
+## Why this package is unusual
+
+HumanEval and MBPP ask a model to write code and then **run it**. A benchmark runner that
+`exec()`s model output on your machine is a remote-code-execution primitive with extra steps.
+
+[ADR 019](../../docs/planning/adr/019-benchmark-sandbox.md) answers that by refusing rather
+than degrading, and records the clause as **not revisitable**. There is no in-process fallback,
+no `--i-know-what-im-doing` flag, and no configuration that produces one.
+
+That is enforced in two places, deliberately:
+
+- **The type.** `BenchResult` cannot be constructed for a code-executing task without a sandbox
+  attestation, and `isolation: "none"` fails validation. A runner that finds no container
+  runtime can emit nothing at all.
+- **The runner.** It checks for a runtime *before* generating anything, so the refusal arrives
+  in a second rather than after an hour of generation.
+
+## What a refusal looks like
+
+```
+error: humaneval executes model-generated code and no container runtime is available
+  -> Install Docker Desktop and start it, then re-run. `metrik bench --check` shows
+     what was probed. Tasks that do not execute code (mmlu) run without a container.
+```
+
+Note the last sentence: MMLU is multiple choice and needs no container. Requiring one for
+everything would push people to disable the check they actually need.

metrik_bench-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "metrik-bench"
+version = "0.1.0"
+description = "Metrik Benchmark Lab — sandboxed evaluation, or a refusal. Never an unsandboxed run."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [{ name = "The Metrik Authors" }]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+# No harness dependency yet. `lm-eval-harness` arrives pinned (ADR 022) with the first task
+# that needs it; this package currently owns runtime detection and the refusal path, which
+# deliberately need nothing but the standard library.
+dependencies = [
+    "metrik-sdk==0.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Asmodeus14/Metrik"
+Repository = "https://github.com/Asmodeus14/Metrik"
+Issues = "https://github.com/Asmodeus14/Metrik/issues"
+
+[tool.uv.sources]
+metrik-sdk = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/metrik"]

metrik_bench-0.1.0/src/metrik/bench/__init__.py

@@ -0,0 +1,29 @@
+"""Metrik Benchmark Lab: sandboxed evaluation, or a refusal.
+
+The runner is not implemented yet, and the order is deliberate -- the gate ships before the
+thing it gates (ADR 019). What exists is runtime detection, the task catalogue, and the
+refusal path, so that a machine which cannot contain model-generated code establishes that
+before anything is generated rather than after.
+"""
+
+from __future__ import annotations
+
+from metrik.bench.runtime import (
+    CANDIDATES,
+    Availability,
+    RuntimeInfo,
+    detect_runtimes,
+    require_runtime,
+)
+from metrik.bench.tasks import TASKS, TaskSpec, resolve_task
+
+__all__ = [
+    "CANDIDATES",
+    "TASKS",
+    "Availability",
+    "RuntimeInfo",
+    "TaskSpec",
+    "detect_runtimes",
+    "require_runtime",
+    "resolve_task",
+]

metrik_bench-0.1.0/src/metrik/bench/runtime.py

@@ -0,0 +1,169 @@
+"""Finding a container runtime, or establishing that there is not one.
+
+``artifacts.md`` §6.12, ADR 019. The one job here is to answer *"can this machine contain
+model-generated code?"* honestly, before anything is generated.
+
+Two distinctions this makes that a naive check does not.
+
+**Installed is not running.** ``docker --version`` answers from the CLI binary alone and
+succeeds happily while Docker Desktop is stopped. The question that matters is whether a
+container can actually be started, which only ``docker info`` answers -- it talks to the
+daemon. Treating the binary as proof is how a runner concludes it is safe and then discovers
+otherwise at the first sample.
+
+**Absent is not broken.** "No runtime installed" and "runtime installed but the daemon is
+down" need different instructions, so they are different results rather than one boolean.
+
+Probing is a capability check, never a hardcoded binary name: ``SandboxAttestation.runtime``
+records whichever was used, so adding Podman later is an entry in :data:`CANDIDATES` rather
+than a change to anything that consumes the result.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Final
+
+from metrik.sdk import UnsupportedError
+
+__all__ = [
+    "CANDIDATES",
+    "PROBE_TIMEOUT_S",
+    "Availability",
+    "RuntimeInfo",
+    "detect_runtimes",
+    "require_runtime",
+]
+
+#: Runtimes probed, in preference order. Docker first per ``plan.md`` §13 decision 4 -- it is
+#: what most contributors already have. Podman is probed anyway because it costs one
+#: subprocess and a machine that has it is a machine that can run benchmarks.
+CANDIDATES: Final[tuple[str, ...]] = ("docker", "podman")
+
+#: A stopped Docker Desktop can leave `docker info` hanging on a named pipe. A benchmark run
+#: waiting indefinitely to find out whether it may start is worse than a fast refusal.
+PROBE_TIMEOUT_S: Final = 10.0
+
+
+class Availability(StrEnum):
+    """Why a runtime can or cannot be used."""
+
+    READY = "ready"
+    #: Binary present, daemon unreachable. The common Docker Desktop case, and the one where
+    #: telling someone to "install Docker" wastes their time.
+    NOT_RUNNING = "not_running"
+    NOT_INSTALLED = "not_installed"
+    #: Present and responding, but the probe itself failed -- a permissions problem, usually.
+    UNUSABLE = "unusable"
+
+
+@dataclass(frozen=True)
+class RuntimeInfo:
+    """What one candidate runtime looks like on this machine."""
+
+    name: str
+    availability: Availability
+    version: str | None = None
+    detail: str = ""
+
+    @property
+    def ready(self) -> bool:
+        return self.availability is Availability.READY
+
+    @property
+    def remediation(self) -> str:
+        """What this specific machine needs, rather than generic advice."""
+        if self.availability is Availability.NOT_RUNNING:
+            return f"{self.name} is installed but not responding. Start it and re-run."
+        if self.availability is Availability.NOT_INSTALLED:
+            return f"Install {self.name}."
+        if self.availability is Availability.UNUSABLE:
+            return f"{self.name} responded with an error: {self.detail}"
+        return ""
+
+
+def _run(command: list[str]) -> tuple[int, str]:
+    """Run a probe, tolerating every way it can fail to answer."""
+    try:
+        completed = subprocess.run(
+            command,
+            capture_output=True,
+            text=True,
+            timeout=PROBE_TIMEOUT_S,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return 124, "timed out"
+    except OSError as exc:
+        return 127, str(exc)
+    output = (completed.stdout or completed.stderr or "").strip()
+    return completed.returncode, output.splitlines()[0] if output else ""
+
+
+def _probe(name: str) -> RuntimeInfo:
+    if shutil.which(name) is None:
+        return RuntimeInfo(name, Availability.NOT_INSTALLED, detail="not on PATH")
+
+    # `info`, not `--version`. The version comes from the CLI binary and says nothing about
+    # whether a container can be started; `info` requires the daemon to answer.
+    code, output = _run([name, "info", "--format", "{{.ServerVersion}}"])
+    if code == 0 and output:
+        return RuntimeInfo(name, Availability.READY, version=output)
+    if code == 124:
+        return RuntimeInfo(
+            name,
+            Availability.NOT_RUNNING,
+            detail=f"`{name} info` timed out after {PROBE_TIMEOUT_S:g}s",
+        )
+    # A non-zero `info` with the binary present almost always means the daemon is down; the
+    # message is kept so an unusual cause is still visible rather than flattened.
+    return RuntimeInfo(name, Availability.NOT_RUNNING, detail=output or f"`{name} info` failed")
+
+
+def detect_runtimes() -> list[RuntimeInfo]:
+    """Probe every candidate and report all of them.
+
+    All of them, not the first that works: ``metrik bench --check`` has to be able to say what
+    was looked for and what each one said, or "no runtime available" is unactionable.
+    """
+    return [_probe(name) for name in CANDIDATES]
+
+
+def require_runtime(*, task_id: str) -> RuntimeInfo:
+    """Return a usable runtime, or refuse.
+
+    The refusal is the feature. ADR 019 permits no in-process fallback and no flag that enables
+    one, so this raises and the caller stops -- there is deliberately no second return value
+    meaning "carry on unsafely".
+
+    Called *before* generation, so a machine that cannot contain the run finds out in a second
+    rather than after an hour of decoding.
+    """
+    found = detect_runtimes()
+    for runtime in found:
+        if runtime.ready:
+            return runtime
+
+    installed = [r for r in found if r.availability is not Availability.NOT_INSTALLED]
+    if installed:
+        # Something is there but not answering. Say which, and what to do about that one --
+        # "install Docker" to someone who has Docker installed reads as a broken tool.
+        lines = "; ".join(f"{r.name}: {r.detail}" for r in installed)
+        remediation = " ".join(r.remediation for r in installed)
+    else:
+        lines = "none of " + ", ".join(CANDIDATES) + " is installed"
+        remediation = (
+            "Install Docker Desktop and start it (plan.md sec. 13 decision 4), or Podman. "
+        )
+
+    raise UnsupportedError(
+        f"{task_id} executes model-generated code and no container runtime is available ({lines})",
+        remediation=(
+            f"{remediation}Run `metrik bench --check` to see what was probed. Tasks that do "
+            f"not execute code still run without a container. There is no unsandboxed mode "
+            f"(ADR 019)."
+        ),
+    )

metrik_bench-0.1.0/src/metrik/bench/tasks.py

@@ -0,0 +1,72 @@
+"""The task catalogue: which benchmarks exist, and which of them run code.
+
+``executes_code`` is declared per task and never inferred from the id. A pattern match on
+names like ``*eval*`` would be wrong in both directions -- MMLU contains "eval" in plenty of
+spellings and does not execute anything, while a new agentic benchmark might execute
+everything and be called something else entirely. The failure that matters is a
+code-executing task defaulting to "safe", so nothing here guesses.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Final
+
+from metrik.sdk import UserError
+
+__all__ = ["TASKS", "TaskSpec", "resolve_task"]
+
+
+@dataclass(frozen=True)
+class TaskSpec:
+    """A benchmark this build knows how to run."""
+
+    id: str
+    version: str
+    #: The field that gates sandboxing (``artifacts.md`` §6.12). Declared, never inferred.
+    executes_code: bool
+    summary: str
+    n_items: int | None = None
+    harness: str = "lm_eval_harness"
+
+
+TASKS: Final[dict[str, TaskSpec]] = {
+    "humaneval": TaskSpec(
+        id="humaneval",
+        version="1.0",
+        executes_code=True,
+        summary="164 Python problems; generated solutions are executed against unit tests.",
+        n_items=164,
+        harness="metrik_code_exec",
+    ),
+    "mbpp": TaskSpec(
+        id="mbpp",
+        version="1.0",
+        executes_code=True,
+        summary="Mostly Basic Python Problems; generated solutions are executed.",
+        n_items=500,
+        harness="metrik_code_exec",
+    ),
+    "mmlu": TaskSpec(
+        id="mmlu",
+        version="1.0",
+        executes_code=False,
+        summary="Multiple choice across 57 subjects. Nothing is executed.",
+        n_items=14042,
+    ),
+}
+
+
+def resolve_task(task_id: str) -> TaskSpec:
+    """Look up a task, or fail with the list of what exists.
+
+    An unknown task is refused rather than treated as a custom one. Accepting it would mean
+    guessing ``executes_code``, and the safe-looking guess is the dangerous one.
+    """
+    spec = TASKS.get(task_id.lower())
+    if spec is None:
+        raise UserError(
+            f"unknown task {task_id!r}",
+            remediation=f"Known tasks: {', '.join(sorted(TASKS))}. Run `metrik bench --check`.",
+        )
+    return spec

metrik_bench-0.1.0/tests/test_runtime.py

@@ -0,0 +1,140 @@
+"""Runtime detection and the refusal path.
+
+The property under test is that there is **no way through** without a runtime. ADR 019 permits
+no in-process fallback and no flag enabling one, so `require_runtime` either returns something
+usable or raises -- there is deliberately no third outcome.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from typing import Any
+
+import pytest
+
+from metrik.bench import Availability, RuntimeInfo, detect_runtimes, require_runtime, resolve_task
+from metrik.bench import runtime as runtime_module
+from metrik.sdk import UnsupportedError, UserError
+
+pytestmark = pytest.mark.unit
+
+
+def _fake_probe(monkeypatch: pytest.MonkeyPatch, results: dict[str, RuntimeInfo]) -> None:
+    monkeypatch.setattr(
+        runtime_module,
+        "_probe",
+        lambda name: results.get(name, RuntimeInfo(name, Availability.NOT_INSTALLED)),
+    )
+
+
+# -- the refusal -------------------------------------------------------------------------------
+
+
+def test_no_runtime_means_refusal(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_probe(monkeypatch, {})
+    with pytest.raises(UnsupportedError, match="no container runtime is available"):
+        require_runtime(task_id="humaneval")
+
+
+def test_the_refusal_says_there_is_no_unsandboxed_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The message has to close the door, or the next question is "how do I skip it"."""
+    _fake_probe(monkeypatch, {})
+    with pytest.raises(UnsupportedError) as caught:
+        require_runtime(task_id="humaneval")
+    assert "no unsandboxed mode" in (caught.value.remediation or "")
+
+
+def test_an_installed_but_stopped_runtime_gets_different_advice(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Telling someone with Docker installed to "install Docker" reads as a broken tool."""
+    _fake_probe(
+        monkeypatch,
+        {"docker": RuntimeInfo("docker", Availability.NOT_RUNNING, detail="daemon unreachable")},
+    )
+    with pytest.raises(UnsupportedError) as caught:
+        require_runtime(task_id="humaneval")
+    remediation = caught.value.remediation or ""
+    assert "not responding" in remediation
+    assert "Install docker." not in remediation
+
+
+def test_a_ready_runtime_is_returned(monkeypatch: pytest.MonkeyPatch) -> None:
+    _fake_probe(monkeypatch, {"docker": RuntimeInfo("docker", Availability.READY, version="27.0")})
+    assert require_runtime(task_id="humaneval").name == "docker"
+
+
+def test_the_second_candidate_is_used_when_the_first_is_down(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _fake_probe(
+        monkeypatch,
+        {
+            "docker": RuntimeInfo("docker", Availability.NOT_RUNNING),
+            "podman": RuntimeInfo("podman", Availability.READY, version="5.1"),
+        },
+    )
+    assert require_runtime(task_id="humaneval").name == "podman"
+
+
+# -- installed is not running --------------------------------------------------------------------
+
+
+def test_a_present_binary_with_a_dead_daemon_is_not_ready(monkeypatch: pytest.MonkeyPatch) -> None:
+    """`docker --version` succeeds with Docker Desktop stopped. Only `info` reaches the daemon,
+    and only the daemon can start a container."""
+    monkeypatch.setattr(shutil, "which", lambda _n: "/usr/bin/docker")
+    monkeypatch.setattr(
+        runtime_module, "_run", lambda _c: (1, "Cannot connect to the Docker daemon")
+    )
+    probed = runtime_module._probe("docker")
+    assert probed.availability is Availability.NOT_RUNNING
+    assert not probed.ready
+
+
+def test_a_hanging_probe_becomes_a_refusal_not_a_hang(monkeypatch: pytest.MonkeyPatch) -> None:
+    """A stopped Docker Desktop can leave `info` blocked on a named pipe. Waiting forever to
+    find out whether a run may start is worse than refusing quickly."""
+
+    def _timeout(*_a: Any, **_k: Any) -> Any:
+        raise subprocess.TimeoutExpired(cmd="docker", timeout=runtime_module.PROBE_TIMEOUT_S)
+
+    monkeypatch.setattr(shutil, "which", lambda _n: "/usr/bin/docker")
+    monkeypatch.setattr(subprocess, "run", _timeout)
+    probed = runtime_module._probe("docker")
+    assert probed.availability is Availability.NOT_RUNNING
+    assert "timed out" in probed.detail
+
+
+def test_a_missing_binary_is_reported_as_absent(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(shutil, "which", lambda _n: None)
+    assert runtime_module._probe("docker").availability is Availability.NOT_INSTALLED
+
+
+def test_detection_reports_every_candidate(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Not just the first that works: `--check` cannot explain "nothing available" without
+    saying what was looked for."""
+    _fake_probe(monkeypatch, {})
+    assert [r.name for r in detect_runtimes()] == list(runtime_module.CANDIDATES)
+
+
+# -- the task catalogue -------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("task_id", "executes"), [("humaneval", True), ("mbpp", True), ("mmlu", False)]
+)
+def test_executes_code_is_declared_per_task(task_id: str, executes: bool) -> None:
+    assert resolve_task(task_id).executes_code is executes
+
+
+def test_an_unknown_task_is_refused_rather_than_assumed_safe() -> None:
+    """Accepting it would mean guessing `executes_code`, and the safe-looking guess is the
+    dangerous one."""
+    with pytest.raises(UserError, match="unknown task"):
+        resolve_task("some-new-agentic-benchmark")
+
+
+def test_task_lookup_is_case_insensitive() -> None:
+    assert resolve_task("HumanEval").id == "humaneval"