benchkit-cli 0.1.0__py3-none-any.whl

benchkit/__init__.py

@@ -0,0 +1 @@
+"""Benchmark local LLMs with real eval suites."""

benchkit/_pi_proxy.py

@@ -0,0 +1,219 @@
+"""Restricted inference proxy used by the Pi task sandbox.
+
+The Pi container lives on an internal Docker network with no direct egress.
+This small process runs in a separate container attached to both that network
+and Docker's bridge network, forwards only OpenAI-compatible inference routes,
+and keeps the upstream API key out of the agent's filesystem and environment.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import http.client
+import json
+import os
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlsplit
+
+ALLOWED_PATHS = ("/v1/chat/completions", "/v1/models")
+MAX_BODY_BYTES = 32 * 1024 * 1024
+SCAFFOLD_PATH = "/tmp/benchkit_pi_scaffold.json"
+HOP_BY_HOP = {
+    "connection",
+    "keep-alive",
+    "proxy-authenticate",
+    "proxy-authorization",
+    "te",
+    "trailers",
+    "transfer-encoding",
+    "upgrade",
+}
+
+
+def _message_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return ""
+    return "".join(
+        str(part.get("text") or "")
+        for part in content
+        if isinstance(part, dict) and part.get("type") in {"text", "input_text"}
+    )
+
+
+def _capture_scaffold(request_data: dict) -> None:
+    """Persist the exact scaffold sent by Pi on the first model request."""
+    if os.path.exists(SCAFFOLD_PATH):
+        return
+    messages = request_data.get("messages") or []
+    system_prompt = "\n\n".join(
+        _message_text(message.get("content"))
+        for message in messages
+        if isinstance(message, dict) and message.get("role") in {"system", "developer"}
+    )
+    tools = request_data.get("tools") or []
+    tool_names = [
+        str(tool.get("function", {}).get("name") or "")
+        for tool in tools
+        if isinstance(tool, dict) and isinstance(tool.get("function"), dict)
+    ]
+    tool_names = [name for name in tool_names if name]
+    tool_schema = json.dumps(tools, ensure_ascii=False, sort_keys=True)
+    payload = {
+        "system_prompt": system_prompt,
+        "system_prompt_sha256": hashlib.sha256(system_prompt.encode()).hexdigest(),
+        "system_prompt_chars": len(system_prompt),
+        "tools_available": tool_names,
+        "tool_schema_sha256": hashlib.sha256(tool_schema.encode()).hexdigest(),
+    }
+    for field in ("max_completion_tokens", "max_tokens"):
+        value = request_data.get(field)
+        if isinstance(value, int) and value > 0:
+            payload["max_output_tokens"] = value
+            payload["max_output_tokens_field"] = field
+            break
+    try:
+        with open(SCAFFOLD_PATH, "x", encoding="utf-8") as handle:
+            json.dump(payload, handle, ensure_ascii=False)
+    except FileExistsError:
+        pass
+
+
+def _upstream_target(path: str) -> tuple[str, str, int, str]:
+    base = os.environ["BENCHKIT_UPSTREAM"].rstrip("/")
+    parsed = urlsplit(base)
+    request = urlsplit(path)
+    suffix = request.path[3:] if request.path.startswith("/v1") else request.path
+    prefix = parsed.path.rstrip("/")
+    target = f"{prefix}{suffix}" or "/"
+    query = "&".join(part for part in (parsed.query, request.query) if part)
+    if query:
+        target += f"?{query}"
+    port = parsed.port or (443 if parsed.scheme == "https" else 80)
+    return parsed.scheme, parsed.hostname or "", port, target
+
+
+def _models_payload() -> bytes:
+    model = os.environ["BENCHKIT_MODEL"]
+    return json.dumps(
+        {"object": "list", "data": [{"id": model, "object": "model"}]}
+    ).encode()
+
+
+def _upstream_timeout() -> float:
+    try:
+        return max(0.1, float(os.environ.get("BENCHKIT_UPSTREAM_TIMEOUT", "600")))
+    except ValueError:
+        return 600.0
+
+
+class ProxyHandler(BaseHTTPRequestHandler):
+    """Stream a deliberately tiny subset of an OpenAI-compatible API."""
+
+    protocol_version = "HTTP/1.1"
+
+    def do_GET(self) -> None:
+        if self.path == "/health":
+            self._plain(200, b"ok")
+            return
+        self._forward()
+
+    def do_POST(self) -> None:
+        self._forward()
+
+    def _forward(self) -> None:
+        route = self.path.split("?", 1)[0]
+        if route not in ALLOWED_PATHS:
+            self._plain(404, b"route unavailable")
+            return
+        if route == "/v1/models":
+            if self.command != "GET":
+                self._plain(405, b"method unavailable")
+                return
+            self._json(200, _models_payload())
+            return
+        if self.command != "POST":
+            self._plain(405, b"method unavailable")
+            return
+
+        length = int(self.headers.get("content-length", "0") or 0)
+        if length > MAX_BODY_BYTES:
+            self._plain(413, b"request too large")
+            return
+        body = self.rfile.read(length) if length else None
+        try:
+            request_data = json.loads(body or b"{}")
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            self._plain(400, b"invalid JSON")
+            return
+        _capture_scaffold(request_data)
+        if request_data.get("model") != os.environ["BENCHKIT_MODEL"]:
+            self._plain(403, b"model unavailable")
+            return
+
+        scheme, host, port, target = _upstream_target(self.path)
+        connection_type = (
+            http.client.HTTPSConnection
+            if scheme == "https"
+            else http.client.HTTPConnection
+        )
+        connection = connection_type(host, port, timeout=_upstream_timeout())
+        headers = {
+            key: value
+            for key, value in self.headers.items()
+            if key.lower() not in HOP_BY_HOP | {"host", "authorization"}
+        }
+        if api_key := os.environ.get("BENCHKIT_UPSTREAM_API_KEY"):
+            headers["Authorization"] = f"Bearer {api_key}"
+
+        response_started = False
+        try:
+            connection.request(self.command, target, body=body, headers=headers)
+            response = connection.getresponse()
+            self.send_response(response.status, response.reason)
+            for key, value in response.getheaders():
+                if key.lower() not in HOP_BY_HOP | {"content-length"}:
+                    self.send_header(key, value)
+            self.send_header("Connection", "close")
+            self.end_headers()
+            response_started = True
+            while chunk := response.read(64 * 1024):
+                self.wfile.write(chunk)
+                self.wfile.flush()
+        except (OSError, http.client.HTTPException) as exc:
+            if not response_started and not self.wfile.closed:
+                self._plain(502, f"upstream error: {exc}".encode())
+        finally:
+            connection.close()
+            self.close_connection = True
+
+    def _plain(self, status: int, body: bytes) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", "text/plain; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(body)
+        self.close_connection = True
+
+    def _json(self, status: int, body: bytes) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        self.wfile.write(body)
+        self.close_connection = True
+
+    def log_message(self, _format: str, *_args: object) -> None:
+        return
+
+
+def main() -> None:
+    server = ThreadingHTTPServer(("0.0.0.0", 8080), ProxyHandler)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

benchkit/answer_key_guard.ts

@@ -0,0 +1,49 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { appendFileSync, realpathSync } from "node:fs";
+import { isAbsolute, normalize, resolve } from "node:path";
+
+const AUDIT_LOG = "/tmp/benchkit-answer-key-guard.jsonl";
+const ANSWER_COMPONENT = /(^|[\\/])\.(meta|approaches)([\\/]|$)/i;
+const ANSWER_NAME = /(^|[\\/])[^\\/]*(example|reference|proof)[^\\/]*([\\/]|$)/i;
+
+function forbidden(value: unknown, cwd: string): boolean {
+  if (typeof value !== "string" || !value.trim()) return false;
+  const candidate = value.trim();
+  const normalized = normalize(isAbsolute(candidate) ? candidate : resolve(cwd, candidate));
+  let resolved = normalized;
+  try { resolved = realpathSync(normalized); } catch { /* missing/glob path */ }
+  return ANSWER_COMPONENT.test(candidate) || ANSWER_NAME.test(candidate)
+    || ANSWER_COMPONENT.test(normalized) || ANSWER_NAME.test(normalized)
+    || ANSWER_COMPONENT.test(resolved) || ANSWER_NAME.test(resolved);
+}
+
+function inspectedValues(toolName: string, input: Record<string, unknown>): string[] {
+  if (toolName === "bash") return [String(input.command ?? "")];
+  if (["read", "grep", "find", "ls"].includes(toolName)) {
+    return ["path", "pattern", "glob", "query"]
+      .map((key) => input[key])
+      .filter((value): value is string => typeof value === "string");
+  }
+  return [];
+}
+
+export default function answerKeyGuard(pi: ExtensionAPI) {
+  pi.on("tool_call", async (event, ctx) => {
+    const input = event.input as Record<string, unknown>;
+    const matches = inspectedValues(event.toolName, input).filter((value) =>
+      forbidden(value, ctx.cwd),
+    );
+    if (!matches.length) return undefined;
+
+    appendFileSync(AUDIT_LOG, JSON.stringify({
+      tool: event.toolName,
+      arguments: input,
+      matches,
+      cwd: ctx.cwd,
+    }) + "\n", { encoding: "utf8", mode: 0o600 });
+    return {
+      block: true,
+      reason: "BenchKit denied access to an answer-key path",
+    };
+  });
+}

benchkit/artifacts.py

@@ -0,0 +1,87 @@
+"""Run-scoped staging for per-task binary artifacts.
+
+Benchmarks produce screenshots and generated pages while a run is still in
+flight, but the results directory only exists once :func:`benchkit.report.save`
+stamps it. Artifacts are therefore written to a staging directory keyed by the
+process, and collected into the results directory at save time with the paths
+in each task record rewritten to be relative to the report.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import re
+import shutil
+import uuid
+from pathlib import Path
+
+STAGING_ROOT = Path("results") / ".artifacts"
+_RUN_ID = uuid.uuid4().hex[:12]
+_COUNTER = 0
+
+# Artifact keys in a task's ``workspace`` payload, and where they are collected.
+COLLECTED = (
+    ("screenshot", "screenshots", ".png"),
+    ("screenshot_thumbnail", "screenshots", ".jpg"),
+    # mc-arena photographs one build from three fixed cameras; the plain
+    # ``screenshot`` above is the three of them side by side.
+    ("screenshot_iso", "screenshots", "-iso.png"),
+    ("screenshot_side", "screenshots", "-side.png"),
+    ("screenshot_top", "screenshots", "-top.png"),
+    ("page_html", "pages", ".html"),
+    ("response_text", "pages", ".txt"),
+    ("script_py", "builds", ".py"),
+    ("blocks_json", "builds", ".json"),
+)
+
+
+def staging_root() -> Path:
+    return STAGING_ROOT / _RUN_ID
+
+
+def task_dir(prefix: str) -> Path:
+    """Create a fresh directory for one evaluation's artifacts."""
+    global _COUNTER
+    _COUNTER += 1
+    directory = staging_root() / f"{_slug(prefix)}-{_COUNTER:04d}"
+    directory.mkdir(parents=True, exist_ok=True)
+    # Absolute, so a path recorded mid-run still resolves at report time.
+    return directory.resolve()
+
+
+def _slug(value: str) -> str:
+    text = re.sub(r"[^A-Za-z0-9._-]+", "-", str(value)).strip("-")
+    return text[:80] or "artifact"
+
+
+def collect(out: Path, results: list[dict]) -> None:
+    """Move staged artifacts into the report and relativize their paths."""
+    root = staging_root().resolve()
+    for result in results:
+        model = _slug(str(result.get("model") or "model"))
+        for task in result.get("tasks") or []:
+            workspace = task.get("workspace")
+            if not isinstance(workspace, dict):
+                continue
+            for key, subdir, suffix in COLLECTED:
+                source = workspace.get(key)
+                if not isinstance(source, str) or not source:
+                    continue
+                path = Path(source)
+                if not path.is_absolute() or not path.is_file():
+                    workspace.pop(key, None)
+                    continue
+                if root not in path.resolve().parents:
+                    continue
+                name = f"{model}__{_slug(str(task.get('task_id') or 'task'))}{suffix}"
+                target = out / subdir / name
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copyfile(path, target)
+                workspace[key] = f"{subdir}/{name}"
+
+
+def cleanup() -> None:
+    """Remove this process's staging directory, and the root when it empties."""
+    shutil.rmtree(staging_root(), ignore_errors=True)
+    with contextlib.suppress(OSError):
+        STAGING_ROOT.rmdir()

benchkit/benchmarks/__init__.py

@@ -0,0 +1,105 @@
+"""Available benchmarks."""
+
+from benchkit.benchmarks.aider_polyglot import AiderPolyglot
+from benchkit.benchmarks.arc import ARC
+from benchkit.benchmarks.banking77 import Banking77
+from benchkit.benchmarks.boolq import BoolQ
+from benchkit.benchmarks.chartqa import ChartQA
+from benchkit.benchmarks.code_verifier import CodeVerifier
+from benchkit.benchmarks.evalplus import HumanEvalPlus, MBPPPlus
+from benchkit.benchmarks.git_surgery import GitSurgery
+from benchkit.benchmarks.gpqa import GPQA
+from benchkit.benchmarks.gsm8k import GSM8K
+from benchkit.benchmarks.hellaswag import HellaSwag
+from benchkit.benchmarks.humaneval import HumanEval
+from benchkit.benchmarks.ifeval import IFEval
+from benchkit.benchmarks.mbpp import MBPP
+from benchkit.benchmarks.mc_arena import MCArena
+from benchkit.benchmarks.medfailbench import MedFailBench
+from benchkit.benchmarks.mmlu import MMLU
+from benchkit.benchmarks.mmlu_pro import MMLUPro
+from benchkit.benchmarks.mmvp import MMVP
+from benchkit.benchmarks.ocrbench import OCRBench
+from benchkit.benchmarks.openbookqa import OpenBookQA
+from benchkit.benchmarks.patcheval import PatchEval
+from benchkit.benchmarks.piqa import PIQA
+from benchkit.benchmarks.ruler import RULER, RULERFull
+from benchkit.benchmarks.sanity import Sanity
+from benchkit.benchmarks.treejs_arena import TreeJSArena
+from benchkit.benchmarks.truthfulqa import TruthfulQA
+from benchkit.benchmarks.vstar import VStar
+from benchkit.benchmarks.winogrande import WinoGrande
+from benchkit.benchmarks.xstest import XSTest
+
+REGISTRY: dict[str, type] = {
+    # Generative suites first, then the multiple-choice ones.
+    "aider-polyglot": AiderPolyglot,
+    "git-surgery": GitSurgery,
+    "patcheval": PatchEval,
+    "sanity": Sanity,
+    "mc-arena": MCArena,
+    "treejs-arena": TreeJSArena,
+    "humaneval": HumanEval,
+    "humaneval-plus": HumanEvalPlus,
+    "mbpp": MBPP,
+    "mbpp-plus": MBPPPlus,
+    "gsm8k": GSM8K,
+    "ifeval": IFEval,
+    "ruler": RULER,
+    "ruler-full": RULERFull,
+    "xstest": XSTest,
+    "medfailbench": MedFailBench,
+    "gpqa": GPQA,
+    "mmlu-pro": MMLUPro,
+    "banking77": Banking77,
+    "code-verifier": CodeVerifier,
+    # Vision suites send images and run on the direct harness only.
+    "ocrbench": OCRBench,
+    "chartqa": ChartQA,
+    "mmvp": MMVP,
+    "vstar": VStar,
+    # Multiple-choice suites.
+    "mmlu": MMLU,
+    "arc": ARC,
+    "openbookqa": OpenBookQA,
+    "winogrande": WinoGrande,
+    "piqa": PIQA,
+    "boolq": BoolQ,
+    "truthfulqa": TruthfulQA,
+    "hellaswag": HellaSwag,
+}
+
+DESCRIPTIONS: dict[str, str] = {
+    "aider-polyglot": "repository editing across six languages with the Pi agent",
+    "git-surgery": "stateful Git operations in isolated repositories with Pi",
+    "patcheval": "real Python bug fixes with externally isolated hidden tests",
+    "mc-arena": "frozen Minecraft build prompts, scored on the block list a generated script prints",
+    "treejs-arena": "frozen creative three.js scenes, scored only on whether they render",
+    "sanity": "25 curated checks across code, math, instructions, science, and commonsense",
+    "humaneval": "Python function completions with the original unit tests",
+    "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs",
+    "mbpp": "short Python functions from natural-language specifications",
+    "mbpp-plus": "sanitized MBPP tasks with 39k+ EvalPlus test inputs",
+    "gsm8k": "multi-step grade-school math problems with exact numeric answers",
+    "ifeval": "prompts with code-checkable instruction-following constraints",
+    "ruler": "practical long-context retrieval across all 13 RULER task families",
+    "ruler-full": "research-scale RULER with 500 samples per task and context",
+    "xstest": "safe compliance and unsafe refusal using an offline string matcher",
+    "medfailbench": "clinical safety screening with deterministic offline rules",
+    "arc": "challenging grade-school science questions with four choices",
+    "gpqa": "expert-written graduate science questions designed to resist search",
+    "mmlu-pro": "harder MMLU successor: ten options and reasoning-heavy questions",
+    "banking77": "route a banking customer message to one of 77 intents",
+    "code-verifier": "judge whether a model-written HumanEval+ solution passes its tests",
+    "ocrbench": "text recognition and text-centric VQA across ten image categories",
+    "chartqa": "questions about real charts, scored with 5% relaxed numeric accuracy",
+    "mmvp": "pairs of look-alike images that differ in one detail, two options",
+    "vstar": "small details in high-resolution images, two or four options",
+    "mmlu": "zero-shot coverage of 57 academic and professional subjects",
+    "openbookqa": "elementary science questions requiring facts plus reasoning",
+    "winogrande": "commonsense pronoun resolution in ambiguous sentences",
+    "piqa": "choose the most physically plausible solution to everyday tasks",
+    "boolq": "answer yes/no questions using evidence from a passage",
+    "truthfulqa": "avoid common misconceptions and select the truthful answer",
+    "hellaswag": "choose the most plausible continuation of a real-world scenario",
+}