inferencebench-code 0.0.2__py3-none-any.whl

inferencebench_code/__init__.py

@@ -0,0 +1,12 @@
+"""InferenceBench code-generation plugin."""
+
+from inferencebench_code.plugin import EXPECTED_METRICS, CodeGenerationPlugin
+from inferencebench_code.schemas import BenchmarkSpec, EngineKind, RunContext
+
+__all__ = [
+    "EXPECTED_METRICS",
+    "BenchmarkSpec",
+    "CodeGenerationPlugin",
+    "EngineKind",
+    "RunContext",
+]

inferencebench_code/benchmarks/humaneval-mini.yaml

@@ -0,0 +1,15 @@
+benchmark_id: code.generation.humaneval-mini
+suite_version: 1.0.0
+description: Five stdlib-only Python tasks, pass@1 with a 5-second wall-clock timeout.
+modality: code
+kind: generation
+dataset:
+  id: builtin-humaneval-mini
+  path: humaneval-mini.jsonl
+slo_template: code.generation.standard
+warmup:
+  discard_runs: 0
+language: python
+scoring: pass_at_1
+k: 1
+timeout_s: 5.0

inferencebench_code/benchmarks/mbpp-mini.yaml

@@ -0,0 +1,15 @@
+benchmark_id: code.generation.mbpp-mini
+suite_version: 1.0.0
+description: Mostly Basic Python Problems (MBPP)-style stdlib-only tasks, pass@1 with a 5-second wall-clock timeout.
+modality: code
+kind: generation
+dataset:
+  id: builtin-mbpp-mini
+  path: mbpp-mini.jsonl
+slo_template: code.generation.standard
+warmup:
+  discard_runs: 0
+language: python
+scoring: pass_at_1
+k: 1
+timeout_s: 5.0

inferencebench_code/datasets/humaneval-mini.jsonl

@@ -0,0 +1,5 @@
+{"task_id": "add_two_numbers", "entry_point": "add", "prompt": "def add(a, b):\n    \"\"\"Return the sum of a and b.\"\"\"\n", "canonical_solution": "def add(a, b):\n    return a + b\n", "tests": "assert add(1, 2) == 3\nassert add(0, 0) == 0\nassert add(-1, 1) == 0\nassert add(-5, -7) == -12\nassert add(100, 250) == 350\nassert add(2.5, 0.5) == 3.0\nassert add(-100, 100) == 0\n"}
+{"task_id": "reverse_string", "entry_point": "reverse_string", "prompt": "def reverse_string(s):\n    \"\"\"Return the reverse of the string s.\"\"\"\n", "canonical_solution": "def reverse_string(s):\n    return s[::-1]\n", "tests": "assert reverse_string('') == ''\nassert reverse_string('a') == 'a'\nassert reverse_string('hello') == 'olleh'\nassert reverse_string('abcde') == 'edcba'\nassert reverse_string('  spaces  ') == '  secaps  '\nassert reverse_string('racecar') == 'racecar'\nassert reverse_string('AbC') == 'CbA'\n"}
+{"task_id": "fibonacci_iter", "entry_point": "fib", "prompt": "def fib(n):\n    \"\"\"Return the n-th Fibonacci number (0-indexed: fib(0)=0, fib(1)=1).\"\"\"\n", "canonical_solution": "def fib(n):\n    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n    return a\n", "tests": "assert fib(0) == 0\nassert fib(1) == 1\nassert fib(2) == 1\nassert fib(3) == 2\nassert fib(5) == 5\nassert fib(10) == 55\nassert fib(15) == 610\n"}
+{"task_id": "count_vowels", "entry_point": "count_vowels", "prompt": "def count_vowels(s):\n    \"\"\"Count the number of vowels (a, e, i, o, u) in the lowercase string s.\"\"\"\n", "canonical_solution": "def count_vowels(s):\n    return sum(1 for c in s if c in 'aeiou')\n", "tests": "assert count_vowels('') == 0\nassert count_vowels('bcdfg') == 0\nassert count_vowels('aeiou') == 5\nassert count_vowels('hello') == 2\nassert count_vowels('python') == 1\nassert count_vowels('queue') == 4\nassert count_vowels('rhythm') == 0\n"}
+{"task_id": "is_palindrome", "entry_point": "is_palindrome", "prompt": "def is_palindrome(s):\n    \"\"\"Return True if s is a palindrome (case-insensitive). Empty string is a palindrome.\"\"\"\n", "canonical_solution": "def is_palindrome(s):\n    t = s.lower()\n    return t == t[::-1]\n", "tests": "assert is_palindrome('') is True\nassert is_palindrome('a') is True\nassert is_palindrome('racecar') is True\nassert is_palindrome('Racecar') is True\nassert is_palindrome('hello') is False\nassert is_palindrome('Level') is True\nassert is_palindrome('abba') is True\n"}

inferencebench_code/datasets/mbpp-mini.jsonl

@@ -0,0 +1,5 @@
+{"task_id": "mbpp-001", "prompt": "Write a function sum_list(items) that returns the sum of a list of integers.", "tests": "assert sum_list([1,2,3]) == 6\nassert sum_list([]) == 0\nassert sum_list([-1, -2, 3]) == 0\nassert sum_list([10, 20, 30, 40]) == 100\n", "canonical_solution": "def sum_list(items):\n    return sum(items)\n", "entry_point": "sum_list"}
+{"task_id": "mbpp-002", "prompt": "Write a function max_of_three(a, b, c) that returns the largest of three numbers.", "tests": "assert max_of_three(1, 2, 3) == 3\nassert max_of_three(5, 2, 4) == 5\nassert max_of_three(-1, -2, -3) == -1\nassert max_of_three(7, 7, 7) == 7\n", "canonical_solution": "def max_of_three(a, b, c):\n    return max(a, b, c)\n", "entry_point": "max_of_three"}
+{"task_id": "mbpp-003", "prompt": "Write a function count_evens(nums) that returns the count of even integers in the list nums.", "tests": "assert count_evens([1, 2, 3, 4]) == 2\nassert count_evens([]) == 0\nassert count_evens([2, 4, 6, 8]) == 4\nassert count_evens([1, 3, 5]) == 0\n", "canonical_solution": "def count_evens(nums):\n    return sum(1 for n in nums if n % 2 == 0)\n", "entry_point": "count_evens"}
+{"task_id": "mbpp-004", "prompt": "Write a function gcd(a, b) that returns the greatest common divisor of two non-negative integers.", "tests": "assert gcd(12, 18) == 6\nassert gcd(7, 5) == 1\nassert gcd(100, 75) == 25\nassert gcd(0, 9) == 9\n", "canonical_solution": "def gcd(a, b):\n    while b:\n        a, b = b, a % b\n    return a\n", "entry_point": "gcd"}
+{"task_id": "mbpp-005", "prompt": "Write a function unique_sorted(items) that returns a sorted list of the unique values in items.", "tests": "assert unique_sorted([3, 1, 2, 3, 1]) == [1, 2, 3]\nassert unique_sorted([]) == []\nassert unique_sorted([5]) == [5]\nassert unique_sorted([2, 2, 2, 2]) == [2]\n", "canonical_solution": "def unique_sorted(items):\n    return sorted(set(items))\n", "entry_point": "unique_sorted"}

inferencebench_code/plugin.py

@@ -0,0 +1,532 @@
+"""CodeGenerationPlugin — entry point for ``code.generation`` benchmarks.
+
+HumanEval-style execution-based scoring: for each fixture row we send the
+function-signature prompt to the model, extract Python code from the
+response, execute it against the bundled unit tests in an isolated
+subprocess, and aggregate per-task pass/fail into a ``pass_at_1`` headline.
+
+Structural twin of :class:`inferencebench_quality.plugin.LLMQualityPlugin`:
+plugin contract, ModelClient wiring, signing flow, and envelope shape all
+mirror that module so cross-plugin tooling (summary / compare / diff /
+audit) treats code envelopes the same as quality envelopes.
+
+**Safety:** :func:`run` prints a yellow banner on every invocation as a
+reminder that model output is executed locally. See the package README for
+the full safety boundary; ``runner.py`` is best-effort defence-in-depth, not
+a sandbox.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import sys
+import time
+from pathlib import Path
+
+import yaml
+
+from inferencebench.envelope import (
+    DatasetSpec as EnvDatasetSpec,
+)
+from inferencebench.envelope import (
+    EngineConfig,
+    Envelope,
+    EnvelopeBuilder,
+    ModelConfig,
+    Quantization,
+    SigningMode,
+    sign_envelope,
+)
+from inferencebench.harness import (
+    CompletionResult,
+    ModelClient,
+    Sample,
+    collect_hardware_fingerprint,
+    collect_software_provenance,
+)
+from inferencebench.harness.metrics import EnergyReport, Percentiles, TelemetryWindow
+from inferencebench_code.runner import RunResult, run_unit_tests
+from inferencebench_code.schemas import BenchmarkSpec, EngineKind, RunContext
+from inferencebench_code.scoring import extract_python_code
+
+_SAFETY_BANNER = (
+    "\033[33m"  # yellow
+    "WARNING: code.generation executes model-generated Python locally. "
+    "The runner is best-effort (python -I subprocess with timeout + forbidden-import "
+    "pre-scan) — NOT a sandbox. Use only with trusted models and bundled fixtures."
+    "\033[0m"
+)
+
+# Engines that require ``base_url`` (self-hosted OpenAI-compatible servers).
+_SELF_HOSTED_ENGINES = frozenset({EngineKind.VLLM, EngineKind.SGLANG})
+
+
+def _fixtures_cache_root() -> Path:
+    """Resolve the bench-fixtures cache root for ``fixtures://`` dataset URIs."""
+    override = os.environ.get("BENCH_FIXTURES_ROOT")
+    if override:
+        return Path(override)
+    return Path.home() / ".cache" / "inferencebench" / "fixtures"
+
+
+def _json_num(v: float) -> str:
+    """JSON-safe numeric encoder: NaN/inf become null."""
+    if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
+        return "null"
+    return repr(v)
+
+
+def _compute_fixture_hash(items: list[dict[str, str]]) -> str:
+    """SHA-256 over the canonical-JSON-encoded fixture rows."""
+    canonical = json.dumps(items, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _build_client(context: RunContext, *, timeout_s: float = 60.0) -> ModelClient:
+    """Build a :class:`ModelClient` from the run context.
+
+    Same routing convention as the quality plugin: self-hosted OpenAI-
+    compatible engines (vLLM, SGLang) get the ``openai/`` LiteLLM prefix
+    applied exactly once; provider-hosted engines (OpenAI, Anthropic)
+    leave the model id untouched.
+    """
+    model_id = context.model_id
+    api_key: str | None
+    if context.engine_kind in _SELF_HOSTED_ENGINES:
+        if model_id.startswith("openai/"):
+            model_id = model_id[len("openai/") :]
+        model_id = f"openai/{model_id}"
+        api_key = context.api_key or "EMPTY"
+    else:
+        api_key = context.api_key or None
+    return ModelClient(
+        model=model_id,
+        api_key=api_key,
+        base_url=context.base_url or None,
+        timeout_s=timeout_s,
+    )
+
+
+def _ensure_signature_present(prompt: str, generated: str) -> str:
+    """Prepend ``prompt`` (function signature + docstring) to ``generated`` if missing.
+
+    Some models reply with the function body only; we glue the signature back
+    on so the subprocess can call the named entry point. We detect "missing
+    signature" by looking for the first non-empty line of ``prompt`` (the
+    ``def ...`` line) in ``generated``; if it isn't present we prepend.
+    """
+    head = ""
+    for line in prompt.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("def "):
+            head = stripped
+            break
+    if head and head not in generated:
+        return prompt.rstrip() + "\n" + generated
+    return generated
+
+
+# Metrics this plugin is expected to emit. Consumed by ``bench coverage``.
+EXPECTED_METRICS: tuple[str, ...] = (
+    "pass_at_1",
+    "pass_at_1_p05",
+    "pass_at_1_p50",
+    "pass_at_1_p95",
+    "timeout_rate",
+    "ok_rate",
+    "n_samples",
+    "ttft_p50_ms",
+    "total_p50_ms",
+)
+
+
+class CodeGenerationPlugin:
+    """Plugin entry point. Registered via ``inferencebench.plugins`` entrypoint group."""
+
+    suite_id = "code.generation"
+    version = "0.0.2"
+    description = (
+        "Code-generation benchmarks (HumanEval-style execution-based scoring; "
+        "executes model output locally — see README for the safety boundary)."
+    )
+
+    # ----------------------------------------------------------- benchmarks #
+    def list_benchmarks(self) -> list[BenchmarkSpec]:
+        bench_dir = self._benchmarks_dir()
+        specs: list[BenchmarkSpec] = []
+        if not bench_dir.exists():
+            return specs
+        for yml in sorted(bench_dir.glob("*.yaml")):
+            specs.append(self._load_yaml(yml))
+        return specs
+
+    def get_benchmark(self, benchmark_id: str) -> BenchmarkSpec:
+        for spec in self.list_benchmarks():
+            if spec.benchmark_id == benchmark_id:
+                return spec
+        msg = f"benchmark_id not found: {benchmark_id}"
+        raise KeyError(msg)
+
+    # ------------------------------------------------------------- validate #
+    def validate(self, spec: BenchmarkSpec, context: RunContext) -> list[str]:
+        warnings: list[str] = []
+        if not context.model_id:
+            warnings.append("model_id is empty")
+        if context.engine_kind in _SELF_HOSTED_ENGINES and not context.base_url:
+            warnings.append(
+                f"{context.engine_kind.value} needs base_url (e.g. http://localhost:8000/v1)"
+            )
+        if not self._dataset_path(spec).exists():
+            warnings.append(f"fixture not found: {spec.dataset.path}")
+        return warnings
+
+    # ------------------------------------------------------------------ run #
+    def run(self, spec: BenchmarkSpec, context: RunContext) -> Envelope:
+        """Execute the benchmark and return a SIGNED envelope.
+
+        Prints the safety banner to stderr on every call.
+        """
+        print(_SAFETY_BANNER, file=sys.stderr, flush=True)
+
+        client = _build_client(context)
+        items = self._load_fixture(spec)
+        fixture_hash = _compute_fixture_hash(items)
+
+        samples, passed_flags, timeout_flags, telemetry = self._score_items(
+            client, items, timeout_s=spec.timeout_s
+        )
+
+        # Best-effort diagnostic dump — never blocks the run on I/O errors.
+        self._dump_samples(context, samples)
+
+        envelope = self._build_envelope(
+            spec,
+            context,
+            samples=samples,
+            passed_flags=passed_flags,
+            timeout_flags=timeout_flags,
+            dataset_hash=fixture_hash,
+            energy=telemetry.summarise(samples),
+        )
+
+        signing_mode = context.extra.get("signing_mode", "dev")
+        dev_key_path = context.extra.get("dev_key_path")
+        if signing_mode == "dev":
+            if not dev_key_path:
+                msg = "dev signing requires context.extra['dev_key_path']"
+                raise ValueError(msg)
+            return sign_envelope(
+                envelope,
+                mode=SigningMode.DEV,
+                dev_key_path=Path(str(dev_key_path)),
+            )
+        return sign_envelope(envelope, mode=SigningMode.KEYLESS)
+
+    # -------------------------------------------------------- core scoring #
+    def _score_items(
+        self,
+        client: ModelClient,
+        items: list[dict[str, str]],
+        *,
+        timeout_s: float,
+    ) -> tuple[list[Sample], list[bool], list[bool], TelemetryWindow]:
+        """Iterate fixture items sequentially, scoring each model response.
+
+        For each fixture row:
+
+        1. Send the prompt to the model (max_tokens=512, streamed).
+        2. Extract a Python block from the response.
+        3. Glue the function signature back on if the model omitted it.
+        4. Execute the result against the row's unit tests with a
+           ``timeout_s`` wall clock.
+
+        Records a :class:`Sample` per row (including ``passed``,
+        ``duration_s``, ``timeout``, and an ``error_summary`` string) and
+        returns parallel lists of per-row ``passed`` and ``timeout`` flags
+        for the aggregator, plus the :class:`TelemetryWindow` for energy
+        accounting.
+        """
+        samples: list[Sample] = []
+        passed_flags: list[bool] = []
+        timeout_flags: list[bool] = []
+
+        telemetry = TelemetryWindow()
+        with telemetry:
+            for idx, item in enumerate(items):
+                prompt = item["prompt"]
+                tests = item["tests"]
+                t_arrival = time.perf_counter() * 1000.0
+                try:
+                    result: CompletionResult = client.complete(
+                        prompt, stream=True, max_tokens=512
+                    )
+                except Exception as exc:
+                    samples.append(
+                        Sample(
+                            request_idx=idx,
+                            arrival_ms=t_arrival,
+                            start_ms=t_arrival,
+                            ttft_ms=float("nan"),
+                            total_ms=float("nan"),
+                            tpot_ms=float("nan"),
+                            tokens_in=0,
+                            tokens_out=0,
+                            cost_usd=0.0,
+                            finish_reason="error",
+                            ok=False,
+                            error=str(exc),
+                        )
+                    )
+                    passed_flags.append(False)
+                    timeout_flags.append(False)
+                    continue
+
+                extracted = extract_python_code(result.text)
+                solution = _ensure_signature_present(prompt, extracted)
+                run_result: RunResult = run_unit_tests(solution, tests, timeout_s=timeout_s)
+                passed_flags.append(run_result.passed)
+                timeout_flags.append(run_result.timeout)
+
+                error_summary = self._summarize_error(run_result)
+                sample_extra: dict[str, str | int | float | bool] = {
+                    "task_id": item.get("task_id", ""),
+                    "passed": run_result.passed,
+                    "duration_s": run_result.duration_s,
+                    "timeout_flag": run_result.timeout,
+                }
+                if error_summary:
+                    sample_extra["error_summary"] = error_summary
+
+                samples.append(
+                    Sample(
+                        request_idx=idx,
+                        arrival_ms=t_arrival,
+                        start_ms=t_arrival,
+                        ttft_ms=result.ttft_ms,
+                        total_ms=result.total_ms,
+                        tpot_ms=result.tpot_ms,
+                        tokens_in=result.tokens_in,
+                        tokens_out=result.tokens_out,
+                        cost_usd=result.cost_usd,
+                        finish_reason=result.finish_reason,
+                        ok=True,
+                        extra=sample_extra,
+                    )
+                )
+        return samples, passed_flags, timeout_flags, telemetry
+
+    @staticmethod
+    def _summarize_error(result: RunResult) -> str:
+        """Distil ``RunResult`` into a short ``error_summary`` string.
+
+        Empty when the run passed. Otherwise: ``"timeout"`` for wall-clock
+        kills, ``"forbidden_import: <name>"`` for pre-scan refusals (the
+        stderr is already shaped that way by the runner), and the last
+        line of stderr (typically the AssertionError or Exception class)
+        for normal failures.
+        """
+        if result.passed:
+            return ""
+        if result.timeout:
+            return "timeout"
+        if result.stderr.startswith("forbidden_import"):
+            return result.stderr.split("\n", 1)[0]
+        last_lines = [line.strip() for line in result.stderr.strip().splitlines() if line.strip()]
+        return last_lines[-1] if last_lines else "exit_nonzero"
+
+    # ------------------------------------------------------------ samples #
+    def _dump_samples(self, context: RunContext, samples: list[Sample]) -> None:
+        """Write per-task samples (incl. pass flag + duration) to ``samples-<ts>.jsonl``.
+
+        Mirrors the llm-quality plugin's diagnostic dump — failures here
+        never block the run.
+        """
+        try:
+            out_dir = Path(context.output_dir)
+            out_dir.mkdir(parents=True, exist_ok=True)
+            ts = int(time.time())
+            path = out_dir / f"samples-{ts}.jsonl"
+            with path.open("w", encoding="utf-8") as fp:
+                for s in samples:
+                    extra = s.extra or {}
+                    passed = bool(extra.get("passed", False))
+                    duration = extra.get("duration_s")
+                    timeout_flag = bool(extra.get("timeout_flag", False))
+                    duration_part = (
+                        ',"duration_s":' + _json_num(float(duration))
+                        if isinstance(duration, (int, float))
+                        else ""
+                    )
+                    fp.write(
+                        '{"request_idx":'
+                        + str(s.request_idx)
+                        + ',"ok":'
+                        + ("true" if s.ok else "false")
+                        + ',"passed":'
+                        + ("true" if passed else "false")
+                        + ',"timeout":'
+                        + ("true" if timeout_flag else "false")
+                        + ',"ttft_ms":'
+                        + _json_num(s.ttft_ms)
+                        + ',"total_ms":'
+                        + _json_num(s.total_ms)
+                        + ',"tokens_out":'
+                        + str(s.tokens_out)
+                        + duration_part
+                        + ',"finish_reason":"'
+                        + (s.finish_reason or "")
+                        + '"'
+                        + "}\n"
+                    )
+        except OSError:
+            pass  # diagnostics-only — never block the run
+
+    # ---------------------------------------------------------- file paths #
+    def _benchmarks_dir(self) -> Path:
+        return Path(__file__).parent / "benchmarks"
+
+    def _datasets_dir(self) -> Path:
+        return Path(__file__).parent / "datasets"
+
+    def _dataset_path(self, spec: BenchmarkSpec) -> Path:
+        raw = spec.dataset.path
+        if raw.startswith("fixtures://"):
+            return _fixtures_cache_root() / f"{raw[len('fixtures://') :]}.jsonl"
+        return self._datasets_dir() / raw
+
+    def _load_yaml(self, path: Path) -> BenchmarkSpec:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        return BenchmarkSpec.model_validate(raw)
+
+    def _load_fixture(self, spec: BenchmarkSpec) -> list[dict[str, str]]:
+        path = self._dataset_path(spec)
+        if not path.exists():
+            if spec.dataset.path.startswith("fixtures://"):
+                key = spec.dataset.path[len("fixtures://") :]
+                msg = f"fixture not cached: {path}. Run `bench fixtures fetch {key}` first."
+                raise FileNotFoundError(msg)
+            msg = f"fixture not found: {path}"
+            raise FileNotFoundError(msg)
+        items: list[dict[str, str]] = []
+        with path.open("r", encoding="utf-8") as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if not isinstance(obj, dict):
+                    continue
+                if "task_id" not in obj or "prompt" not in obj or "tests" not in obj:
+                    continue
+                items.append(
+                    {
+                        "task_id": str(obj["task_id"]),
+                        "prompt": str(obj["prompt"]),
+                        "tests": str(obj["tests"]),
+                        "canonical_solution": str(obj.get("canonical_solution", "")),
+                        "entry_point": str(obj.get("entry_point", "")),
+                    }
+                )
+        if not items:
+            msg = f"fixture is empty: {path}"
+            raise ValueError(msg)
+        return items
+
+    # ---------------------------------------------------------- envelope #
+    def _build_envelope(
+        self,
+        spec: BenchmarkSpec,
+        context: RunContext,
+        *,
+        samples: list[Sample],
+        passed_flags: list[bool],
+        timeout_flags: list[bool],
+        dataset_hash: str,
+        energy: EnergyReport | None = None,
+    ) -> Envelope:
+        hw = collect_hardware_fingerprint()
+        sw = collect_software_provenance()
+
+        metrics: dict[str, float | int | str | None] = {}
+
+        ok_samples = [s for s in samples if s.ok]
+        n_total = len(samples)
+        metrics["n_samples"] = float(n_total)
+        metrics["n_ok"] = float(len(ok_samples))
+        metrics["ok_rate"] = float(len(ok_samples)) / float(n_total) if n_total else 0.0
+
+        if passed_flags:
+            scores = [1.0 if p else 0.0 for p in passed_flags]
+            mean_pass = sum(scores) / len(scores)
+            metrics["pass_at_1"] = mean_pass
+            if len(scores) >= 2:
+                pcts = Percentiles(scores, percentiles=(5.0, 50.0, 95.0))
+                metrics["pass_at_1_p05"] = pcts.p5
+                metrics["pass_at_1_p50"] = pcts.p50
+                metrics["pass_at_1_p95"] = pcts.p95
+            else:
+                metrics["pass_at_1_p05"] = mean_pass
+                metrics["pass_at_1_p50"] = mean_pass
+                metrics["pass_at_1_p95"] = mean_pass
+
+        if timeout_flags:
+            metrics["timeout_rate"] = float(sum(timeout_flags)) / float(len(timeout_flags))
+
+        # Latency aggregates — useful for "quality at what cost" comparisons.
+        ttft_vals = [s.ttft_ms for s in ok_samples if math.isfinite(s.ttft_ms)]
+        total_vals = [s.total_ms for s in ok_samples if math.isfinite(s.total_ms)]
+        if ttft_vals:
+            metrics["ttft_p50_ms"] = Percentiles(ttft_vals).p50
+        if total_vals:
+            metrics["total_p50_ms"] = Percentiles(total_vals).p50
+
+        # Energy / power summary from telemetry (None on plugins that haven't
+        # threaded a TelemetryWindow through yet). Mirrors llm-inference.
+        if energy is not None:
+            if energy.gpu_power_avg_w > 0:
+                metrics["power_avg_w"] = energy.gpu_power_avg_w
+                metrics["power_peak_w"] = energy.gpu_power_peak_w
+            if energy.total_energy_joules > 0:
+                metrics["energy_joules_total"] = energy.total_energy_joules
+                if energy.joules_per_token == energy.joules_per_token:  # not NaN
+                    metrics["joules_per_token"] = energy.joules_per_token
+
+        tokens_out_total = sum(s.tokens_out for s in ok_samples)
+        if tokens_out_total:
+            metrics["tokens_out_total"] = float(tokens_out_total)
+
+        cost_total = sum(s.cost_usd for s in ok_samples)
+        if tokens_out_total and cost_total > 0:
+            metrics["cost_usd_per_million_tokens"] = (cost_total / tokens_out_total) * 1e6
+            metrics["cost_source"] = "provider"
+
+        builder = EnvelopeBuilder(
+            suite_id=spec.benchmark_id,
+            suite_version=spec.suite_version,
+            model=ModelConfig(
+                id=context.model_id,
+                revision=context.model_revision,
+                provider=context.engine_kind.value,
+                endpoint_hash="0" * 64,
+            ),
+            engine=EngineConfig(
+                name=context.engine_kind.value,
+                version=context.engine_version or "unknown",
+                config_hash="0" * 64,
+            ),
+            hardware_fingerprint=hw,
+            software_provenance=sw,
+            dataset=EnvDatasetSpec(id=spec.dataset.id, hash=dataset_hash),
+            seed=0,
+            quantization=(
+                Quantization(format=context.quantization_format)
+                if context.quantization_format
+                else None
+            ),
+            metrics=metrics,
+            slo_template=spec.slo_template,
+        )
+        return builder.build()

inferencebench_code/runner.py

@@ -0,0 +1,144 @@
+"""Subprocess-based unit-test runner for the code-generation plugin.
+
+Executes a model-generated Python solution alongside fixture unit tests in
+an isolated ``python -I`` subprocess with a wall-clock timeout. **This is
+not a real sandbox.** See ``README.md`` for the safety boundary; the
+shortlist of forbidden-import heuristics here is defence-in-depth, not
+defence-in-full.
+
+Each :func:`run_unit_tests` invocation writes solution + tests to a
+temporary file, runs it under ``subprocess.run`` with the supplied
+``timeout_s``, captures stdout/stderr, and returns a :class:`RunResult`.
+The temp file is always deleted via ``try/finally`` even on timeout or
+unhandled errors.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+import time
+from dataclasses import dataclass
+
+# Substrings that immediately disqualify a solution. Cheap pre-scan, not
+# a parser — there are dozens of ways around it (eval, __import__, etc).
+# That's fine; this is one of several layers, and the bundled fixtures
+# never exercise stdlib edges.
+_FORBIDDEN_IMPORTS: tuple[str, ...] = (
+    "subprocess",
+    "os.system",
+    "socket",
+    "urllib",
+    "multiprocessing",
+    "ctypes",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class RunResult:
+    """Outcome of one subprocess execution.
+
+    ``passed`` is True only when the subprocess exited 0 within the wall
+    clock. ``timeout`` is True when ``subprocess.TimeoutExpired`` fired
+    (in which case ``duration_s`` reflects the timeout, not the real
+    wall time). ``stdout`` and ``stderr`` are decoded UTF-8 strings.
+    """
+
+    passed: bool
+    stdout: str
+    stderr: str
+    timeout: bool
+    duration_s: float
+
+
+def _scan_forbidden(solution: str) -> str | None:
+    """Return the first forbidden token found in ``solution`` or None.
+
+    Case-sensitive substring match; we accept the false-negative risk
+    in exchange for a vanishingly small false-positive rate.
+    """
+    for token in _FORBIDDEN_IMPORTS:
+        if token in solution:
+            return token
+    return None
+
+
+def run_unit_tests(
+    solution: str,
+    tests: str,
+    *,
+    timeout_s: float = 5.0,
+) -> RunResult:
+    """Execute ``solution + tests`` in an isolated subprocess and report pass/fail.
+
+    Arguments:
+        solution: Python code defining the function under test.
+        tests: Python code that imports / references the function (already
+            in scope because solution + tests are concatenated into one
+            file) and exercises it with ``assert`` statements.
+        timeout_s: Wall-clock budget. The subprocess is killed when it
+            elapses and :class:`RunResult` is returned with ``timeout=True``.
+
+    Returns:
+        :class:`RunResult` with ``passed``, captured streams, timeout flag,
+        and observed duration in seconds.
+    """
+    forbidden = _scan_forbidden(solution)
+    if forbidden is not None:
+        return RunResult(
+            passed=False,
+            stdout="",
+            stderr=f"forbidden_import: solution references {forbidden!r}",
+            timeout=False,
+            duration_s=0.0,
+        )
+
+    body = solution + "\n\n# --- tests ---\n" + tests + "\n"
+    # delete=False so we control the lifetime; the finally block removes it.
+    tmp = tempfile.NamedTemporaryFile(
+        mode="w",
+        suffix=".py",
+        delete=False,
+        encoding="utf-8",
+    )
+    tmp_path = tmp.name
+    try:
+        tmp.write(body)
+        tmp.close()
+        start = time.perf_counter()
+        try:
+            completed = subprocess.run(
+                [sys.executable, "-I", tmp_path],
+                capture_output=True,
+                timeout=timeout_s,
+                check=False,
+            )
+        except subprocess.TimeoutExpired as exc:
+            duration = time.perf_counter() - start
+            stdout_bytes = exc.stdout or b""
+            stderr_bytes = exc.stderr or b""
+            return RunResult(
+                passed=False,
+                stdout=stdout_bytes.decode("utf-8", errors="replace"),
+                stderr=stderr_bytes.decode("utf-8", errors="replace"),
+                timeout=True,
+                duration_s=duration,
+            )
+        duration = time.perf_counter() - start
+        stdout = completed.stdout.decode("utf-8", errors="replace")
+        stderr = completed.stderr.decode("utf-8", errors="replace")
+        return RunResult(
+            passed=completed.returncode == 0,
+            stdout=stdout,
+            stderr=stderr,
+            timeout=False,
+            duration_s=duration,
+        )
+    finally:
+        # Always remove the temp file — even on TimeoutExpired / KeyboardInterrupt.
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass

inferencebench_code/schemas.py

@@ -0,0 +1,93 @@
+"""Pydantic schemas for code-generation benchmark specs + run context."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EngineKind(StrEnum):
+    """Engines this plugin can drive.
+
+    Code-generation scoring is dominated by per-prompt API calls (one model
+    invocation per fixture row, then local execution of the response). We
+    surface the same four engine kinds the rest of the suite uses so the
+    plugin slots into existing cross-vendor comparisons unchanged.
+    """
+
+    VLLM = "vllm"
+    SGLANG = "sglang"
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+
+
+class DatasetConfig(BaseModel):
+    """Dataset under evaluation.
+
+    For the code-generation plugin the dataset is a small bundled JSONL
+    fixture; each line is one HumanEval-style task with ``task_id``,
+    ``prompt``, ``tests``, ``canonical_solution`` and ``entry_point`` keys.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    id: Annotated[str, Field(min_length=1)]
+    path: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description=("Path to the fixture JSONL relative to the plugin's datasets/ directory."),
+        ),
+    ]
+
+
+class WarmupConfig(BaseModel):
+    """Warmup parameters.
+
+    Code-generation runs are per-task and order-independent so the default
+    is zero discarded runs. Knob retained for future revisions (warm-up of
+    a JIT-compiled model or sandbox cold-start, etc.).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    discard_runs: Annotated[int, Field(ge=0)] = 0
+
+
+class BenchmarkSpec(BaseModel):
+    """One code-generation benchmark — fixture + scoring strategy + metadata."""
+
+    model_config = ConfigDict(extra="forbid")
+    benchmark_id: Annotated[str, Field(min_length=1)]
+    suite_version: Annotated[str, Field(pattern=r"^\d+\.\d+\.\d+(-[\w.]+)?$")]
+    description: str = ""
+    modality: Literal["code"] = "code"
+    kind: Literal["generation"] = "generation"
+    dataset: DatasetConfig
+    slo_template: str = "code.generation.standard"
+    warmup: WarmupConfig = Field(default_factory=WarmupConfig)
+    language: Literal["python"] = "python"
+    scoring: Literal["pass_at_1", "pass_at_k"] = "pass_at_1"
+    k: Annotated[int, Field(ge=1)] = 1
+    timeout_s: Annotated[float, Field(gt=0.0)] = 5.0
+
+
+class RunContext(BaseModel):
+    """Per-invocation context (where to send requests, where to write results).
+
+    Mirrors the llm-quality plugin so cross-plugin tooling can reuse the
+    same context object shape.
+    """
+
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+    model_id: Annotated[str, Field(min_length=1)]
+    model_revision: Annotated[str, Field(min_length=7, max_length=40)] = "unknown00"
+    engine_kind: EngineKind
+    engine_version: str = ""
+    base_url: str = ""
+    api_key: str = ""
+    quantization_format: str = ""
+    hardware_class: str = ""
+    output_dir: Path
+    extra: dict[str, str | int | float | bool] = Field(default_factory=dict)

inferencebench_code/scoring.py

@@ -0,0 +1,79 @@
+"""Pure scoring helpers for the code-generation plugin.
+
+Two helpers:
+
+* :func:`extract_python_code` extracts the first fenced ``python`` block
+  from a model response. The HumanEval prompting convention is to ask the
+  model to return code in a markdown fence; we strip the fence and return
+  the inner code. When no fence is present we treat the whole response
+  as code (some smaller models skip the fences when given a function
+  signature stub).
+
+* :func:`compute_pass_at_k` returns the unbiased pass@k estimator from
+  the HumanEval paper (Chen et al. 2021): for ``n`` samples with ``c``
+  passing, ``pass@k = 1 - C(n-c, k) / C(n, k)``. For Phase 1 we only
+  run ``k=1`` (so this collapses to the mean) but the helper ships so
+  future revisions can compute pass@10/100 from richer sampling runs.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+
+# Match a fenced python block — accept ``` python``` or ``` py ```. We
+# extract only the inner body and intentionally do not require a closing
+# newline so partial streamed responses still parse.
+_PY_FENCE = re.compile(
+    r"```(?:python|py)\s*\n(.*?)(?:```|\Z)",
+    re.IGNORECASE | re.DOTALL,
+)
+# Bare ``` fence (no language tag) — accepted as a fallback when the
+# model omits the language hint.
+_BARE_FENCE = re.compile(r"```\s*\n(.*?)(?:```|\Z)", re.DOTALL)
+
+
+def extract_python_code(text: str) -> str:
+    """Extract the first fenced ``python`` block from ``text``.
+
+    Returns the inner code with leading/trailing whitespace stripped.
+    Falls back to a bare triple-fence, then to the whole response (also
+    stripped) when no fence is present.
+
+    Multi-fence responses return the **first** block — HumanEval-style
+    prompts ask for one solution; later fences usually contain test
+    repetition or example I/O.
+    """
+    match = _PY_FENCE.search(text)
+    if match is not None:
+        return match.group(1).strip()
+    match = _BARE_FENCE.search(text)
+    if match is not None:
+        return match.group(1).strip()
+    return text.strip()
+
+
+def compute_pass_at_k(results: list[bool], k: int) -> float:
+    """Return the HumanEval-paper unbiased pass@k estimator.
+
+    Arguments:
+        results: list of per-sample pass/fail booleans (one model attempt each).
+        k: how many top samples per task we would have picked.
+
+    Formula: ``pass@k = 1 - C(n - c, k) / C(n, k)`` where ``n = len(results)``
+    and ``c = sum(results)``. When ``n - c < k`` the binomial coefficient is
+    zero and pass@k collapses to 1.0 (every k-subset must contain at least
+    one passing sample). When ``k > n`` we clip ``k = n`` — the estimator is
+    only defined for ``k <= n``.
+    """
+    n = len(results)
+    if n == 0:
+        return 0.0
+    if k < 1:
+        return 0.0
+    k = min(k, n)
+    c = sum(1 for r in results if r)
+    if n - c < k:
+        return 1.0
+    # math.comb is exact for ints; cast to float for the ratio.
+    return 1.0 - math.comb(n - c, k) / math.comb(n, k)

inferencebench_code-0.0.2.dist-info/METADATA

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: inferencebench-code
+Version: 0.0.2
+Summary: Code-generation plugin for InferenceBench Suite (HumanEval-style execution-based scoring).
+Project-URL: Homepage, https://github.com/yobitelcomm/bench
+Author-email: Yobitel Communications <bench@yobitel.com>
+License: Apache-2.0
+Keywords: ai,benchmark,code-generation,humaneval,llm,ml
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: inferencebench-envelope
+Requires-Dist: inferencebench-harness
+Requires-Dist: pydantic~=2.9
+Requires-Dist: pyyaml~=6.0
+Description-Content-Type: text/markdown
+
+# inferencebench-code
+
+Code-generation plugin for the InferenceBench Suite.
+
+HumanEval-style execution-based benchmarks: the plugin sends a function-signature
+prompt to the model, extracts the Python code from its response, executes it
+against bundled unit tests in a subprocess, and reports `pass_at_1`.
+
+Suite ID: `code.generation`
+
+Bundled benchmarks:
+
+- `code.generation.humaneval-mini` — 5 stdlib-only Python tasks, `pass_at_1`
+  scoring with a 5-second per-task wall-clock timeout.
+
+## SAFETY WARNING — read before running
+
+**This plugin executes model-generated code.** Every run prints a yellow banner
+reminding you of that. The execution layer is *best-effort* defence-in-depth,
+not a real sandbox:
+
+- Each task's solution + tests are written to a temp file and invoked with
+  `python -I` (isolated mode) under a `subprocess.run(timeout=...)` wall clock.
+- A cheap substring pre-scan refuses any solution that imports `subprocess`,
+  `os.system`, `socket`, `urllib`, `multiprocessing`, or `ctypes`.
+- The bundled fixtures are stdlib-only, no I/O, no network.
+
+This is **deliberately not airtight**. Phase 2 adds real isolation (firejail /
+nsjail / container-per-task). Until then: only run code-generation benchmarks
+against models you trust, on machines you can afford to throw away, and never
+against the bundled fixtures replaced with untrusted input.
+
+## Metrics
+
+The envelope's `metrics` block includes:
+
+| Metric             | Direction       | Meaning                                   |
+| ------------------ | --------------- | ----------------------------------------- |
+| `pass_at_1`        | higher is better | mean of per-task passed booleans          |
+| `pass_at_1_p05/50/95` | higher is better | bootstrap quantiles of per-sample scores |
+| `timeout_rate`     | lower is better  | fraction of tasks that hit the wall clock |
+| `ttft_p50_ms`      | -               | model time-to-first-token, median         |
+| `total_p50_ms`     | -               | model total request time, median          |
+| `tokens_out_total` | -               | total generated tokens across the run     |
+| `ok_rate`          | -               | fraction of model calls that succeeded    |
+| `n_samples`        | -               | fixture row count                         |

inferencebench_code-0.0.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+inferencebench_code/__init__.py,sha256=T29BdO0kbnz2t1ejan52jWySJme5kWFUIDxxTbi7-Wo,326
+inferencebench_code/plugin.py,sha256=UGJm52c4EfUawk_lHdlbwtj8aOB_LNQQm6i5FBdB0OY,21096
+inferencebench_code/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+inferencebench_code/runner.py,sha256=i2VxybIFNiIJaME1QCUz-EEVLk3GSAmKndd03nIL4ww,4759
+inferencebench_code/schemas.py,sha256=PQIAlenv2bnw1zB-1rI2HX7U0U0ljJGVIAF1q5EMvbk,3099
+inferencebench_code/scoring.py,sha256=uhfJ_zX3-sE6oFv-dAzrOLj8OhG6FwJK5SLRuP5Eyec,2968
+inferencebench_code/benchmarks/humaneval-mini.yaml,sha256=PW1_C6cK0d0FMr52gVkR_4__2YwkxHefoocygADQo1o,373
+inferencebench_code/benchmarks/mbpp-mini.yaml,sha256=DuJx0YMNpj6C0kBP9aRh4DPoUcRXsbNXDT_Fl3pcQG4,388
+inferencebench_code/datasets/humaneval-mini.jsonl,sha256=x48LbK_KWBhxzXgpy_v0UdpJlqMC3d0gxXKatOKbnoo,2455
+inferencebench_code/datasets/mbpp-mini.jsonl,sha256=5iGqpXO3K_Lkj_55gqjmrZAe6-liofdstz5weYZjTic,1944
+inferencebench_code-0.0.2.dist-info/METADATA,sha256=-8HBN4l892swGJR9JReVn_D7netgrWkpSCb7Eb-qgXE,3207
+inferencebench_code-0.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+inferencebench_code-0.0.2.dist-info/entry_points.txt,sha256=I7YPRKPTPjYYbctMhLt9NarFlxhsmQfS5lIkCZLW5_A,91
+inferencebench_code-0.0.2.dist-info/RECORD,,

inferencebench_code-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

inferencebench_code-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[inferencebench.plugins]
+code.generation = inferencebench_code.plugin:CodeGenerationPlugin