human-eval-rust 2.1.0__py3-none-any.whl

data/HumanEval_rust_extended.jsonl

@@ -0,0 +1,2 @@
+{"task_id": "Extended/0", "prompt": "fn identity(x: i32) -> i32 {", "canonical_solution": "    x\n}\n", "test": "#[test] fn test_identity() { assert_eq!(identity(5), 5); }", "entry_point": "identity"}
+{"task_id": "Extended/1", "prompt": "fn sum_slice(values: &[i32]) -> i32 {", "canonical_solution": "    values.iter().sum()\n}\n", "test": "#[test] fn test_sum() { assert_eq!(sum_slice(&[1,2,3]), 6); }", "entry_point": "sum_slice"}

data/example_rust_problem.jsonl

@@ -0,0 +1 @@
+{"task_id": "test_rust/0", "prompt": "fn return_one() -> i32 {\n", "canonical_solution": "    1\n}\n", "test": "\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn test_return_one() {\n        assert_eq!(return_one(), 1);\n    }\n}\n", "entry_point": "return_one"}

data/example_rust_samples.jsonl

@@ -0,0 +1,4 @@
+{"task_id": "test_rust/0", "completion": "    1\n}\n"}
+{"task_id": "test_rust/0", "completion": "    0\n}\n"}
+{"task_id": "test_rust/0", "completion": "    2\n}\n"}
+{"task_id": "test_rust/0", "completion": "    let mut value = 0;\n    for _ in 0..1 {\n        value += 1;\n    }\n    value\n}\n"}

human_eval/__init__.py

@@ -0,0 +1,23 @@
+"""
+HumanEval Rust evaluation package.
+
+Provides evaluation harness for the HumanEval Rust problem solving dataset.
+
+Copyright (c) 2025 Dave Tofflemire, SigilDERG Project
+Version: 2.1.0
+"""
+
+__version__ = "2.1.0"
+
+# Export rust_execution module so it can be imported
+# Use relative import to avoid circular dependency issues
+from . import rust_execution
+
+
+class EvaluationError(Exception):
+    """Raised when evaluation cannot proceed due to data issues."""
+
+    pass
+
+
+__all__ = ["rust_execution", "__version__", "EvaluationError"]

human_eval/data.py

@@ -0,0 +1,74 @@
+"""
+Data loading utilities for HumanEval Rust dataset.
+
+Provides functions to read and write JSONL files containing problems and completions.
+
+Copyright (c) 2025 Dave Tofflemire, SigilDERG Project
+Version: 2.1.0
+"""
+
+import gzip
+import importlib.resources
+import json
+import os
+from collections.abc import Iterable
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL_RUST = os.path.join(ROOT, "..", "data", "HumanEval_rust.jsonl")
+
+
+def get_human_eval_dataset(language: str | None = None) -> str:
+    """Returns path to HumanEval dataset, using importlib.resources."""
+
+    if language and language.lower() != "rust":
+        raise ValueError(f"Only Rust is supported. Got language: {language}")
+
+    with importlib.resources.as_file(
+        importlib.resources.files("human_eval").joinpath("../data/HumanEval_rust.jsonl")
+    ) as path:
+        return str(path)
+
+
+def read_problems(evalset_file: str | None = None) -> dict[str, dict]:
+    """Reads problems from the specified file, or defaults to the Rust dataset."""
+
+    if evalset_file is None:
+        evalset_file = get_human_eval_dataset()
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, "rt", encoding="utf-8") as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r", encoding="utf-8") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = "ab"
+    else:
+        mode = "wb"
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode="wb") as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode("utf-8"))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode("utf-8"))

human_eval/evaluate_functional_correctness.py

@@ -0,0 +1,112 @@
+"""
+Command-line entry point for HumanEval Rust functional correctness evaluation.
+
+Provides CLI interface using Fire for evaluating Rust code completions.
+
+Copyright (c) 2025 Dave Tofflemire, SigilDERG Project
+Version: 2.1.0
+"""
+
+import sys
+
+import fire
+
+from human_eval.data import get_human_eval_dataset
+from human_eval.evaluation import evaluate_functional_correctness
+
+
+def entry_point(
+    sample_file: str,
+    k: str = "1,10,100",
+    n_workers: int = 24,  # Optimized for H100: 24 workers (26 vCPUs - 2 reserved)
+    timeout: float = 10.0,  # Optimized for H100: 10s timeout (was 3.0s) for compilation latency
+    problem_file: str | None = None,
+    language: str | None = None,
+    sandbox_mode: str | None = None,
+    allow_no_sandbox: bool = False,
+    enforce_policy: bool = True,
+):
+    """
+    Evaluate HumanEval Rust completions and write a "<input>_results.jsonl"
+    file containing pass/fail metadata.
+
+    Arguments:
+      sample_file: Path to a JSONL file containing Rust completions with
+        `task_id` and `completion` fields.
+      k: Comma-separated list of pass@k values to compute (e.g. "1,10,100").
+      n_workers: Number of parallel workers to use when running tests.
+      timeout: Per-sample timeout in seconds for compilation/execution.
+      problem_file: Optional dataset override. If omitted, defaults to the
+        Rust HumanEval dataset.
+      language: Kept for API compatibility but only "rust" is supported.
+        If not provided, defaults to "rust".
+      sandbox_mode: Sandbox mode ("firejail" or "none").
+        - firejail (recommended): Uses Firejail for Linux process isolation
+        - none: No sandboxing (UNSAFE - only for local dev with trusted code)
+        If not specified, auto-detects Firejail availability.
+      allow_no_sandbox: Allow proceeding without sandbox in non-interactive mode.
+        Use with --sandbox-mode=none or when Firejail is unavailable.
+        Required for automated pipelines that accept unsandboxed execution.
+      enforce_policy: Whether to enforce pattern-based policy filtering (default: True).
+        Set to False for pure HumanEval compatibility without security filtering.
+        Use --no-enforce-policy to disable policy enforcement.
+    """
+    k_list: list[int] = list(map(int, k.split(",")))
+    if problem_file is None:
+        problem_file = get_human_eval_dataset(language)
+
+    # Resolve sandbox mode with user interaction if needed
+    try:
+        from human_eval.sandbox import check_firejail_available, resolve_sandbox_mode
+
+        # Determine if we're in interactive mode (stdin is a TTY)
+        non_interactive = not sys.stdin.isatty()
+
+        resolved_mode = resolve_sandbox_mode(
+            sandbox_mode=sandbox_mode,
+            allow_no_sandbox=allow_no_sandbox,
+            non_interactive=non_interactive,
+        )
+
+        if resolved_mode == "firejail":
+            status = check_firejail_available()
+            print(f"Using Firejail sandboxing ({status.version})", file=sys.stderr)
+        elif resolved_mode == "none":
+            if not allow_no_sandbox:
+                print(
+                    "⚠ WARNING: Running without sandbox. This is UNSAFE for untrusted code!",
+                    file=sys.stderr,
+                )
+
+        sandbox_mode = resolved_mode
+
+    except ImportError:
+        # Sandbox module not available
+        sandbox_mode = "none"
+        print(
+            "WARNING: Sandbox module not available, running without sandboxing",
+            file=sys.stderr,
+        )
+    except SystemExit:
+        # User cancelled the prompt
+        raise
+
+    results = evaluate_functional_correctness(
+        sample_file,
+        k_list,
+        n_workers,
+        timeout,
+        problem_file,
+        language,
+        sandbox_mode,
+        enforce_policy,
+    )
+    print(results)
+
+
+def main():
+    fire.Fire(entry_point)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

human_eval/evaluation.py

@@ -0,0 +1,281 @@
+"""
+Functional correctness evaluation for HumanEval Rust completions.
+
+Implements pass@k estimation and parallel test execution.
+
+Copyright (c) 2025 Dave Tofflemire, SigilDERG Project
+Version: 2.1.0
+"""
+
+import itertools
+import subprocess
+from collections import Counter, defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import numpy as np
+import tqdm
+
+from human_eval import EvaluationError
+from human_eval.data import (
+    get_human_eval_dataset,
+    read_problems,
+    stream_jsonl,
+    write_jsonl,
+)
+from human_eval.execution import check_correctness
+from human_eval.rust_execution import check_main_free
+
+
+def estimate_pass_at_k(
+    num_samples: int | list[int] | np.ndarray,
+    num_correct: list[int] | np.ndarray,
+    k: int,
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return float(1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+    )
+
+
+def _resolve_language(language: str | None, problem_file: str) -> str:
+    """
+    Resolves the language for evaluation. Only Rust is supported.
+    """
+    if language and language.lower() != "rust":
+        raise ValueError(
+            f"Only Rust is supported. Got language: {language}. "
+            "This evaluator only supports Rust code evaluation."
+        )
+    return "rust"
+
+
+def _get_rustc_version() -> str:
+    result = subprocess.run(["rustc", "--version"], capture_output=True, text=True)
+    return result.stdout.strip() if result.returncode == 0 else "unknown"
+
+
+def evaluate_functional_correctness(
+    sample_file: str,
+    k: list[int] = [1, 10, 100],
+    n_workers: int = 4,
+    timeout: float = 3.0,
+    problem_file: str | None = None,
+    language: str | None = None,
+    sandbox_mode: str | None = None,
+    enforce_policy: bool = True,
+):
+    """
+    Evaluates the functional correctness of generated samples, and writes
+    results to f"{sample_file}_results.jsonl" (one JSON object per sample result).
+
+    Args:
+        sample_file: Path to JSONL file with completions
+        k: List of pass@k values to compute
+        n_workers: Number of parallel workers
+        timeout: Per-sample timeout in seconds
+        problem_file: Optional problem dataset file
+        language: Language (only "rust" supported)
+        sandbox_mode: Sandbox mode ("firejail", "none", or None for auto-detect)
+        enforce_policy: Whether to enforce pattern-based policy filtering (default: True).
+            Set to False for pure HumanEval compatibility without security filtering.
+    """
+
+    if problem_file is None:
+        problem_file = get_human_eval_dataset(language or "rust")
+
+    resolved_language = _resolve_language(language, problem_file)
+
+    problems = read_problems(problem_file)
+
+    # Check the generated samples against test suites.
+    with ThreadPoolExecutor(max_workers=n_workers) as executor:
+
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        print("Reading samples...")
+        for sample in tqdm.tqdm(stream_jsonl(sample_file)):
+            task_id = sample["task_id"]
+            problem = problems.get(task_id)
+
+            if problem is None:
+                raise KeyError(f"Unknown task_id '{task_id}' in {sample_file}.")
+
+            completion = sample["completion"]
+            args = (
+                problem,
+                completion,
+                timeout,
+                completion_id[task_id],
+                resolved_language,
+                sandbox_mode,
+                enforce_policy,
+            )
+            future = executor.submit(check_correctness, *args)
+            futures.append(future)
+            completion_id[task_id] += 1
+            n_samples += 1
+
+        if len(completion_id) != len(problems):
+            missing = set(problems.keys()) - set(completion_id.keys())
+            raise EvaluationError(
+                f"Missing completions for {len(missing)} problems: {list(missing)[:5]}..."
+            )
+
+        print("Running test suites...")
+        all_results_list = []
+        for future in tqdm.tqdm(as_completed(futures), total=len(futures)):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+            all_results_list.append(result)
+
+    # Track compile rate and main-free rate
+    compile_ok_count = sum(1 for r in all_results_list if r.get("compile_ok") is True)
+    compile_total = sum(1 for r in all_results_list if r.get("compile_ok") is not None)
+    compile_rate = compile_ok_count / compile_total if compile_total > 0 else 0.0
+
+    main_free_count = sum(1 for r in all_results_list if r.get("main_free") is True)
+    main_free_rate = (
+        main_free_count / len(all_results_list) if all_results_list else 0.0
+    )
+
+    clippy_total = sum(1 for r in all_results_list if r.get("clippy_ok") is not None)
+    clippy_pass = sum(1 for r in all_results_list if r.get("clippy_ok") is True)
+    clippy_rate = clippy_pass / clippy_total if clippy_total else 0.0
+
+    compile_times = [
+        r.get("compile_time_ms")
+        for r in all_results_list
+        if r.get("compile_time_ms") is not None
+    ]
+    binary_sizes = [
+        r.get("binary_size_bytes")
+        for r in all_results_list
+        if r.get("binary_size_bytes") is not None
+    ]
+
+    # Calculate pass@k.
+    total, correct = [], []
+    for result in results.values():
+        result.sort()
+        passed = [r[1]["passed"] for r in result]
+        total.append(len(passed))
+        correct.append(sum(passed))
+    total = np.array(total)
+    correct = np.array(correct)
+
+    ks = k
+    pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+        for k in ks
+        if (total >= k).all()
+    }
+
+    pass_at_k["compile_rate"] = compile_rate
+    pass_at_k["main_free_rate"] = main_free_rate
+    pass_at_k["clippy_pass_rate"] = clippy_rate
+    pass_at_k["avg_compile_time_ms"] = (
+        float(np.mean(compile_times)) if compile_times else 0.0
+    )
+    pass_at_k["avg_binary_size_bytes"] = (
+        float(np.mean(binary_sizes)) if binary_sizes else 0.0
+    )
+    pass_at_k["rustc_version"] = _get_rustc_version()
+
+    # Print metrics
+    print("\nMetrics:")
+    print(f"  Compile rate: {compile_rate:.4f} ({compile_rate * 100:.2f}%)")
+    print(f"  Main-free rate: {main_free_rate:.4f} ({main_free_rate * 100:.2f}%)")
+    print(f"  Clippy pass rate: {clippy_rate:.4f} ({clippy_rate * 100:.2f}%)")
+    if compile_times:
+        print(f"  Avg compile time (ms): {np.mean(compile_times):.2f}")
+    if binary_sizes:
+        print(f"  Avg binary size (bytes): {np.mean(binary_sizes):.2f}")
+    for metric, value in sorted(pass_at_k.items()):
+        if metric not in (
+            "compile_rate",
+            "main_free_rate",
+            "clippy_pass_rate",
+            "avg_compile_time_ms",
+            "avg_binary_size_bytes",
+            "rustc_version",
+        ):
+            print(f"  {metric}: {value:.4f} ({value * 100:.2f}%)")
+    print(f"  rustc: {pass_at_k['rustc_version']}")
+
+    # Finally, save the results in one file:
+    # Writes to "<sample_file>_results.jsonl" (one JSON object per sample result)
+    # Ensure all completions are included (never drop silently)
+    def combine_results():
+        # Read all samples to ensure we don't miss any
+        samples_by_task = defaultdict(list)
+        for sample in stream_jsonl(sample_file):
+            samples_by_task[sample["task_id"]].append(sample)
+
+        # Match results with samples
+        for task_id in sorted(samples_by_task.keys()):
+            task_samples = samples_by_task[task_id]
+            task_results = results.get(task_id, [])
+            task_results.sort()
+
+            # Ensure we have a result for every sample
+            for i, sample in enumerate(task_samples):
+                if i < len(task_results):
+                    result = task_results[i][1]
+                    sample.update(
+                        {
+                            "compile_ok": result.get("compile_ok"),
+                            "test_ok": result.get("test_ok"),
+                            "clippy_ok": result.get("clippy_ok"),
+                            "compile_time_ms": result.get("compile_time_ms"),
+                            "binary_size_bytes": result.get("binary_size_bytes"),
+                            "error_type": result.get("error_type"),
+                            "stderr": result.get("stderr", ""),
+                            "main_free": result.get("main_free"),
+                            "result": result.get("result", ""),
+                            "passed": result.get("passed", False),
+                        }
+                    )
+                else:
+                    # Missing result - create placeholder (never drop silently)
+                    sample.update(
+                        {
+                            "compile_ok": None,
+                            "test_ok": None,
+                            "clippy_ok": None,
+                            "compile_time_ms": None,
+                            "binary_size_bytes": None,
+                            "error_type": "runtime_error",
+                            "stderr": "missing result",
+                            "main_free": check_main_free(sample.get("completion", "")),
+                            "result": "filtered: missing result",
+                            "passed": False,
+                        }
+                    )
+                assert sample is not None
+                yield sample
+
+    out_file = sample_file + "_results.jsonl"
+    print(f"Writing results to {out_file}...")
+    write_jsonl(out_file, tqdm.tqdm(combine_results(), total=n_samples))
+
+    return pass_at_k

human_eval/execution.py

@@ -0,0 +1,186 @@
+"""
+Execution utilities for HumanEval Rust evaluation.
+
+Provides timeout handling, reliability guards, and correctness checking.
+
+Copyright (c) 2025 Dave Tofflemire, SigilDERG Project
+Version: 2.1.0
+"""
+
+# pyright: reportAttributeAccessIssue=false, reportArgumentType=false
+
+import contextlib
+import faulthandler
+import os
+import platform
+import tempfile
+import threading
+
+
+def check_correctness(
+    problem: dict,
+    completion: str,
+    timeout: float,
+    completion_id: int | None = None,
+    language: str | None = None,
+    sandbox_mode: str | None = None,
+    enforce_policy: bool = True,
+) -> dict:
+    """
+    Evaluates the functional correctness of a Rust completion by compiling
+    and running the test suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    :param sandbox_mode: Optional sandbox mode ("docker", "firejail", "none", or None for auto-detect)
+    :param enforce_policy: Whether to enforce pattern-based policy filtering (default: True).
+        Set to False for pure HumanEval compatibility without security filtering.
+    """
+    # Import here to avoid circular import (rust_execution imports from execution)
+    from human_eval import rust_execution
+
+    # Language parameter is kept for API compatibility but only Rust is supported
+    if language and language.lower() != "rust":
+        raise ValueError(f"Only Rust is supported. Got language: {language}")
+
+    return rust_execution.rust_check_correctness(
+        problem, completion, timeout, completion_id, sandbox_mode, enforce_policy
+    )
+
+
+@contextlib.contextmanager
+def time_limit(seconds: float):
+    """Thread-safe timeout context manager using Timer."""
+
+    timed_out = threading.Event()
+
+    def timeout_handler():
+        timed_out.set()
+
+    timer = threading.Timer(seconds, timeout_handler)
+    timer.start()
+    try:
+        yield timed_out
+    finally:
+        if timer:
+            timer.cancel()
+
+    if timed_out.is_set():
+        raise TimeoutException("Timed out!")
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    original_unlink = os.unlink
+    original_rmdir = os.rmdir
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+        os.unlink = original_unlink
+        os.rmdir = original_rmdir
+
+
+class TimeoutException(Exception):
+    pass
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: int | None = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(
+            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        resource.setrlimit(
+            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(
+                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
+            )
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None