benchkit-cli 0.1.1__tar.gz → 0.2.0__tar.gz

{benchkit_cli-0.1.1/src/benchkit_cli.egg-info → benchkit_cli-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.1.1
+Version: 0.2.0
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -136,8 +136,11 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
+| CRUXEval-O | `cruxeval-o` | 800 | Predict what a short Python function returns (`-cot` for the step-by-step prompt) |
+| CRUXEval-I | `cruxeval-i` | 800 | Find an input that produces a given output, checked by running it (`-cot` too) |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |
 | IFEval | `ifeval` | 541 | Verifiable instruction following |
 | RULER | `ruler` | 39 × 6 | Practical long-context retrieval from 4K to 128K |

{benchkit_cli-0.1.1 → benchkit_cli-0.2.0}/README.md

@@ -118,8 +118,11 @@ uv run benchkit --help
 | Aider Polyglot | `aider-polyglot` | 225 | Repository editing across six languages with Pi |
 | HumanEval | `humaneval` | 164 | Python function completion |
 | HumanEval+ | `humaneval-plus` | 164 | HumanEval with expanded EvalPlus tests |
+| HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
+| CRUXEval-O | `cruxeval-o` | 800 | Predict what a short Python function returns (`-cot` for the step-by-step prompt) |
+| CRUXEval-I | `cruxeval-i` | 800 | Find an input that produces a given output, checked by running it (`-cot` too) |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |
 | IFEval | `ifeval` | 541 | Verifiable instruction following |
 | RULER | `ruler` | 39 × 6 | Practical long-context retrieval from 4K to 128K |

{benchkit_cli-0.1.1 → benchkit_cli-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchkit-cli"
-version = "0.1.1"
+version = "0.2.0"
 description = "Benchmark your local LLMs with real eval suites"
 readme = "README.md"
 requires-python = ">=3.11"

{benchkit_cli-0.1.1 → benchkit_cli-0.2.0}/src/benchkit/benchmarks/__init__.py

@@ -6,7 +6,13 @@ from benchkit.benchmarks.banking77 import Banking77
 from benchkit.benchmarks.boolq import BoolQ
 from benchkit.benchmarks.chartqa import ChartQA
 from benchkit.benchmarks.code_verifier import CodeVerifier
-from benchkit.benchmarks.evalplus import HumanEvalPlus, MBPPPlus
+from benchkit.benchmarks.cruxeval import (
+    CRUXEvalInput,
+    CRUXEvalInputCoT,
+    CRUXEvalOutput,
+    CRUXEvalOutputCoT,
+)
+from benchkit.benchmarks.evalplus import HumanEvalPlus, HumanEvalPlusTR, MBPPPlus
 from benchkit.benchmarks.git_surgery import GitSurgery
 from benchkit.benchmarks.gpqa import GPQA
 from benchkit.benchmarks.gsm8k import GSM8K
@@ -41,8 +47,13 @@ REGISTRY: dict[str, type] = {
     "treejs-arena": TreeJSArena,
     "humaneval": HumanEval,
     "humaneval-plus": HumanEvalPlus,
+    "humaneval-plus-tr": HumanEvalPlusTR,
     "mbpp": MBPP,
     "mbpp-plus": MBPPPlus,
+    "cruxeval-o": CRUXEvalOutput,
+    "cruxeval-i": CRUXEvalInput,
+    "cruxeval-o-cot": CRUXEvalOutputCoT,
+    "cruxeval-i-cot": CRUXEvalInputCoT,
     "gsm8k": GSM8K,
     "ifeval": IFEval,
     "ruler": RULER,
@@ -78,8 +89,13 @@ DESCRIPTIONS: dict[str, str] = {
     "sanity": "25 curated checks across code, math, instructions, science, and commonsense",
     "humaneval": "Python function completions with the original unit tests",
     "humaneval-plus": "HumanEval with 122k+ tougher EvalPlus test inputs",
+    "humaneval-plus-tr": "HumanEval+ fully translated to Turkish, names included",
     "mbpp": "short Python functions from natural-language specifications",
     "mbpp-plus": "sanitized MBPP tasks with 39k+ EvalPlus test inputs",
+    "cruxeval-o": "predict what a short Python function returns for a given input",
+    "cruxeval-i": "find an input that makes a short Python function return a given output",
+    "cruxeval-o-cot": "CRUXEval output prediction with the step-by-step prompt",
+    "cruxeval-i-cot": "CRUXEval input prediction with the step-by-step prompt",
     "gsm8k": "multi-step grade-school math problems with exact numeric answers",
     "ifeval": "prompts with code-checkable instruction-following constraints",
     "ruler": "practical long-context retrieval across all 13 RULER task families",

benchkit_cli-0.2.0/src/benchkit/benchmarks/cruxeval.py

@@ -0,0 +1,380 @@
+"""CRUXEval - predict what a short Python function returns, or what it was given.
+
+800 functions, each with one input and the output running it produces. The
+output task (``cruxeval-o``) shows the input and asks for the output; the input
+task (``cruxeval-i``) shows the output and asks for any input that produces it.
+Both come in the upstream direct and chain-of-thought prompts, as separate
+registry keys so the gap between them shows up as two jobs side by side.
+
+Scoring needs no judge. A predicted output is parsed with ``ast.literal_eval``
+and compared with ``==`` against the recorded output, so ``'abc'`` and
+``"abc"``, or ``(1, 2)`` written with odd spacing, all match. The one fallback:
+when the recorded output is a string and the prediction is not a literal at
+all, the bare text is compared with that string, so ``== abc`` counts for
+``'abc'``. Anything else that is not a literal - an unsimplified expression, a
+call - is wrong, which is stricter than upstream (it evaluates expressions)
+and keeps model-written code out of output scoring entirely.
+
+A predicted input is never string-matched, because many inputs can produce
+the same output: the call is run against the function in the sandbox and
+passes when it returns the recorded output.
+"""
+
+import ast
+import json
+import re
+import secrets
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.evaluation import EvaluationResult
+from benchkit.executor import execute_with_feedback
+
+DATASET = Path(__file__).parent.parent / "datasets" / "cruxeval.jsonl"
+
+# Prompts are the upstream ones from facebookresearch/cruxeval (MIT), verbatim.
+DIRECT_OUTPUT = """You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def f(n):
+    return n
+assert f(17) == ??
+[/PYTHON]
+[ANSWER]
+assert f(17) == 17
+[/ANSWER]
+
+[PYTHON]
+def f(s):
+    return s + "a"
+assert f("x9j") == ??
+[/PYTHON]
+[ANSWER]
+assert f("x9j") == "x9ja"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f({input}) == ??
+[/PYTHON]
+[ANSWER]
+"""
+
+COT_OUTPUT = """You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Execute the program step by step before arriving at an answer, and provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def f(s):
+    s = s + s
+    return "b" + s + "a"
+assert f("hi") == ??
+[/PYTHON]
+[THOUGHT]
+Let's execute the code step by step:
+
+1. The function f is defined, which takes a single argument s.
+2. The function is called with the argument "hi", so within the function, s is initially "hi".
+3. Inside the function, s is concatenated with itself, so s becomes "hihi".
+4. The function then returns a new string that starts with "b", followed by the value of s (which is now "hihi"), and ends with "a".
+5. The return value of the function is therefore "bhihia".
+[/THOUGHT]
+[ANSWER]
+assert f("hi") == "bhihia"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f({input}) == ??
+[/PYTHON]
+[THOUGHT]
+"""
+
+DIRECT_INPUT = """You will be given a function f and an output in the form f(??) == output. Find any input such that executing f on the input leads to the given output. There may be multiple answers, but you should only output one. In [ANSWER] and [/ANSWER] tags, complete the assertion with one such input that will produce the output when executing the function.
+
+[PYTHON]
+def f(my_list):
+    count = 0
+    for i in my_list:
+        if len(i) % 2 == 0:
+            count += 1
+    return count
+assert f(??) == 3
+[/PYTHON]
+[ANSWER]
+assert f(["mq", "px", "zy"]) == 3
+[/ANSWER]
+
+[PYTHON]
+def f(s1, s2):
+    return s1 + s2
+assert f(??) == "banana"
+[/PYTHON]
+[ANSWER]
+assert f("ba", "nana") == "banana"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f(??) == {output}
+[/PYTHON]
+[ANSWER]
+"""
+
+COT_INPUT = """You will be given a function f and an output in the form f(??) == output. Your task is to find any input such that executing f on the input leads to the given output. There may be multiple answers, but only output one. First, think step by step. You MUST surround the answer with [ANSWER] and [/ANSWER] tags. Express your answer as a passing assertion containing the input and the given output.
+
+[PYTHON]
+def f(x):
+    return x + 1
+assert f(??) == 17
+[/PYTHON]
+[THOUGHT]
+To find an input such that executing f on the input leads to the given output, we can work backwards from the given assertion. We know that f(??) == 17.
+
+Since the function f(x) returns x + 1, for f(??) to be equal to 17, the value of ?? should be 16.
+[/THOUGHT]
+[ANSWER]
+assert f(16) == 17
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f(??) == {output}
+[/PYTHON]
+[THOUGHT]
+"""
+
+TEMPLATES = {
+    ("output", "direct"): DIRECT_OUTPUT,
+    ("output", "cot"): COT_OUTPUT,
+    ("input", "direct"): DIRECT_INPUT,
+    ("input", "cot"): COT_INPUT,
+}
+
+NO_ASSERTION = (
+    "No assertion of the form assert f(...) == ... was found in the answer. "
+    "Reply with one such assertion inside [ANSWER] and [/ANSWER] tags."
+)
+
+
+def _answer_block(response: str) -> str:
+    """The text the model meant as its answer.
+
+    The prompt itself opens ``[ANSWER]`` (direct) or ``[THOUGHT]`` (CoT), so a
+    response may carry both tags, only the closing one, or neither. The last
+    complete block wins; a Markdown fence inside it is unwrapped.
+    """
+    text = strip_think_tags(response)
+    if "[ANSWER]" in text:
+        text = text.rsplit("[ANSWER]", 1)[1].split("[/ANSWER]", 1)[0]
+    elif "[/ANSWER]" in text:
+        text = text.rsplit("[/ANSWER]", 1)[0]
+        text = text.rsplit("[/THOUGHT]", 1)[-1]
+    fences = re.findall(r"```[\w-]*\n?(.*?)```", text, re.DOTALL)
+    if fences:
+        text = fences[-1]
+    return text.strip()
+
+
+def _assertion(block: str) -> str | None:
+    """The last ``assert`` statement in the block, if any."""
+    lines = block.splitlines()
+    for index in range(len(lines) - 1, -1, -1):
+        if lines[index].lstrip().startswith("assert"):
+            tail = "\n".join(line.strip() for line in lines[index:])
+            return tail if _parses(tail) else lines[index].strip()
+    return None
+
+
+def _parses(source: str) -> bool:
+    try:
+        ast.parse(source)
+    except SyntaxError:
+        return False
+    return True
+
+
+def _is_f_call(node: ast.expr) -> bool:
+    return (
+        isinstance(node, ast.Call)
+        and isinstance(node.func, ast.Name)
+        and node.func.id == "f"
+    )
+
+
+def _equality(source: str) -> tuple[str, ast.expr, ast.expr] | None:
+    """Split ``[assert] left == right`` into its parsed sides."""
+    text = source.strip()
+    if text.startswith("assert"):
+        text = text[len("assert") :].strip()
+    try:
+        node = ast.parse(text, mode="eval").body
+    except SyntaxError:
+        return None
+    if (
+        isinstance(node, ast.Compare)
+        and len(node.ops) == 1
+        and isinstance(node.ops[0], ast.Eq)
+    ):
+        return text, node.left, node.comparators[0]
+    return None
+
+
+def predicted_output(response: str) -> str | None:
+    """The output the model predicted, as source text."""
+    block = _answer_block(response)
+    candidate = _assertion(block) or block
+    split = _equality(candidate)
+    if split is not None:
+        text, left, right = split
+        side = left if _is_f_call(right) and not _is_f_call(left) else right
+        return ast.get_source_segment(text, side)
+    # Not parseable: a bare word after ``==`` is still a usable prediction.
+    if "==" in candidate:
+        candidate = candidate.rsplit("==", 1)[1]
+    candidate = candidate.split("#", 1)[0].strip()
+    return candidate or None
+
+
+def predicted_call(response: str) -> str | None:
+    """The ``f(...)`` call the model proposed, as source text."""
+    block = _answer_block(response)
+    candidate = _assertion(block) or block
+    split = _equality(candidate)
+    if split is not None:
+        text, left, right = split
+        for side in (left, right):
+            if _is_f_call(side):
+                return ast.get_source_segment(text, side)
+        return None
+    try:
+        node = ast.parse(candidate, mode="eval").body
+    except SyntaxError:
+        return None
+    if _is_f_call(node):
+        return ast.get_source_segment(candidate, node)
+    # A bare input, such as ``"ba", "nana"``, is the argument list itself.
+    return f"f({candidate})"
+
+
+def outputs_match(predicted: str | None, expected: str) -> bool:
+    """Literal comparison of a predicted output with the recorded one."""
+    if predicted is None:
+        return False
+    target = ast.literal_eval(expected)
+    try:
+        value = ast.literal_eval(predicted)
+    except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
+        return isinstance(target, str) and predicted.strip() == target
+    try:
+        return bool(value == target)
+    except Exception:
+        return False
+
+
+class _CRUXEval:
+    name: str
+    task: str  # "output" or "input"
+    style: str  # "direct" or "cot"
+    task_count = 800
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET, encoding="utf-8") as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["function"],
+                        metadata={
+                            "input": d["input"],
+                            "output": d["output"],
+                            # The demo client answers with this.
+                            "canonical_solution": (
+                                f"[ANSWER]\nassert f({d['input']}) == "
+                                f"{d['output']}\n[/ANSWER]"
+                            ),
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        template = TEMPLATES[(self.task, self.style)]
+        return template.format(
+            code=task.prompt,
+            input=task.metadata["input"],
+            output=task.metadata["output"],
+        )
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return self.evaluate_with_feedback(task, response).passed
+
+    def summary_fields(self, records: list[object]) -> dict:
+        """Record which half and which prompt style a score came from."""
+        return {"cruxeval_task": self.task, "cruxeval_prompt": self.style}
+
+
+class _OutputPrediction(_CRUXEval):
+    task = "output"
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        predicted = predicted_output(response)
+        if predicted is None:
+            return EvaluationResult(0.0, NO_ASSERTION)
+        if outputs_match(predicted, task.metadata["output"]):
+            return EvaluationResult(1.0)
+        return EvaluationResult(
+            0.0,
+            "The predicted output does not equal the value the function returns. "
+            "Trace the code again; the answer must be a Python literal.",
+        )
+
+
+class _InputPrediction(_CRUXEval):
+    task = "input"
+    evaluation_activity = "running the predicted input"
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        call = predicted_call(response)
+        if call is None:
+            return EvaluationResult(0.0, NO_ASSERTION)
+        # Same check as upstream, ``assert output == f(input)``. The input is
+        # model-written code, so a pass also needs a fresh sentinel printed
+        # after the assertion: an input that exits early cannot fake one.
+        sentinel = secrets.token_hex(16)
+        program = (
+            f"{task.prompt}\n\n"
+            f"assert {task.metadata['output']} == {call}\n"
+            f"print({sentinel!r})\n"
+        )
+        result = execute_with_feedback(program, timeout=5, sentinel=sentinel)
+        if result.passed:
+            return EvaluationResult(1.0)
+        if "assertion failed" in result.feedback:
+            return EvaluationResult(
+                0.0,
+                "The proposed input ran, but the function returned a different "
+                "value than the given output.",
+            )
+        return EvaluationResult(0.0, result.feedback)
+
+
+class CRUXEvalOutput(_OutputPrediction):
+    name = "cruxeval-o"
+    style = "direct"
+
+
+class CRUXEvalOutputCoT(_OutputPrediction):
+    name = "cruxeval-o-cot"
+    style = "cot"
+
+
+class CRUXEvalInput(_InputPrediction):
+    name = "cruxeval-i"
+    style = "direct"
+
+
+class CRUXEvalInputCoT(_InputPrediction):
+    name = "cruxeval-i-cot"
+    style = "cot"

{benchkit_cli-0.1.1 → benchkit_cli-0.2.0}/src/benchkit/benchmarks/downloads.py

@@ -164,3 +164,14 @@ MMLU_PRO = RemoteFile(
     sha256="92ec532d9e2bae342ec7256005c20b6e5defbed76504280df775fb77068f077e",
     size=10_260_787,
 )
+
+# Pinned to a dataset commit on the Hub; the sha256 covers the exact bytes.
+HUMANEVAL_PLUS_TR = RemoteFile(
+    name="humanevalplus_tr.jsonl",
+    url=(
+        "https://huggingface.co/datasets/DogukanUrker/HumanEvalPlus-TR/resolve/"
+        "e424171b8934581819fcdeddace2af28a664966f/humanevalplus_tr.jsonl"
+    ),
+    sha256="072172a4d7934b8ce27d8ad0c83b139b80296876bd2f8a8afad1476bdaa97955",
+    size=7_635_249,
+)

{benchkit_cli-0.1.1 → benchkit_cli-0.2.0}/src/benchkit/benchmarks/evalplus.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import contextlib
+import json
 import multiprocessing
 import os
 import platform
@@ -14,7 +15,7 @@ from pathlib import Path
 from typing import Any
 
 from benchkit.benchmarks.base import Task
-from benchkit.benchmarks.downloads import Dataset
+from benchkit.benchmarks.downloads import HUMANEVAL_PLUS_TR, Dataset
 from benchkit.benchmarks.utils import strip_think_tags
 from benchkit.executor import sandbox_env
 
@@ -23,6 +24,12 @@ HUMANEVAL_SYSTEM = (
     "Output ONLY the function body. "
     "No markdown fences, no explanation, no examples."
 )
+# Full Turkish translation, ASCII only like the dataset itself.
+HUMANEVAL_TR_SYSTEM = (
+    "Asagidaki Python fonksiyonunu tamamla. "
+    "YALNIZCA fonksiyon govdesini yaz. "
+    "Markdown blogu, aciklama veya ornek ekleme."
+)
 MBPP_SYSTEM = (
     "Write a Python function to solve the following task. "
     "Output ONLY the function. "
@@ -118,9 +125,14 @@ def _evalplus_api():
     )
 
 
-@lru_cache(maxsize=2)
+@lru_cache(maxsize=3)
 def _problems(dataset: str) -> dict[str, dict[str, Any]]:
     get_human_eval_plus, get_mbpp_plus, *_ = _evalplus_api()
+    if dataset == "humaneval-tr":
+        # Same schema as HumanEvalPlus.jsonl: translated prompt, entry point,
+        # canonical solution and contract, original inputs and tolerances.
+        rows = HUMANEVAL_PLUS_TR.ensure().read_text(encoding="utf-8").splitlines()
+        return {row["task_id"]: row for row in map(json.loads, filter(None, rows))}
     if dataset == "humaneval":
         return get_human_eval_plus()
     if dataset == "mbpp":
@@ -275,14 +287,14 @@ class _EvalPlusBenchmark:
     def evaluate(self, task: Task, response: str) -> bool:
         *_, pass_status, _, check_correctness, _ = _evalplus_api()
         problem = _problems(self.dataset)[task.id]
-        if self.dataset == "humaneval":
+        if self.dataset.startswith("humaneval"):
             solution = _human_solution(problem, response)
         else:
             solution = _extract_code(response)
 
         with _fork_multiprocessing():
             result = check_correctness(
-                self.dataset,
+                "mbpp" if self.dataset == "mbpp" else "humaneval",
                 completion_id=0,
                 problem=problem,
                 solution=solution,
@@ -302,6 +314,16 @@ class HumanEvalPlus(_EvalPlusBenchmark):
     )
 
 
+class HumanEvalPlusTR(_EvalPlusBenchmark):
+    name = "humaneval-plus-tr"
+    dataset = "humaneval-tr"
+    task_count = 164
+    system = HUMANEVAL_TR_SYSTEM
+    data_source = HUMANEVAL_PLUS_TR.source(
+        "HumanEval+ fully translated to Turkish (docstrings and identifiers)"
+    )
+
+
 class MBPPPlus(_EvalPlusBenchmark):
     name = "mbpp-plus"
     dataset = "mbpp"