benchkit-cli 0.1.2__tar.gz → 0.2.0__tar.gz

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchkit-cli
-Version: 0.1.2
+Version: 0.2.0
 Summary: Benchmark your local LLMs with real eval suites
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -139,6 +139,8 @@ uv run benchkit --help
 | HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
+| CRUXEval-O | `cruxeval-o` | 800 | Predict what a short Python function returns (`-cot` for the step-by-step prompt) |
+| CRUXEval-I | `cruxeval-i` | 800 | Find an input that produces a given output, checked by running it (`-cot` too) |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |
 | IFEval | `ifeval` | 541 | Verifiable instruction following |
 | RULER | `ruler` | 39 × 6 | Practical long-context retrieval from 4K to 128K |

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/README.md

@@ -121,6 +121,8 @@ uv run benchkit --help
 | HumanEval+ TR | `humaneval-plus-tr` | 164 | HumanEval+ fully translated to Turkish, function and variable names included |
 | MBPP | `mbpp` | 500 | Short Python programming tasks |
 | MBPP+ | `mbpp-plus` | 378 | MBPP with expanded EvalPlus tests |
+| CRUXEval-O | `cruxeval-o` | 800 | Predict what a short Python function returns (`-cot` for the step-by-step prompt) |
+| CRUXEval-I | `cruxeval-i` | 800 | Find an input that produces a given output, checked by running it (`-cot` too) |
 | GSM8K | `gsm8k` | 1,319 | Multi-step grade-school math |
 | IFEval | `ifeval` | 541 | Verifiable instruction following |
 | RULER | `ruler` | 39 × 6 | Practical long-context retrieval from 4K to 128K |

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchkit-cli"
-version = "0.1.2"
+version = "0.2.0"
 description = "Benchmark your local LLMs with real eval suites"
 readme = "README.md"
 requires-python = ">=3.11"

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/src/benchkit/benchmarks/__init__.py

@@ -6,6 +6,12 @@ from benchkit.benchmarks.banking77 import Banking77
 from benchkit.benchmarks.boolq import BoolQ
 from benchkit.benchmarks.chartqa import ChartQA
 from benchkit.benchmarks.code_verifier import CodeVerifier
+from benchkit.benchmarks.cruxeval import (
+    CRUXEvalInput,
+    CRUXEvalInputCoT,
+    CRUXEvalOutput,
+    CRUXEvalOutputCoT,
+)
 from benchkit.benchmarks.evalplus import HumanEvalPlus, HumanEvalPlusTR, MBPPPlus
 from benchkit.benchmarks.git_surgery import GitSurgery
 from benchkit.benchmarks.gpqa import GPQA
@@ -44,6 +50,10 @@ REGISTRY: dict[str, type] = {
     "humaneval-plus-tr": HumanEvalPlusTR,
     "mbpp": MBPP,
     "mbpp-plus": MBPPPlus,
+    "cruxeval-o": CRUXEvalOutput,
+    "cruxeval-i": CRUXEvalInput,
+    "cruxeval-o-cot": CRUXEvalOutputCoT,
+    "cruxeval-i-cot": CRUXEvalInputCoT,
     "gsm8k": GSM8K,
     "ifeval": IFEval,
     "ruler": RULER,
@@ -82,6 +92,10 @@ DESCRIPTIONS: dict[str, str] = {
     "humaneval-plus-tr": "HumanEval+ fully translated to Turkish, names included",
     "mbpp": "short Python functions from natural-language specifications",
     "mbpp-plus": "sanitized MBPP tasks with 39k+ EvalPlus test inputs",
+    "cruxeval-o": "predict what a short Python function returns for a given input",
+    "cruxeval-i": "find an input that makes a short Python function return a given output",
+    "cruxeval-o-cot": "CRUXEval output prediction with the step-by-step prompt",
+    "cruxeval-i-cot": "CRUXEval input prediction with the step-by-step prompt",
     "gsm8k": "multi-step grade-school math problems with exact numeric answers",
     "ifeval": "prompts with code-checkable instruction-following constraints",
     "ruler": "practical long-context retrieval across all 13 RULER task families",

benchkit_cli-0.2.0/src/benchkit/benchmarks/cruxeval.py

@@ -0,0 +1,380 @@
+"""CRUXEval - predict what a short Python function returns, or what it was given.
+
+800 functions, each with one input and the output running it produces. The
+output task (``cruxeval-o``) shows the input and asks for the output; the input
+task (``cruxeval-i``) shows the output and asks for any input that produces it.
+Both come in the upstream direct and chain-of-thought prompts, as separate
+registry keys so the gap between them shows up as two jobs side by side.
+
+Scoring needs no judge. A predicted output is parsed with ``ast.literal_eval``
+and compared with ``==`` against the recorded output, so ``'abc'`` and
+``"abc"``, or ``(1, 2)`` written with odd spacing, all match. The one fallback:
+when the recorded output is a string and the prediction is not a literal at
+all, the bare text is compared with that string, so ``== abc`` counts for
+``'abc'``. Anything else that is not a literal - an unsimplified expression, a
+call - is wrong, which is stricter than upstream (it evaluates expressions)
+and keeps model-written code out of output scoring entirely.
+
+A predicted input is never string-matched, because many inputs can produce
+the same output: the call is run against the function in the sandbox and
+passes when it returns the recorded output.
+"""
+
+import ast
+import json
+import re
+import secrets
+from pathlib import Path
+
+from benchkit.benchmarks.base import Task
+from benchkit.benchmarks.utils import strip_think_tags
+from benchkit.evaluation import EvaluationResult
+from benchkit.executor import execute_with_feedback
+
+DATASET = Path(__file__).parent.parent / "datasets" / "cruxeval.jsonl"
+
+# Prompts are the upstream ones from facebookresearch/cruxeval (MIT), verbatim.
+DIRECT_OUTPUT = """You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def f(n):
+    return n
+assert f(17) == ??
+[/PYTHON]
+[ANSWER]
+assert f(17) == 17
+[/ANSWER]
+
+[PYTHON]
+def f(s):
+    return s + "a"
+assert f("x9j") == ??
+[/PYTHON]
+[ANSWER]
+assert f("x9j") == "x9ja"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f({input}) == ??
+[/PYTHON]
+[ANSWER]
+"""
+
+COT_OUTPUT = """You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Execute the program step by step before arriving at an answer, and provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def f(s):
+    s = s + s
+    return "b" + s + "a"
+assert f("hi") == ??
+[/PYTHON]
+[THOUGHT]
+Let's execute the code step by step:
+
+1. The function f is defined, which takes a single argument s.
+2. The function is called with the argument "hi", so within the function, s is initially "hi".
+3. Inside the function, s is concatenated with itself, so s becomes "hihi".
+4. The function then returns a new string that starts with "b", followed by the value of s (which is now "hihi"), and ends with "a".
+5. The return value of the function is therefore "bhihia".
+[/THOUGHT]
+[ANSWER]
+assert f("hi") == "bhihia"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f({input}) == ??
+[/PYTHON]
+[THOUGHT]
+"""
+
+DIRECT_INPUT = """You will be given a function f and an output in the form f(??) == output. Find any input such that executing f on the input leads to the given output. There may be multiple answers, but you should only output one. In [ANSWER] and [/ANSWER] tags, complete the assertion with one such input that will produce the output when executing the function.
+
+[PYTHON]
+def f(my_list):
+    count = 0
+    for i in my_list:
+        if len(i) % 2 == 0:
+            count += 1
+    return count
+assert f(??) == 3
+[/PYTHON]
+[ANSWER]
+assert f(["mq", "px", "zy"]) == 3
+[/ANSWER]
+
+[PYTHON]
+def f(s1, s2):
+    return s1 + s2
+assert f(??) == "banana"
+[/PYTHON]
+[ANSWER]
+assert f("ba", "nana") == "banana"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f(??) == {output}
+[/PYTHON]
+[ANSWER]
+"""
+
+COT_INPUT = """You will be given a function f and an output in the form f(??) == output. Your task is to find any input such that executing f on the input leads to the given output. There may be multiple answers, but only output one. First, think step by step. You MUST surround the answer with [ANSWER] and [/ANSWER] tags. Express your answer as a passing assertion containing the input and the given output.
+
+[PYTHON]
+def f(x):
+    return x + 1
+assert f(??) == 17
+[/PYTHON]
+[THOUGHT]
+To find an input such that executing f on the input leads to the given output, we can work backwards from the given assertion. We know that f(??) == 17.
+
+Since the function f(x) returns x + 1, for f(??) to be equal to 17, the value of ?? should be 16.
+[/THOUGHT]
+[ANSWER]
+assert f(16) == 17
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert f(??) == {output}
+[/PYTHON]
+[THOUGHT]
+"""
+
+TEMPLATES = {
+    ("output", "direct"): DIRECT_OUTPUT,
+    ("output", "cot"): COT_OUTPUT,
+    ("input", "direct"): DIRECT_INPUT,
+    ("input", "cot"): COT_INPUT,
+}
+
+NO_ASSERTION = (
+    "No assertion of the form assert f(...) == ... was found in the answer. "
+    "Reply with one such assertion inside [ANSWER] and [/ANSWER] tags."
+)
+
+
+def _answer_block(response: str) -> str:
+    """The text the model meant as its answer.
+
+    The prompt itself opens ``[ANSWER]`` (direct) or ``[THOUGHT]`` (CoT), so a
+    response may carry both tags, only the closing one, or neither. The last
+    complete block wins; a Markdown fence inside it is unwrapped.
+    """
+    text = strip_think_tags(response)
+    if "[ANSWER]" in text:
+        text = text.rsplit("[ANSWER]", 1)[1].split("[/ANSWER]", 1)[0]
+    elif "[/ANSWER]" in text:
+        text = text.rsplit("[/ANSWER]", 1)[0]
+        text = text.rsplit("[/THOUGHT]", 1)[-1]
+    fences = re.findall(r"```[\w-]*\n?(.*?)```", text, re.DOTALL)
+    if fences:
+        text = fences[-1]
+    return text.strip()
+
+
+def _assertion(block: str) -> str | None:
+    """The last ``assert`` statement in the block, if any."""
+    lines = block.splitlines()
+    for index in range(len(lines) - 1, -1, -1):
+        if lines[index].lstrip().startswith("assert"):
+            tail = "\n".join(line.strip() for line in lines[index:])
+            return tail if _parses(tail) else lines[index].strip()
+    return None
+
+
+def _parses(source: str) -> bool:
+    try:
+        ast.parse(source)
+    except SyntaxError:
+        return False
+    return True
+
+
+def _is_f_call(node: ast.expr) -> bool:
+    return (
+        isinstance(node, ast.Call)
+        and isinstance(node.func, ast.Name)
+        and node.func.id == "f"
+    )
+
+
+def _equality(source: str) -> tuple[str, ast.expr, ast.expr] | None:
+    """Split ``[assert] left == right`` into its parsed sides."""
+    text = source.strip()
+    if text.startswith("assert"):
+        text = text[len("assert") :].strip()
+    try:
+        node = ast.parse(text, mode="eval").body
+    except SyntaxError:
+        return None
+    if (
+        isinstance(node, ast.Compare)
+        and len(node.ops) == 1
+        and isinstance(node.ops[0], ast.Eq)
+    ):
+        return text, node.left, node.comparators[0]
+    return None
+
+
+def predicted_output(response: str) -> str | None:
+    """The output the model predicted, as source text."""
+    block = _answer_block(response)
+    candidate = _assertion(block) or block
+    split = _equality(candidate)
+    if split is not None:
+        text, left, right = split
+        side = left if _is_f_call(right) and not _is_f_call(left) else right
+        return ast.get_source_segment(text, side)
+    # Not parseable: a bare word after ``==`` is still a usable prediction.
+    if "==" in candidate:
+        candidate = candidate.rsplit("==", 1)[1]
+    candidate = candidate.split("#", 1)[0].strip()
+    return candidate or None
+
+
+def predicted_call(response: str) -> str | None:
+    """The ``f(...)`` call the model proposed, as source text."""
+    block = _answer_block(response)
+    candidate = _assertion(block) or block
+    split = _equality(candidate)
+    if split is not None:
+        text, left, right = split
+        for side in (left, right):
+            if _is_f_call(side):
+                return ast.get_source_segment(text, side)
+        return None
+    try:
+        node = ast.parse(candidate, mode="eval").body
+    except SyntaxError:
+        return None
+    if _is_f_call(node):
+        return ast.get_source_segment(candidate, node)
+    # A bare input, such as ``"ba", "nana"``, is the argument list itself.
+    return f"f({candidate})"
+
+
+def outputs_match(predicted: str | None, expected: str) -> bool:
+    """Literal comparison of a predicted output with the recorded one."""
+    if predicted is None:
+        return False
+    target = ast.literal_eval(expected)
+    try:
+        value = ast.literal_eval(predicted)
+    except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
+        return isinstance(target, str) and predicted.strip() == target
+    try:
+        return bool(value == target)
+    except Exception:
+        return False
+
+
+class _CRUXEval:
+    name: str
+    task: str  # "output" or "input"
+    style: str  # "direct" or "cot"
+    task_count = 800
+
+    def load_tasks(self) -> list[Task]:
+        tasks = []
+        with open(DATASET, encoding="utf-8") as f:
+            for line in f:
+                d = json.loads(line)
+                tasks.append(
+                    Task(
+                        id=d["task_id"],
+                        prompt=d["function"],
+                        metadata={
+                            "input": d["input"],
+                            "output": d["output"],
+                            # The demo client answers with this.
+                            "canonical_solution": (
+                                f"[ANSWER]\nassert f({d['input']}) == "
+                                f"{d['output']}\n[/ANSWER]"
+                            ),
+                        },
+                    )
+                )
+        return tasks
+
+    def build_prompt(self, task: Task) -> str:
+        template = TEMPLATES[(self.task, self.style)]
+        return template.format(
+            code=task.prompt,
+            input=task.metadata["input"],
+            output=task.metadata["output"],
+        )
+
+    def evaluate(self, task: Task, response: str) -> bool:
+        return self.evaluate_with_feedback(task, response).passed
+
+    def summary_fields(self, records: list[object]) -> dict:
+        """Record which half and which prompt style a score came from."""
+        return {"cruxeval_task": self.task, "cruxeval_prompt": self.style}
+
+
+class _OutputPrediction(_CRUXEval):
+    task = "output"
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        predicted = predicted_output(response)
+        if predicted is None:
+            return EvaluationResult(0.0, NO_ASSERTION)
+        if outputs_match(predicted, task.metadata["output"]):
+            return EvaluationResult(1.0)
+        return EvaluationResult(
+            0.0,
+            "The predicted output does not equal the value the function returns. "
+            "Trace the code again; the answer must be a Python literal.",
+        )
+
+
+class _InputPrediction(_CRUXEval):
+    task = "input"
+    evaluation_activity = "running the predicted input"
+
+    def evaluate_with_feedback(self, task: Task, response: str) -> EvaluationResult:
+        call = predicted_call(response)
+        if call is None:
+            return EvaluationResult(0.0, NO_ASSERTION)
+        # Same check as upstream, ``assert output == f(input)``. The input is
+        # model-written code, so a pass also needs a fresh sentinel printed
+        # after the assertion: an input that exits early cannot fake one.
+        sentinel = secrets.token_hex(16)
+        program = (
+            f"{task.prompt}\n\n"
+            f"assert {task.metadata['output']} == {call}\n"
+            f"print({sentinel!r})\n"
+        )
+        result = execute_with_feedback(program, timeout=5, sentinel=sentinel)
+        if result.passed:
+            return EvaluationResult(1.0)
+        if "assertion failed" in result.feedback:
+            return EvaluationResult(
+                0.0,
+                "The proposed input ran, but the function returned a different "
+                "value than the given output.",
+            )
+        return EvaluationResult(0.0, result.feedback)
+
+
+class CRUXEvalOutput(_OutputPrediction):
+    name = "cruxeval-o"
+    style = "direct"
+
+
+class CRUXEvalOutputCoT(_OutputPrediction):
+    name = "cruxeval-o-cot"
+    style = "cot"
+
+
+class CRUXEvalInput(_InputPrediction):
+    name = "cruxeval-i"
+    style = "direct"
+
+
+class CRUXEvalInputCoT(_InputPrediction):
+    name = "cruxeval-i-cot"
+    style = "cot"

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/src/benchkit/benchmarks/ruler_generator.py

@@ -9,6 +9,7 @@ import string
 import urllib.request
 import uuid
 from collections.abc import Callable
+from functools import lru_cache
 from pathlib import Path
 
 from benchkit.benchmarks.base import Task
@@ -45,6 +46,14 @@ TASKS_PER_BUCKET = DEFAULT_SAMPLES_PER_KIND * len(TASK_KINDS)
 OUTPUT_RESERVE_TOKENS = 256
 FALLBACK_CHARS_PER_TOKEN = 4
 FIT_TOLERANCE = 0.01
+# cwe and fwe share a 2,000-word vocabulary. cwe draws one new uncommon word
+# per unit (~40 chars), so its vocabulary grows with the bucket; 2,000 words
+# stop at about 18k tokens.
+VOCABULARY_SIZE = 2000
+# Every renderer has a floor below which extra units change nothing (cwe and
+# qa at one, fwe at ten, qa_2's ten gold documents), so the fallback sizes a
+# unit between two counts above all of them.
+_PROBE_UNITS = (16, 80)
 
 _VARIABLE_NOISE = (
     "The grass is green. The sky is blue. The sun is yellow. "
@@ -195,8 +204,9 @@ def _aggregation_task(bucket: int, sample: int, kind: str) -> Task:
     }
     if kind in {"cwe", "fwe"}:
         rng = random.Random(seed)
+        size = max(VOCABULARY_SIZE, bucket // 8) if kind == "cwe" else VOCABULARY_SIZE
         vocabulary = [
-            "".join(rng.choices(string.ascii_lowercase, k=6)) for _ in range(2000)
+            "".join(rng.choices(string.ascii_lowercase, k=6)) for _ in range(size)
         ]
         metadata["vocabulary"] = list(dict.fromkeys(vocabulary))
         metadata["answers"] = metadata["vocabulary"][: 10 if kind == "cwe" else 3]
@@ -266,6 +276,7 @@ def _essay_words() -> list[str]:
     return text.split()
 
 
+@lru_cache(maxsize=len(QA_FILES))
 def _qa_records(kind: str) -> tuple[list[dict], list[str]]:
     raw = json.loads(_qa_path(kind).read_text(encoding="utf-8"))
     if kind == "qa_1":
@@ -410,7 +421,19 @@ def _render_niah(task: Task, filler_units: int) -> str:
         target_values=list(metadata["answers"]),
         distractor_values=[p[1] for p in pairs],
     )
-    context = "\n".join(_insert_at_positions(filler, needles, seed ^ 0xA51CE))
+    lines = _insert_at_positions(filler, needles, seed ^ 0xA51CE)
+    if metadata["haystack"] == "essay":
+        # Running prose, not one word per line: a newline per word costs far
+        # more tokens than the character estimate assumes.
+        needle_set = set(needles)
+        chunks: list[list[str]] = [[]]
+        for line in lines:
+            if line in needle_set:
+                chunks.extend([[line], []])
+            else:
+                chunks[-1].append(line)
+        lines = [" ".join(chunk) for chunk in chunks if chunk]
+    context = "\n".join(lines)
     return (
         "Some special magic numbers are hidden within the following text. "
         "Memorize the records because you will be asked for one key afterward.\n\n"
@@ -463,11 +486,17 @@ def render_prompt(task: Task, filler_units: int) -> str:
 
 
 def _fallback_units(task: Task, target_tokens: int) -> int:
-    base = render_prompt(task, 0)
-    one = render_prompt(task, 1)
-    chars_per_unit = max(1, len(one) - len(base))
+    low, high = _PROBE_UNITS
+    low_chars = len(render_prompt(task, low))
+    chars_per_unit = max(
+        1.0, (len(render_prompt(task, high)) - low_chars) / (high - low)
+    )
     target_chars = target_tokens * FALLBACK_CHARS_PER_TOKEN
-    return max(0, (target_chars - len(base)) // chars_per_unit)
+    return max(0, low + math.floor((target_chars - low_chars) / chars_per_unit))
+
+
+def _estimate_tokens(prompt: str) -> int:
+    return math.ceil(len(prompt) / FALLBACK_CHARS_PER_TOKEN)
 
 
 def fit_prompt(
@@ -479,9 +508,12 @@ def fit_prompt(
     """Fit a generated prompt just below its context bucket.
 
     When the server exposes a tokenizer, a short secant-style calibration keeps
-    the prompt within one percent of the target. Generic servers fall back to a
-    conservative character-to-token estimate.
+    the prompt within one percent of the target. Generic servers run the same
+    calibration against a conservative character-to-token estimate, and the
+    measured count comes back as ``None``.
     """
+    estimated = token_count is None
+    count = _estimate_tokens if token_count is None else token_count
     bucket = int(task.metadata["context_tokens"])
     target = max(1, bucket - OUTPUT_RESERVE_TOKENS)
     units = (
@@ -490,10 +522,7 @@ def fit_prompt(
         else _fallback_units(task, target)
     )
     prompt = render_prompt(task, units)
-    if token_count is None:
-        return prompt, units, None
-
-    measured = token_count(prompt)
+    measured = count(prompt)
     if not isinstance(measured, int) or measured <= 0:
         return prompt, units, None
 
@@ -502,7 +531,7 @@ def fit_prompt(
         if measured <= target and measured >= target * (1 - FIT_TOLERANCE):
             break
         if base_tokens is None:
-            base_tokens = token_count(render_prompt(task, 0))
+            base_tokens = count(render_prompt(task, 0))
             if not isinstance(base_tokens, int) or base_tokens < 0:
                 base_tokens = 0
 
@@ -513,23 +542,24 @@ def fit_prompt(
             next_units = max(0, units - 1) if measured > target else units + 1
         units = next_units
         prompt = render_prompt(task, units)
-        next_measured = token_count(prompt)
+        next_measured = count(prompt)
         if not isinstance(next_measured, int) or next_measured <= 0:
             return prompt, units, None
         measured = next_measured
 
     # Never knowingly send a prompt above the bucket. The linear adjustment is
     # normally exact; this final correction handles unusual tokenizer merges.
-    for _ in range(2):
+    # Without a tokenizer it is cheap, so keep going until the estimate fits.
+    for _ in range(16 if estimated else 2):
         if measured <= target or units == 0:
             break
         excess = measured - target
         tokens_per_unit = max(1.0, measured / max(units, 1))
         units = max(0, units - math.ceil(excess / tokens_per_unit) - 1)
         prompt = render_prompt(task, units)
-        next_measured = token_count(prompt)
+        next_measured = count(prompt)
         if not isinstance(next_measured, int) or next_measured <= 0:
             return prompt, units, None
         measured = next_measured
 
-    return prompt, units, measured
+    return prompt, units, None if estimated else measured

{benchkit_cli-0.1.2 → benchkit_cli-0.2.0}/src/benchkit/datasets/README.md

@@ -79,6 +79,20 @@ them evenly. Regenerate both with `uv run python scripts/build_banking77.py`.
   Encoders" (2020), <https://arxiv.org/abs/2003.04807>
 - Dataset license: [Creative Commons Attribution 4.0 International][cc-by-4.0]
 
+## CRUXEval
+
+`cruxeval.jsonl` is a transformed copy of the 800 functions in CRUXEval, each
+with one input and the output running the function produces. Only the schema
+changed - `id` became a namespaced `task_id` and `code` became `function`.
+Regenerate it with `uv run python scripts/build_cruxeval.py`, which pins the
+upstream commit. The prompts in `benchmarks/cruxeval.py` are the upstream
+direct and chain-of-thought prompts.
+
+- Source: <https://github.com/facebookresearch/cruxeval>
+- Paper: Gu et al., "CRUXEval: A Benchmark for Code Reasoning, Understanding
+  and Execution" (2024), <https://arxiv.org/abs/2401.03065>
+- Dataset license: MIT
+
 ## OpenBookQA
 
 `openbookqa.jsonl` is a transformed copy of the 500-question OpenBookQA test