eval-framework 0.2.7__tar.gz → 0.2.9__tar.gz

{eval_framework-0.2.7 → eval_framework-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.7
+Version: 0.2.9
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -235,6 +235,7 @@ Requires-Dist: python-iso639>=2025.2.18
 Requires-Dist: wandb>=0.23.0,<1
 Requires-Dist: boto3>=1.40.54,<2
 Requires-Dist: numpy>=1.26.4
+Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,comet,optional,mistral] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=10,<11 ; extra == 'api'
@@ -283,7 +284,7 @@ Description-Content-Type: text/markdown
 [![Docs](https://img.shields.io/badge/docs-online-blue)](https://aleph-alpha-research.github.io/eval-framework/)
 [![Stars](https://img.shields.io/github/stars/Aleph-Alpha-Research/eval-framework)](https://github.com/Aleph-Alpha-Research/eval-framework/stargazers)
 
-![eval-framework](docs/eval-framework.png "https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/refs/heads/main/docs/eval-framework.png")
+![eval-framework](https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/refs/heads/main/docs/eval-framework.png)
 
 </div>
 
@@ -543,6 +544,6 @@ This project has received funding from the European Union’s Digital Europe Pro
 The contents of this publication are the sole responsibility of the OpenEuroLLM consortium and do not necessarily reflect the opinion of the European Union.
 
 <p align="center">
-  <img src="docs/OELLM_1.png" alt="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_1.png" width="100" style="margin-right: 50px;"/>
-  <img src="docs/OELLM_2.png" alt="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_2.png" width="350"/>
+  <img src="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_1.png" width="100" style="margin-right: 50px;"/>
+  <img src="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_2.png" width="350"/>
 </p>

{eval_framework-0.2.7 → eval_framework-0.2.9}/README.md

@@ -13,7 +13,7 @@
 [![Docs](https://img.shields.io/badge/docs-online-blue)](https://aleph-alpha-research.github.io/eval-framework/)
 [![Stars](https://img.shields.io/github/stars/Aleph-Alpha-Research/eval-framework)](https://github.com/Aleph-Alpha-Research/eval-framework/stargazers)
 
-![eval-framework](docs/eval-framework.png "https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/refs/heads/main/docs/eval-framework.png")
+![eval-framework](https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/refs/heads/main/docs/eval-framework.png)
 
 </div>
 
@@ -273,6 +273,6 @@ This project has received funding from the European Union’s Digital Europe Pro
 The contents of this publication are the sole responsibility of the OpenEuroLLM consortium and do not necessarily reflect the opinion of the European Union.
 
 <p align="center">
-  <img src="docs/OELLM_1.png" alt="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_1.png" width="100" style="margin-right: 50px;"/>
-  <img src="docs/OELLM_2.png" alt="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_2.png" width="350"/>
+  <img src="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_1.png" width="100" style="margin-right: 50px;"/>
+  <img src="https://raw.githubusercontent.com/Aleph-Alpha-Research/eval-framework/main/docs/OELLM_2.png" width="350"/>
 </p>

{eval_framework-0.2.7 → eval_framework-0.2.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.7"
+version = "0.2.9"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -42,6 +42,9 @@ dependencies = [
   "wandb>=0.23.0,<1",
   "boto3>=1.40.54,<2",
   "numpy>=1.26.4",
+  # is a dependency of sympy, but not explicitly listed in the requirements.txt
+  # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
+  "antlr4-python3-runtime==4.11.0",
 ]
 
 [project.optional-dependencies]

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/context/determined.py

@@ -31,6 +31,7 @@ class TaskArgs(BaseModel):
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     perturbation_config: PerturbationConfig | None = None
+    repeats: int | None = None
 
 
 class Hyperparameters(BaseModel):
@@ -110,6 +111,7 @@ class DeterminedContext(EvalContext):
             "judge_model_name",
             "judge_model_args",
             "perturbation_config",
+            "repeats",
         ]:
             val_cli = getattr(self, name, None)
             val_hparams = getattr(self.hparams.task_args, name, None)
@@ -152,6 +154,7 @@ class DeterminedContext(EvalContext):
             randomize_judge_order=self.randomize_judge_order,
             delete_output_dir_after_upload=self.hparams.delete_output_dir_after_upload
             or self.delete_output_dir_after_upload,
+            repeats=self.hparams.task_args.repeats or self.repeats,
         )
 
         return self

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/context/eval.py

@@ -75,6 +75,7 @@ class EvalContext(AbstractContextManager):
         perturbation_seed: int | None = None,
         randomize_judge_order: bool = False,
         delete_output_dir_after_upload: bool | None = None,
+        repeats: int | None = None,
     ) -> None:
         self.llm_name = llm_name
         self.models_path = models_path
@@ -99,7 +100,7 @@ class EvalContext(AbstractContextManager):
         self.description = description
         self.randomize_judge_order = randomize_judge_order
         self.delete_output_dir_after_upload = delete_output_dir_after_upload
-
+        self.repeats = repeats
         if perturbation_type or perturbation_probability is not None:
             perturbation = {
                 "type": perturbation_type,

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/context/local.py

@@ -65,6 +65,7 @@ class LocalContext(EvalContext):
             description=self.description,
             randomize_judge_order=self.randomize_judge_order,
             delete_output_dir_after_upload=self.delete_output_dir_after_upload,
+            repeats=self.repeats,
         )
 
         return self

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/metrics/completion/math_reasoning_completion.py

@@ -204,10 +204,15 @@ class MathReasoningCompletion(BaseMetric[Completion]):
         timeout = 10
         # latex parse all ingested ground truth values for math reasoning
         for gt in response.ground_truth_list:
+            if gt is None:
+                continue
             signal.signal(signal.SIGALRM, timeout_handler)  # Set timeout signal
             signal.alarm(timeout)  # Set timeout duration
             try:
-                gt_parsed = parse_latex(gt)  # NOTE: parses f(x)=0,\quadf(x)=x-1,\quadf(x)=-x+1 to Eq(f(x), 0) ONLY
+                gt_normalized = self.normalize_expression(gt)
+                gt_parsed = parse_latex(
+                    gt_normalized
+                )  # NOTE: parses f(x)=0,\quadf(x)=x-1,\quadf(x)=-x+1 to Eq(f(x), 0) ONLY
                 ground_truths.append(gt_parsed)
             except Exception:
                 ground_truths.append(gt)
@@ -229,15 +234,11 @@ class MathReasoningCompletion(BaseMetric[Completion]):
                 )
             ]
         else:
-            # fall back to string comparison
-            # ground truth can be list or str, we have str comparisons
-            assert isinstance(response.ground_truth, str)
-            str_is_correct = self._is_str_correct(normalized_response, response.ground_truth)
-            return [
-                MetricResult(
-                    metric_name=self.NAME, value=float(str_is_correct), higher_is_better=True, error=response.error
-                )
+            normalized_ground_truths = [
+                self.normalize_expression(gt) for gt in response.ground_truth_list if gt is not None
             ]
+            res = self._any_str_correct([normalized_response], normalized_ground_truths)
+            return [MetricResult(metric_name=self.NAME, value=float(res), higher_is_better=True, error=response.error)]
 
     def _any_str_correct(self, response_list: list, ground_truths: list) -> bool:
         """

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/response_generator.py

@@ -1,10 +1,9 @@
 import logging
 import time
 import traceback
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from datetime import UTC, datetime
 from functools import partial
-from typing import Any
 
 from eval_framework.tasks.registry import get_task
 
@@ -14,6 +13,8 @@ except ImportError:
     get_cluster_info = None  # type: ignore[assignment]
 
 
+from typing import Any
+
 from tqdm import tqdm
 
 from eval_framework import __version__ as eval_framework_version
@@ -234,18 +235,21 @@ class ResponseGenerator:
         # If samples_batch_size = 1, samples are run sequentially; in any case, we return here after finishing each
         # individual batch to honor preemption requests and save cached results.
         samples_batch_size = self.config.batch_size
+        repeats = self.config.repeats
 
         # Calculate total samples for progress bar - use num_samples or iterate to count
-        total_num_samples = self.num_samples
-        if total_num_samples is None:
+        if self.num_samples is None:
             # Count samples by iterating (this might be expensive for large datasets)
-            total_num_samples = sum(1 for _ in self.task.iterate_samples(None))
+            total_num_samples = sum(1 for _ in self.task.iterate_samples(None)) * repeats
+        else:
+            total_num_samples = self.num_samples * repeats
 
         samples_batch: list[Sample] = []
         with tqdm(
             total=total_num_samples, desc=f"Processing {self.response_type.value}", disable=get_disable_bar_flag()
         ) as pbar:
-            for i, sample in enumerate(self.task.iterate_samples(self.num_samples)):
+            samples = self.task.iterate_samples(self.num_samples)
+            for i, sample in enumerate(repeat_samples(samples, repeats)):
                 subject = f" - Subject: {sample.subject}"
                 sample_index = i + 1
 
@@ -330,6 +334,7 @@ class ResponseGenerator:
             "llm_name",
             "llm_args",
             "perturbation_config",
+            "repeats",
         ]
         for key in keys:
             if loaded_metadata[key] != current_metadata[key]:
@@ -349,3 +354,17 @@ class ResponseGenerator:
         logger.info("Completions generated and saved.")
 
         return responses, preempted
+
+
+def repeat_samples(samples: Iterable[Sample], repeats: int) -> Iterable[Sample]:
+    """Flatten repeats into a single stream of samples.
+
+    After expansion original sample indices do not point to the same sample anymore. They
+    Original sample can be recovered by `original_index = expanded_index // repeats`.
+    """
+    for sample in samples:
+        base_id = sample.id * repeats
+        for repeat_idx in range(repeats):
+            repeated_sample = sample.model_copy()
+            repeated_sample.id = base_id + repeat_idx
+            yield repeated_sample

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/run.py

@@ -77,6 +77,13 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--num-fewshot", type=int, required=False, default=0, help="The number of fewshot examples to use."
     )
+    parser.add_argument(
+        "--repeats",
+        type=int,
+        required=False,
+        default=1,
+        help="The number of times to repeat each sample in the evaluation.",
+    )
     parser.add_argument("--task-name", type=str, required=False, help="The name of the task to evaluate.")
     parser.add_argument(
         "--randomize-judge-order",
@@ -319,6 +326,7 @@ def run_with_kwargs(kwargs: dict) -> None:
         num_samples=kwargs["num_samples"],
         max_tokens=kwargs["max_tokens"],
         num_fewshot=kwargs["num_fewshot"],
+        repeats=kwargs["repeats"],
         task_name=kwargs["task_name"],
         task_subjects=kwargs["task_subjects"],
         hf_revision=kwargs["hf_revision"],

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -355,6 +355,25 @@ class AIME2024(MATHReasoning):
         return item["answer"].lstrip("0")  # valid answers in this dataset range from 0-999 and have leading zeros
 
 
+class AIME2025(AIME2024):
+    """AIME 2025 dataset: https://huggingface.co/datasets/math-ai/aime25
+
+    This dataset contains a single test split of 30 questions.
+    Data contains
+    problem | answer | id
+
+    pass@1 evaluation
+    """
+
+    NAME = "AIME2025"
+    DATASET_PATH = "math-ai/aime25"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
+        return item["answer"]
+
+
 class MATH500(MATHReasoning):
     """MATH500 dataset: https://huggingface.co/datasets/HuggingFaceH4/MATH-500
 

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/tasks/eval_config.py

@@ -54,7 +54,9 @@ class EvalConfig(BaseConfig):
     save_intermediate_results: Annotated[bool, BeforeValidator(lambda v: True if v is None else v)] = True
     save_logs: Annotated[bool, BeforeValidator(lambda v: True if v is None else v)] = True
     delete_output_dir_after_upload: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
-
+    # how many times to repeat a single sample
+    # can be used to reduce variance of tasks with low number of samples, e.g. AIME24
+    repeats: Annotated[int, BeforeValidator(lambda v: 1 if v is None else v), Field(ge=1)] = 1
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
     @property

{eval_framework-0.2.7 → eval_framework-0.2.9}/src/eval_framework/tasks/task_names.py

@@ -18,6 +18,7 @@ class TaskNameEnum(Enum):
 def register_all_tasks() -> None:
     """Register all the benchmark tasks with the eval framework."""
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024")
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE")