eval-framework 0.2.8__tar.gz → 0.2.10__tar.gz

{eval_framework-0.2.8 → eval_framework-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.8
+Version: 0.2.10
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.2.8 → eval_framework-0.2.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.8"
+version = "0.2.10"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/context/determined.py

@@ -31,6 +31,7 @@ class TaskArgs(BaseModel):
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     perturbation_config: PerturbationConfig | None = None
+    repeats: int | None = None
 
 
 class Hyperparameters(BaseModel):
@@ -110,6 +111,7 @@ class DeterminedContext(EvalContext):
             "judge_model_name",
             "judge_model_args",
             "perturbation_config",
+            "repeats",
         ]:
             val_cli = getattr(self, name, None)
             val_hparams = getattr(self.hparams.task_args, name, None)
@@ -152,6 +154,7 @@ class DeterminedContext(EvalContext):
             randomize_judge_order=self.randomize_judge_order,
             delete_output_dir_after_upload=self.hparams.delete_output_dir_after_upload
             or self.delete_output_dir_after_upload,
+            repeats=self.hparams.task_args.repeats or self.repeats,
         )
 
         return self

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/context/eval.py

@@ -75,6 +75,7 @@ class EvalContext(AbstractContextManager):
         perturbation_seed: int | None = None,
         randomize_judge_order: bool = False,
         delete_output_dir_after_upload: bool | None = None,
+        repeats: int | None = None,
     ) -> None:
         self.llm_name = llm_name
         self.models_path = models_path
@@ -99,7 +100,7 @@ class EvalContext(AbstractContextManager):
         self.description = description
         self.randomize_judge_order = randomize_judge_order
         self.delete_output_dir_after_upload = delete_output_dir_after_upload
-
+        self.repeats = repeats
         if perturbation_type or perturbation_probability is not None:
             perturbation = {
                 "type": perturbation_type,

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/context/local.py

@@ -65,6 +65,7 @@ class LocalContext(EvalContext):
             description=self.description,
             randomize_judge_order=self.randomize_judge_order,
             delete_output_dir_after_upload=self.delete_output_dir_after_upload,
+            repeats=self.repeats,
         )
 
         return self

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/response_generator.py

@@ -1,10 +1,9 @@
 import logging
 import time
 import traceback
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from datetime import UTC, datetime
 from functools import partial
-from typing import Any
 
 from eval_framework.tasks.registry import get_task
 
@@ -14,6 +13,8 @@ except ImportError:
     get_cluster_info = None  # type: ignore[assignment]
 
 
+from typing import Any
+
 from tqdm import tqdm
 
 from eval_framework import __version__ as eval_framework_version
@@ -234,18 +235,21 @@ class ResponseGenerator:
         # If samples_batch_size = 1, samples are run sequentially; in any case, we return here after finishing each
         # individual batch to honor preemption requests and save cached results.
         samples_batch_size = self.config.batch_size
+        repeats = self.config.repeats
 
         # Calculate total samples for progress bar - use num_samples or iterate to count
-        total_num_samples = self.num_samples
-        if total_num_samples is None:
+        if self.num_samples is None:
             # Count samples by iterating (this might be expensive for large datasets)
-            total_num_samples = sum(1 for _ in self.task.iterate_samples(None))
+            total_num_samples = sum(1 for _ in self.task.iterate_samples(None)) * repeats
+        else:
+            total_num_samples = self.num_samples * repeats
 
         samples_batch: list[Sample] = []
         with tqdm(
             total=total_num_samples, desc=f"Processing {self.response_type.value}", disable=get_disable_bar_flag()
         ) as pbar:
-            for i, sample in enumerate(self.task.iterate_samples(self.num_samples)):
+            samples = self.task.iterate_samples(self.num_samples)
+            for i, sample in enumerate(repeat_samples(samples, repeats)):
                 subject = f" - Subject: {sample.subject}"
                 sample_index = i + 1
 
@@ -330,6 +334,7 @@ class ResponseGenerator:
             "llm_name",
             "llm_args",
             "perturbation_config",
+            "repeats",
         ]
         for key in keys:
             if loaded_metadata[key] != current_metadata[key]:
@@ -349,3 +354,17 @@ class ResponseGenerator:
         logger.info("Completions generated and saved.")
 
         return responses, preempted
+
+
+def repeat_samples(samples: Iterable[Sample], repeats: int) -> Iterable[Sample]:
+    """Flatten repeats into a single stream of samples.
+
+    After expansion original sample indices do not point to the same sample anymore. They
+    Original sample can be recovered by `original_index = expanded_index // repeats`.
+    """
+    for sample in samples:
+        base_id = sample.id * repeats
+        for repeat_idx in range(repeats):
+            repeated_sample = sample.model_copy()
+            repeated_sample.id = base_id + repeat_idx
+            yield repeated_sample

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/run.py

@@ -77,6 +77,13 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--num-fewshot", type=int, required=False, default=0, help="The number of fewshot examples to use."
     )
+    parser.add_argument(
+        "--repeats",
+        type=int,
+        required=False,
+        default=1,
+        help="The number of times to repeat each sample in the evaluation.",
+    )
     parser.add_argument("--task-name", type=str, required=False, help="The name of the task to evaluate.")
     parser.add_argument(
         "--randomize-judge-order",
@@ -319,6 +326,7 @@ def run_with_kwargs(kwargs: dict) -> None:
         num_samples=kwargs["num_samples"],
         max_tokens=kwargs["max_tokens"],
         num_fewshot=kwargs["num_fewshot"],
+        repeats=kwargs["repeats"],
         task_name=kwargs["task_name"],
         task_subjects=kwargs["task_subjects"],
         hf_revision=kwargs["hf_revision"],

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/arc.py

@@ -15,7 +15,7 @@ class ARC(BaseTask[str]):
     """ARC dataset: https://huggingface.co/datasets/allenai/ai2_arc"""
 
     NAME = "ARC"
-    DATASET_PATH = "ai2_arc"
+    DATASET_PATH = "allenai/ai2_arc"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/casehold.py

@@ -9,8 +9,10 @@ from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Languag
 
 
 class CASEHOLD(BaseTask[str]):
+    """CASEHOLD dataset: https://huggingface.co/datasets/coastalcph/lex_glue"""
+
     NAME = "CaseHold"
-    DATASET_PATH = "lex_glue"
+    DATASET_PATH = "coastalcph/lex_glue"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/gsm8k.py

@@ -91,7 +91,7 @@ class GSM8KEvalHarness(BaseTask[str]):
     """
 
     NAME = "GSM8KEvalHarness"
-    DATASET_PATH = "gsm8k"
+    DATASET_PATH = "openai/gsm8k"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -355,6 +355,25 @@ class AIME2024(MATHReasoning):
         return item["answer"].lstrip("0")  # valid answers in this dataset range from 0-999 and have leading zeros
 
 
+class AIME2025(AIME2024):
+    """AIME 2025 dataset: https://huggingface.co/datasets/math-ai/aime25
+
+    This dataset contains a single test split of 30 questions.
+    Data contains
+    problem | answer | id
+
+    pass@1 evaluation
+    """
+
+    NAME = "AIME2025"
+    DATASET_PATH = "math-ai/aime25"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
+        return item["answer"]
+
+
 class MATH500(MATHReasoning):
     """MATH500 dataset: https://huggingface.co/datasets/HuggingFaceH4/MATH-500
 
@@ -520,7 +539,7 @@ class GSM8KReasoning(MATHReasoning):
     """
 
     NAME = "GSM8KReasoning"
-    DATASET_PATH = "gsm8k"
+    DATASET_PATH = "openai/gsm8k"
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/truthfulqa.py

@@ -38,7 +38,7 @@ class TRUTHFULQA(BaseTask[str]):
     """TRUTHFULQA dataset: https://huggingface.co/datasets/truthfulqa/truthful_qa"""
 
     NAME = "TruthfulQA"
-    DATASET_PATH = "truthful_qa"
+    DATASET_PATH = "truthfulqa/truthful_qa"
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = ""
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/benchmarks/winogrande.py

@@ -13,10 +13,10 @@ ANSWER_STR_TO_NUM = {"1": 0, "2": 1}
 
 
 class WINOGRANDE(BaseTask[str]):
-    """WINOGRANDE dataset: https://huggingface.co/datasets/winogrande"""
+    """WINOGRANDE dataset: https://huggingface.co/datasets/allenai/winogrande"""
 
     NAME = "Winogrande"
-    DATASET_PATH = "winogrande"
+    DATASET_PATH = "allenai/winogrande"
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/eval_config.py

@@ -54,7 +54,9 @@ class EvalConfig(BaseConfig):
     save_intermediate_results: Annotated[bool, BeforeValidator(lambda v: True if v is None else v)] = True
     save_logs: Annotated[bool, BeforeValidator(lambda v: True if v is None else v)] = True
     delete_output_dir_after_upload: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
-
+    # how many times to repeat a single sample
+    # can be used to reduce variance of tasks with low number of samples, e.g. AIME24
+    repeats: Annotated[int, BeforeValidator(lambda v: 1 if v is None else v), Field(ge=1)] = 1
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
     @property

{eval_framework-0.2.8 → eval_framework-0.2.10}/src/eval_framework/tasks/task_names.py

@@ -18,6 +18,7 @@ class TaskNameEnum(Enum):
 def register_all_tasks() -> None:
     """Register all the benchmark tasks with the eval framework."""
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024")
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC")
     register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE")