eval-framework 0.8.0__tar.gz → 0.8.1__tar.gz

{eval_framework-0.8.0 → eval_framework-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.0
+Version: 0.8.1
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -246,7 +246,7 @@ Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
-Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
+Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework

{eval_framework-0.8.0 → eval_framework-0.8.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.0"
+version = "0.8.1"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -60,7 +60,7 @@ openai = [
 ]
 transformers = [
   "transformers>=4.45.2,<5",
-  "torch>=2.5,<3",
+  "torch>=2.13.0,<3",
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]

{eval_framework-0.8.0 → eval_framework-0.8.1}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -10,10 +10,7 @@ from eval_framework.metrics.completion.math_minerva_completion import (
     MathMinervaCompletionRelaxed,
 )
 from eval_framework.metrics.completion.math_reasoning_completion import MathReasoningCompletion
-from eval_framework.metrics.completion.minerva_math_utils import (
-    extract_answers,
-    normalized_gold_from_solution,
-)
+from eval_framework.metrics.completion.minerva_math_utils import extract_answers, normalized_gold_from_solution
 from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
@@ -775,6 +772,16 @@ class MATHMinerva_OLMES(MATHMinerva):
         return _OLMES_FEWSHOTS[: self.num_fewshot]
 
 
+class MATHMinerva_OLMES_NONL(MATHMinerva_OLMES):
+    NAME = "MATHMinerva_OLMES_NONL"
+
+    def __init__(self, num_fewshot: int = 4) -> None:
+        if num_fewshot != 4:
+            logger.warning("MATHMinerva_OLMES_NONL supports a fixed num_fewshot of 4.")
+        super().__init__(num_fewshot=4)
+        self.stop_sequences = ["Problem:"]
+
+
 class MATHMinervaBPB(MATHMinerva_OLMES):
     REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "MATHMinervaBPB"

{eval_framework-0.8.0 → eval_framework-0.8.1}/src/eval_framework/tasks/task_names.py

@@ -36,6 +36,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava")
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs")