eval-framework 0.5.2__tar.gz → 0.6.3__tar.gz

{eval_framework-0.5.2 → eval_framework-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.5.2
+Version: 0.6.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -217,7 +217,7 @@ Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.0,<6
 Requires-Dist: sacrebleu>=2.6.0,<3
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.9.4,<4
+Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
@@ -231,7 +231,7 @@ Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.4.20
-Requires-Dist: wandb>=0.27.2,<1
+Requires-Dist: wandb>=0.28.0,<1
 Requires-Dist: boto3>=1.43.19,<2
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
@@ -240,11 +240,11 @@ Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
-Requires-Dist: tensorboard==2.20.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.3,<2 ; extra == 'mistral'
+Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
+Requires-Dist: mistral-common>=1.11.5,<2 ; extra == 'mistral'
 Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.62,<3 ; extra == 'openai'
+Requires-Dist: openai>=1.109.1,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.5.2 → eval_framework-0.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.5.2"
+version = "0.6.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -24,7 +24,7 @@ dependencies = [
   "datasets>=5.0.0,<6",
   "sacrebleu>=2.6.0,<3",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.9.4,<4",
+  "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
@@ -38,7 +38,7 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.4.20",
-  "wandb>=0.27.2,<1",
+  "wandb>=0.28.0,<1",
   "boto3>=1.43.19,<2",
   "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
@@ -51,11 +51,11 @@ dependencies = [
 # Model-specific extras
 determined = [
   "determined>=0.38.1,<0.39",
-  "tensorboard==2.20.0"
+  "tensorboard==2.21.0"
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=1.62,<3",
+  "openai>=1.109.1,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -70,7 +70,7 @@ vllm = [
   "torch>=2.5,<3"
 ]
 mistral = [
-  "mistral-common>=1.11.3,<2",
+  "mistral-common>=1.11.5,<2",
   "huggingface-hub>=0.36.2,<0.37",
   "eval_framework[vllm]",
 ]
@@ -91,7 +91,7 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.1.0,<3",
+  "mypy>=2.2.0,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
@@ -100,7 +100,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260518,<3",
   "types-requests>=2.33.0.20260518,<3",
   "plotly>=6.8.0,<7",
-  "ruff>=0.15.18",
+  "ruff>=0.15.20",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +109,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.23,<0.11.24"]
+requires = ["uv_build>=0.11.28,<0.11.29"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]
@@ -145,7 +145,6 @@ known-third-party = ["wandb"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]
-"tests/tests_eval_framework/tasks/benchmarks/test_mmlu_de.py" = ["E501"]
 
 [tool.mypy]
 plugins = "pydantic.mypy"

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/metrics/completion/code_assertion.py

@@ -15,7 +15,7 @@ class CodeCompletionAssertion(BaseMetric[Completion]):
         # this will always be a list, if return is "" this will be an empty list
         code = response.completion
         try:
-            output = run_python_code(code, image="python:3.12-slim")
+            output = run_python_code(code, image="python:3.12-slim", runtime_configs={"mem_limit": "512m"})
         except SandboxTimeoutError:
             # The submitted code timed out (e.g. an infinite loop) -- a failing sample, not an infra
             # problem.

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/base.py

@@ -95,6 +95,12 @@ class BaseTask[SubjectType](ABC):
     SUBJECTS: list[SubjectType]
     HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
+    # The lock file this task resolves its pinned dataset revision from, keyed by ``DATASET_PATH``.
+    # Each task sets this explicitly: point it at a lock file (e.g. ``HF_REVISIONS_LOCKFILE`` or a
+    # frozen one), or ``None`` to opt out of pinning. Deliberately not defaulted so it is never
+    # inherited implicitly (a subclass in another package would otherwise resolve the wrong file).
+    REVISION_LOCKFILE: Path | None
+
     # Words in _get_instruction_text() not to be perturbed. List of words is case insensitive. No special characters
     # or whitespace should be included.
     PERTURBATION_UNMODIFIABLE_WORDS: list[str] | None

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/arc.py

@@ -9,12 +9,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class ARC(BaseTask[str]):
     """ARC dataset: https://huggingface.co/datasets/allenai/ai2_arc"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC"
     DATASET_PATH = "allenai/ai2_arc"
     SAMPLE_SPLIT = "test"
@@ -55,6 +58,8 @@ class ARC_OLMES(ARC):
     loglikelihood over " A"/" B"/ etc.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
@@ -76,6 +81,7 @@ class ARC_OLMES(ARC):
 
 
 class ARC_IDK(ARC):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "ARC_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/arc_de.py

@@ -6,12 +6,15 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class ARC_DE(BaseTask[str]):
     """ARC-DE dataset: https://huggingface.co/datasets/LeoLM/ArcChallenge_de"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC German"
     DATASET_PATH = "LeoLM/ArcChallenge_de"
     SAMPLE_SPLIT = "test"

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/bigcodebench.py

@@ -16,6 +16,7 @@ from eval_framework.tasks.base import (
     Sample,
     SubjectType,
 )
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import (
     BIG_CODE_BENCH_PACKAGE_MAPPING,
     CallableSerializer,
@@ -123,6 +124,8 @@ class BigCodeBench_OLMES(BigCodeBench):
     then compute pass@1 over the 5 samples per problem (post-process if needed).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "BigCodeBench_OLMES"
     SAMPLE_SPLIT = "v0.1.2"
     FEWSHOT_SPLIT = "v0.1.2"

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/copa.py

@@ -8,6 +8,7 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -51,6 +52,8 @@ class COPA_OLMES(COPAEvalHarness):
     space-prefixed labels (" A.", " B."); loglikelihood over " A"/" B".
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "COPA_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/csqa.py

@@ -6,6 +6,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -86,6 +87,8 @@ class CommonsenseQAMC_OLMES(CommonsenseQAMC):
     CommonsenseQA MC with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAMC_OLMES"
     SAMPLE_SPLIT = "train"  # Use train split (largest) to best match OLMES, which evaluates all splits
     FEWSHOT_SPLIT = "train"

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/drop.py

@@ -10,6 +10,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -139,6 +140,8 @@ class DropCompletion(BaseTask[str]):
 class DropCompletion_OLMES(DropCompletion):
     """DropCompletion matching OLMES, using train split for fewshot and max tokens 100."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCompletion_OLMES"
     FEWSHOT_SPLIT = "train"
 
@@ -210,6 +213,8 @@ class DropMC_OLMES(DropMC):
     DropMC with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropMC_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/global_mmlu.py

@@ -9,6 +9,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import RANDOM_SEED, BaseTask, Language, ResponseType
 from eval_framework.tasks.benchmarks.mmlu import MMLU_SUBJECTS
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 GLOBAL_MMLU_LANGUAGES = ["fr", "de", "es", "it", "pt", "ar"]
@@ -469,6 +470,8 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
     https://github.com/aisingapore/SEA-HELM/blob/main/seahelm_tasks/knowledge/global_mmlu/abstract_algebra/config.yaml
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GlobalMMLU"
     DATASET_PATH = "CohereLabs/Global-MMLU"
     SAMPLE_SPLIT = "test"
@@ -534,6 +537,7 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
 
 
 class GlobalMMLU_German(GlobalMMLU):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GlobalMMLU_German"
     SUBJECTS = [("de", subject) for subject in MMLU_SUBJECTS]
     LANGUAGE = Language.DEU

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/goldenswag.py

@@ -8,12 +8,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.benchmarks.hellaswag import HELLASWAG
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class GOLDENSWAG(HELLASWAG):
     """GoldenSwag dataset: https://huggingface.co/datasets/PleIAs/GoldenSwag
     available data set sections: validation"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GoldenSwag"
     DATASET_PATH = "PleIAs/GoldenSwag"
     SAMPLE_SPLIT = "validation"
@@ -21,6 +24,7 @@ class GOLDENSWAG(HELLASWAG):
 
 
 class GOLDENSWAG_IDK(GOLDENSWAG):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GoldenSwag_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -13,6 +13,7 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 logger = logging.getLogger(__name__)
@@ -126,6 +127,8 @@ class GPQA_OLMES(GPQA):
     (" A.", " B.", " C.", " D."); loglikelihood over " A"/" B"/" C"/" D".
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GPQA_OLMES"
 
     def _get_possible_completions_marked(self, item: dict[str, Any]) -> tuple[list[str], int]:

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/gsm8k.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion, AccuracyCompletionOLMES
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
 
 logger = logging.getLogger(__name__)
@@ -155,6 +156,7 @@ class GSM8K(GSM8KEvalHarness):
 
 
 class GSM8K_OLMES(GSM8K):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GSM8K_OLMES"
     METRICS = [AccuracyCompletionOLMES]
 
@@ -219,6 +221,7 @@ class GSM8K_OLMES(GSM8K):
 
 
 class GSM8KBPB(GSM8K_OLMES):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GSM8KBPB"
     TASK_STYLER = BPBStyle(cue_text="Answer:", leading_space_continuations=False)
 

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/hellaswag.py

@@ -10,12 +10,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class HELLASWAG(BaseTask[str]):
     """Hellaswag dataset: https://huggingface.co/datasets/Rowan/hellaswag
     available data set sections: train, validation, test"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "HellaSwag"
     DATASET_PATH = "Rowan/hellaswag"
     SAMPLE_SPLIT = "validation"
@@ -49,6 +52,7 @@ class HELLASWAG(BaseTask[str]):
 
 
 class HELLASWAG_OLMES(HELLASWAG):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "HellaSwag_OLMES"
     SAMPLE_SPLIT = "train"
 

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -4,6 +4,7 @@ from eval_framework.metrics.completion.code_assertion import CodeCompletionAsser
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.shared.types import BaseMetricContext
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 CODE_TO_EXECUTE = """
 {start_of_code}
@@ -84,6 +85,8 @@ class HumanEvalBPB(HumanEval):
     Reports bits-per-byte on the reference completion.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval BPB"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [BitsPerByteLoglikelihood]
@@ -107,6 +110,8 @@ class HumanEval_OLMES(HumanEval):
         llm_args: {sampling_params: {temperature: 0.6, top_p: 0.6}}
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval OLMES"
 
     def __init__(self, num_fewshot: int = 3) -> None:

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/ifeval.py

@@ -3,11 +3,14 @@ from typing import Any
 from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
 from eval_framework.metrics.completion.language_checker import LanguageRawConsistencyChecker
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class IFEval(BaseTask[str]):
     """IFEval: Instruction Following Eval (https://arxiv.org/pdf/2311.07911)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "IFEval"
     DATASET_PATH = "google/IFEval"
     SAMPLE_SPLIT = "train"
@@ -73,6 +76,8 @@ class IFEvalFiSv(IFEval):
 class IFEvalDe(IFEval):
     """German version of the Instruction Following Evaluation (IFEval) benchmark."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "IFEval German"
     DATASET_PATH = "jzhang86/de_ifeval"
     SUBJECTS = [NO_SUBJECT]

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -15,6 +15,7 @@ from eval_framework.metrics.completion.minerva_math_utils import (
     normalized_gold_from_solution,
 )
 from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
 
 # Hendrycks MATH subject splits (shared by MATH, MATHMinervaEvalHarness, MATHMinervaBPB)
@@ -331,6 +332,8 @@ class AIME2024(MATHReasoning):
     pass@1 evaluation
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "AIME2024"
     DATASET_PATH = "HuggingFaceH4/aime_2024"
     SAMPLE_SPLIT = "train"
@@ -387,6 +390,8 @@ class AIME2025(AIME2024):
     pass@1 evaluation
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "AIME2025"
     DATASET_PATH = "math-ai/aime25"
     SAMPLE_SPLIT = "test"
@@ -406,6 +411,8 @@ class AIME2026(AIME2024):
     pass@1 evaluation
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "AIME2026"
     DATASET_PATH = "math-ai/aime26"
     SAMPLE_SPLIT = "test"
@@ -426,6 +433,8 @@ class MATH500(MATHReasoning):
     pass@1 evaluation
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATH500"
     DATASET_PATH = "HuggingFaceH4/MATH-500"
     SAMPLE_SPLIT = "test"
@@ -642,6 +651,8 @@ class GSM8KReasoning(MATHReasoning):
     Zero-shot reasoning version that expects answers in boxed format.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GSM8KReasoning"
     DATASET_PATH = "openai/gsm8k"
     SAMPLE_SPLIT = "test"
@@ -742,6 +753,7 @@ _OLMES_FEWSHOTS = [
 
 
 class MATHMinerva_OLMES(MATHMinerva):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "MATHMinerva_OLMES"
     METRICS = [MathMinervaCompletion, MathMinervaCompletionRelaxed]
 
@@ -755,32 +767,29 @@ class MATHMinerva_OLMES(MATHMinerva):
 
 
 class MATHMinervaBPB(MATHMinerva_OLMES):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "MATHMinervaBPB"
-    TASK_STYLER = BPBStyle(cue_text="Solution:")
-
-    # BPBStyle already adds "Solution:" as that separate assistant message. But the methods we inherit
-    # still put "Solution:" at the end of the question text and leave it out of the fewshot answer.
-    # So we override them here: remove "Solution:" from the question, and add it back in front of the
-    # fewshot answer. Without this, the question ends in "Solution:Solution:" and fewshot answers have
-    # no "Solution:" label at all.
+    TASK_STYLER = BPBStyle(cue_text="Solution:", question_prefix="Problem:\n")
 
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return "Problem:\n" + item["problem"] + "\n\n"
+    # MATHMinervaEvalHarness (an ancestor) overrides _get_instruction_text/_get_fewshot_target_text/
+    # _get_ground_truth directly instead of going through a TASK_STYLER, so we re-override them here to
+    # opt back into TASK_STYLER-driven formatting. _get_raw_question/_get_choices are the required data
+    # hooks the styler reads from; they have no styler-based default and must supply the raw text.
 
-    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        return f"Solution: {item['solution']}"
+    def _get_raw_question(self, item: dict[str, Any]) -> str:
+        return item["problem"]
 
     def _get_choices(self, item: dict[str, Any]) -> list[str]:
-        answer = normalized_gold_from_solution(item["solution"])
-        template = f"\nFinal Answer: The final answer is {answer}. I hope it is correct."
-
-        return [item["solution"] + template]
+        return [item["solution"]]
 
     def _get_correct_index(self, item: dict[str, Any]) -> int:
         return 0
 
-    def _get_raw_question(self, item: dict[str, Any]) -> str:
-        return item["problem"]
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return self.TASK_STYLER.get_instruction_text(self._get_raw_question(item), self._get_choices(item))
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return self.TASK_STYLER.get_fewshot_target_text(self._get_choices(item), self._get_correct_index(item))
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
-        return self._get_choices(item)[0]
+        return self.TASK_STYLER.get_ground_truth(self._get_choices(item), self._get_correct_index(item))

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -9,6 +9,7 @@ from eval_framework.metrics.completion.code_assertion import (
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.shared.types import BaseMetricContext
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 logger = logging.getLogger(__name__)
 
@@ -124,6 +125,8 @@ class MBPPBPB(MBPP):
     Reports bits-per-byte on the reference solution.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MBPP BPB"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [BitsPerByteLoglikelihood]
@@ -271,6 +274,8 @@ class MBPP_OLMES(MBPP):
         repeats: 32
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MBPP_OLMES"
     FEWSHOT_SPLIT = "test"
 

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/medqa.py

@@ -10,6 +10,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -79,6 +80,8 @@ class MedQAMC_OLMES(MedQAMC):
     MedQA multiple choice with OLMES-style prompt: space before each label (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQAMC_OLMES"
     FEWSHOT_SPLIT = "train"
 

{eval_framework-0.5.2 → eval_framework-0.6.3}/src/eval_framework/tasks/benchmarks/mmlu.py

@@ -11,6 +11,7 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 MMLU_SUBJECTS = [
@@ -77,6 +78,8 @@ MMLU_SUBJECTS = [
 class MMLU(BaseTask[str]):
     """MMLU dataset: https://huggingface.co/datasets/cais/mmlu"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MMLU"
     DATASET_PATH = "cais/mmlu"
     SAMPLE_SPLIT = "test"
@@ -123,6 +126,8 @@ class MMLU_OLMES(MMLU):
     MMLU with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MMLU_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
@@ -156,6 +161,7 @@ Answer with the full text of the correct answer."""
 
 
 class MMLU_IDK(MMLU):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "MMLU_IDK"
     METRICS = [
         AccuracyLoglikelihood,
@@ -183,6 +189,8 @@ class MMLU_COT(MMLU):
     Inspired by https://arxiv.org/pdf/2411.15124 (Table 44)
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MMLU_COT"
     RESPONSE_TYPE = ResponseType.COMPLETION
     METRICS = [AccuracyCompletion]