eval-framework 0.6.4__tar.gz → 0.7.2__tar.gz

{eval_framework-0.6.4 → eval_framework-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.6.4
+Version: 0.7.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -215,7 +215,6 @@ Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.0,<6
-Requires-Dist: sacrebleu>=2.6.0,<3
 Requires-Dist: pycountry>=26.2.16,<27
 Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
@@ -231,7 +230,7 @@ Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.4.20
-Requires-Dist: wandb>=0.28.0,<1
+Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
@@ -241,10 +240,10 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.5,<2 ; extra == 'mistral'
+Requires-Dist: mistral-common>=1.11.6,<2 ; extra == 'mistral'
 Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.109.1,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -523,8 +522,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.4 → eval_framework-0.7.2}/README.md

@@ -254,8 +254,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.4 → eval_framework-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.6.4"
+version = "0.7.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -22,7 +22,6 @@ dependencies = [
   "xmltodict>=1.0.4,<1.1",
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.0,<6",
-  "sacrebleu>=2.6.0,<3",
   "pycountry>=26.2.16,<27",
   "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
@@ -38,7 +37,7 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.4.20",
-  "wandb>=0.28.0,<1",
+  "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
   "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
@@ -55,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=1.109.1,<3",
+  "openai>=2.46.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -70,7 +69,7 @@ vllm = [
   "torch>=2.5,<3"
 ]
 mistral = [
-  "mistral-common>=1.11.5,<2",
+  "mistral-common>=1.11.6,<2",
   "huggingface-hub>=0.36.2,<0.37",
   "eval_framework[vllm]",
 ]
@@ -91,16 +90,16 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.2.0,<3",
+  "mypy>=2.3.0,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
   "types-pyyaml>=6.0.12.20260518,<7",
-  "types-python-dateutil>=2.9.0.20260518,<3",
-  "types-requests>=2.33.0.20260518,<3",
+  "types-python-dateutil>=2.9.0.20260716,<3",
+  "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.8.0,<7",
-  "ruff>=0.15.20",
+  "ruff>=0.15.22",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +108,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.28,<0.11.29"]
+requires = ["uv_build>=0.11.29,<0.11.30"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/metrics/completion/language_checker.py

@@ -41,6 +41,9 @@ class LanguageConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         completion_language = response.get_completion_language()
         target_language = response.get_instruction_language()
         if completion_language == target_language == "":
@@ -57,6 +60,9 @@ class LanguageRawConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.raw_completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         raw_completion_language = response.get_raw_completion_language()
         target_language = response.get_instruction_language()
 

eval_framework-0.7.2/src/eval_framework/tasks/__init__.py

@@ -0,0 +1,6 @@
+# Register all tasks on import
+from .task_names import register_all_tasks
+
+register_all_tasks()
+
+del register_all_tasks

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/base.py

@@ -10,12 +10,10 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar
 
 import iso639
 from datasets import DatasetDict, DownloadConfig, load_dataset
-from huggingface_hub import HfApi
-from huggingface_hub.errors import RevisionNotFoundError
 from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
-from eval_framework.tasks.dataset_revisions import DatasetRevision
+from eval_framework.tasks.dataset_revisions import pinned_revision
 from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import Message, Role
 
@@ -93,7 +91,6 @@ class BaseTask[SubjectType](ABC):
     SAMPLE_SPLIT: str
     FEWSHOT_SPLIT: str
     SUBJECTS: list[SubjectType]
-    HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
     # The lock file this task resolves its pinned dataset revision from, keyed by ``DATASET_PATH``.
     # Each task sets this explicitly: point it at a lock file (e.g. ``HF_REVISIONS_LOCKFILE`` or a
@@ -118,15 +115,18 @@ class BaseTask[SubjectType](ABC):
         self.user_prompt_suffix: str | None = None
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
-        self._apply_hf_revision()
+        self.hf_revision: str | None = self._apply_hf_revision()
 
-    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> None:
-        # Precedence: CLI/config override > class HF_REVISION > task-dataset-revisions.json pin.
-        # Applied once at instance creation; not refreshed if the pin file changes mid-run.
+    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
+        # Precedence: CLI/config override > REVISION_LOCKFILE pin.
+        # Tasks without a Hugging Face dataset set REVISION_LOCKFILE to None and are not pinned.
         if custom_hf_revision:
-            self.HF_REVISION = custom_hf_revision
-        elif self.HF_REVISION is None and (pinned := DatasetRevision.pinned_revision(self.__class__.__name__)):
-            self.HF_REVISION = pinned
+            hf_revision = custom_hf_revision
+        elif self.REVISION_LOCKFILE is not None:
+            hf_revision = pinned_revision(self.REVISION_LOCKFILE, self.DATASET_PATH)
+        else:
+            hf_revision = None
+        return hf_revision
 
     @classmethod
     def with_overwrite(
@@ -148,7 +148,7 @@ class BaseTask[SubjectType](ABC):
             logger.info(f"Setting SUBJECTS to `{filtered_subjects}` for the task {instance.__class__.__name__}")
             instance.SUBJECTS = filtered_subjects  # type: ignore[assignment]
 
-        instance._apply_hf_revision(custom_hf_revision)
+        instance.hf_revision = instance._apply_hf_revision(custom_hf_revision)
 
         return instance
 
@@ -187,29 +187,14 @@ class BaseTask[SubjectType](ABC):
             return custom_subjects  # type: ignore[return-value]
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
-        # Check if the HF_REVISION is valid before loading the dataset
-        if self.HF_REVISION:
-            try:
-                _ = HfApi().dataset_info(repo_id=kwargs["path"], revision=self.HF_REVISION, timeout=100.0)
-            except Exception as e:
-                if isinstance(e, RevisionNotFoundError):
-                    raise e
-
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
         download_config = DownloadConfig(cache_dir=cache_dir, max_retries=5)
-        try:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=cache_dir,
-                download_config=download_config,
-            )
-        except Exception:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=f"{Path.home()}/.cache/eval-framework",
-            )
+        return load_dataset(
+            **kwargs,
+            revision=self.hf_revision,
+            cache_dir=cache_dir,
+            download_config=download_config,
+        )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
         dataset = {}

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/copa.py

@@ -17,6 +17,8 @@ class COPAEvalHarness(BaseTask[str]):
     This version uses samples from the validation split as evaluation examples (same as lm-eval-harness).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "COPAEvalHarness"
     DATASET_PATH = "aps/super_glue"
     SAMPLE_SPLIT = "validation"  # 100 examples (same split as lm-eval)
@@ -88,6 +90,7 @@ class COPA(COPAEvalHarness):
 
 
 class COPA_IDKEvalHarness(COPAEvalHarness):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "COPA_IDKEvalHarness"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/csqa.py

@@ -13,6 +13,8 @@ from eval_framework.tasks.utils import get_n_letters
 class CommonsenseQACloze(BaseTask[str]):
     """CommonsenseQA dataset: https://huggingface.co/datasets/tau/commonsense_qa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQACloze"
     DATASET_PATH = "tau/commonsense_qa"
     SAMPLE_SPLIT = "validation"
@@ -53,6 +55,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
     Scores loglikelihood over the full correct choice text; includes bits-per-byte.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAFullTextCloze"
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
 
@@ -65,6 +69,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
 class CommonsenseQAMC(CommonsenseQACloze):
     """Multiple-choice variant of CommonsenseQA where the model selects a letter (A-E)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAMC"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/drop.py

@@ -75,6 +75,8 @@ class DropCompletion(BaseTask[str]):
     Uses DROP F1 and exact match. Stop at new paragraph or repeated prefixes.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCompletion"
     DATASET_PATH = "EleutherAI/drop"
     SAMPLE_SPLIT = "validation"
@@ -163,6 +165,8 @@ class DropCompletion_OLMES(DropCompletion):
 class DropMC(BaseTask[str]):
     """Multiple-choice variant using allenai/drop-gen2mc (passage_original, question_original, choices, answerKey)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropMC"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"
@@ -233,6 +237,8 @@ class DropCloze(BaseTask[str]):
     Includes BitsPerByte on the correct choice.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCloze"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -22,6 +22,8 @@ logger = logging.getLogger(__name__)
 class GPQA(BaseTask[str]):
     """GPQA dataset: https://huggingface.co/datasets/Idavidrein/gpqa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GPQA"
     DATASET_PATH = "Idavidrein/gpqa"
     SAMPLE_SPLIT = "train"
@@ -172,6 +174,7 @@ class GPQA_IDK(GPQA):
 
 
 class GPQA_COT(GPQA):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GPQA_COT"
     RESPONSE_TYPE = ResponseType.COMPLETION
     METRICS = [AccuracyCompletion]

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -28,6 +28,8 @@ class HumanEvalMetricContext(BaseMetricContext):
 class HumanEval(BaseTask[str]):
     """HumanEval dataset: https://huggingface.co/datasets/openai/openai_humaneval/"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval"
     DATASET_PATH = "openai/openai_humaneval"
     SAMPLE_SPLIT = "test"
@@ -128,6 +130,7 @@ class HumanEval_OLMES(HumanEval):
 
 class HumanEvalInstruct(HumanEval):
     # See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/humaneval/humaneval_instruct.yaml
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Human Eval Instruct"
     CUE_PREFIX = "Here is the completed function:\n```python\n"
 

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -482,6 +482,8 @@ class MATH500(MATHReasoning):
 class MATH(MATHReasoning):
     """MATH dataset: https://huggingface.co/datasets/EleutherAI/hendrycks_math"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Math"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -559,6 +561,8 @@ class MATHMinervaEvalHarness(MATHReasoning):
     Metrics: Exact Match, Exact Match (Flex) via MathMinervaCompletion.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinervaEvalHarness"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -595,6 +599,8 @@ class MATHMinerva(MATHMinervaEvalHarness):
     "(The )Final Answer: The (final )answer is ...( I hope it is correct.)", where parentheses are optional.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinerva"
     METRICS = [MathMinervaCompletionRelaxed]
 
@@ -610,6 +616,8 @@ class MATH500Minerva(MATHMinerva):
     Uses HuggingFaceH4/MATH-500 which has a single 'default' config (no subject splits).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATH500Minerva"
     DATASET_PATH = "HuggingFaceH4/MATH-500"
     SAMPLE_SPLIT = "test"
@@ -622,6 +630,7 @@ class MATH500Minerva(MATHMinerva):
 
 
 class MATHLvl5(MATH):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Math Lvl 5"
 
     def _load_dataset(self, subject: SubjectType) -> None:

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -33,6 +33,7 @@ class MBPP(BaseTask[str]):
 
     NAME = "MBPP"
     DATASET_PATH = "google-research-datasets/mbpp"
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/medqa.py

@@ -17,6 +17,8 @@ from eval_framework.tasks.utils import get_n_letters
 class MedQACloze(BaseTask[str]):
     """MedQA cloze (loglikelihood over choice text)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQACloze"
     DATASET_PATH = "davidheineman/medqa-en"
     SAMPLE_SPLIT = "test"
@@ -53,6 +55,8 @@ class MedQACloze(BaseTask[str]):
 class MedQAMC(MedQACloze):
     """MedQA multiple choice (loglikelihood over A/B/C/D/...)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQAMC"
 
     def __init__(self, num_fewshot: int = 0) -> None:

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -82,11 +82,13 @@ class _NaturalQsOpenChoice_Base(BaseTask[str]):
 
 
 class NaturalQsOpenCloze(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenCloze"
     TASK_STYLER = ClozeStyle()
 
 
 class NaturalQsOpenMC(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenMC"
     TASK_STYLER = MCStyle()
 

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/piqa.py

@@ -9,18 +9,17 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
+from eval_framework.tasks.dataset_revisions import FROZEN_HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class PIQA(BaseTask[str]):
     """PIQA dataset: https://huggingface.co/datasets/ybisk/piqa"""
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA"
     DATASET_PATH = "ybisk/piqa"
-    HF_REVISION = "6b3aceb3276e5ab7e51895d73151a718690af38c"
     SAMPLE_SPLIT = "validation"  # 1838 examples (same split as lm-eval)
     FEWSHOT_SPLIT = "test"  # 3084 examples
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
@@ -54,7 +53,7 @@ class PIQA_OLMES(PIQA):
     loglikelihood over " A"/" B".
     """
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA_OLMES"
     SAMPLE_SPLIT = "train"  # Use train split (largest) to best match OLMES, which evaluates all splits
@@ -79,7 +78,7 @@ class PIQA_OLMES(PIQA):
 
 
 class PIQA_IDK(PIQA):
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
     NAME = "PIQA_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/sciq.py

@@ -95,6 +95,7 @@ class SCIQ_OLMES(SCIQ):
 
 
 class SCIQ_IDK(SCIQ):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "SciQ_IDK"
     METRICS = [
         AccuracyLoglikelihood,
@@ -137,6 +138,7 @@ class SCIQEvalHarness(SCIQ):
 
 
 class SCIQEvalHarness_IDK(SCIQEvalHarness):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "SciQ Eval Harness_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/social_iqa.py

@@ -151,6 +151,8 @@ class SocialIQACloze(BaseTask[str]):
     Social IQA cloze: loglikelihood over full answer text.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "SocialIQACloze"
     DATASET_PATH = SOCIAL_I_QA_DATASET_PATH
     SAMPLE_SPLIT = "validation"

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/squad.py

@@ -63,9 +63,9 @@ class SQUAD2(BaseTask[str]):
 
     def _validate_hf_revision(self, dataset_path: str) -> None:
         """Validate HuggingFace revision if specified."""
-        if self.HF_REVISION:
+        if self.hf_revision:
             try:
-                HfApi().dataset_info(repo_id=dataset_path, revision=self.HF_REVISION, timeout=100.0)
+                HfApi().dataset_info(repo_id=dataset_path, revision=self.hf_revision, timeout=100.0)
             except RevisionNotFoundError:
                 raise
 
@@ -76,7 +76,7 @@ class SQUAD2(BaseTask[str]):
 
         return load_dataset(
             **kwargs,
-            revision=self.HF_REVISION,
+            revision=self.hf_revision,
             cache_dir=cache_dir,
             download_config=download_config,
         )
@@ -192,6 +192,8 @@ class SQUAD2BPB(SQUAD2):
     Reports bits-per-byte on the reference answer (first acceptable answer).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "SQuAD2 BPB"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [BitsPerByteLoglikelihood]

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/winogrande.py

@@ -58,6 +58,8 @@ class WINOGRANDE_OLMES(WINOGRANDE):
     loglikelihood over " A"/" B".
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Winogrande_OLMES"
 
     def __init__(self, num_fewshot: int = 0) -> None:

eval_framework-0.7.2/src/eval_framework/tasks/dataset_revisions.py

@@ -0,0 +1,84 @@
+"""Pinned Hugging Face dataset revisions.
+
+A lock file maps dataset paths to pinned commit SHAs. Tasks declare the lock file that
+governs them via their ``REVISION_LOCKFILE`` attribute and resolve their pin from it at
+construction time.
+"""
+
+import json
+import logging
+from functools import lru_cache
+from pathlib import Path
+
+from huggingface_hub import HfApi
+
+logger = logging.getLogger(__name__)
+
+# The revision of the datasets used by the benchmarks is declared in a file, so we can automatically
+# update them in CI without having to parse python code.
+HF_REVISIONS_LOCKFILE = Path(__file__).resolve().parent / "hf-dataset-revisions.json"
+
+# Hand-maintained pins for datasets that must not move, e.g. because newer revisions are
+# incompatible with the task implementation. Never updated by the refresh job.
+FROZEN_HF_REVISIONS_LOCKFILE = Path(__file__).resolve().parent / "frozen-hf-dataset-revisions.json"
+
+
+class HfDatasetRevisions:
+    """Pinned revisions of Hugging Face datasets, mapping dataset path → commit SHA."""
+
+    def __init__(self, revisions: dict[str, str]) -> None:
+        self._revisions = dict(revisions)
+
+    @classmethod
+    def from_file(cls, path: Path) -> "HfDatasetRevisions":
+        return cls(json.loads(path.read_text(encoding="utf-8")))
+
+    def to_dict(self) -> dict[str, str]:
+        return dict(self._revisions)
+
+    def to_file(self, path: Path) -> None:
+        path.write_text(
+            json.dumps(dict(sorted(self._revisions.items())), indent=4, ensure_ascii=False) + "\n",
+            encoding="utf-8",
+        )
+
+    def revision_for(self, dataset_path: str) -> str:
+        """The pinned commit SHA for a dataset. Raises ``KeyError`` if it is not pinned."""
+        return self._revisions[dataset_path]
+
+    def num_revisions(self) -> int:
+        return len(self._revisions)
+
+    def update_to_latest(self, api: HfApi) -> None:
+        """Update every pin to its dataset's latest commit SHA.
+
+        Intended to run on CI to ensure the pinned revisions are up-to-date. If the lookup for a
+        dataset fails, its existing pin is kept.
+        """
+        for path, sha in self._revisions.items():
+            try:
+                latest = api.dataset_info(path, timeout=100.0).sha
+            except Exception as exc:
+                logger.warning("Could not refresh %s (%s); keeping pinned revision %s", path, exc, sha)
+                continue
+            if latest and latest != sha:
+                logger.info("%s: %s -> %s", path, sha, latest)
+            self._revisions[path] = latest or sha
+
+
+@lru_cache
+def _revisions_from_file(lockfile: Path) -> HfDatasetRevisions:
+    return HfDatasetRevisions.from_file(lockfile)
+
+
+def pinned_revision(lockfile: Path, dataset_path: str) -> str:
+    """The commit SHA pinned for ``dataset_path`` in ``lockfile``.
+
+    Resolves the exact dataset revision an eval runs against, so results stay reproducible
+    across dataset updates. Every dataset used by a task is expected to be pinned; a missing
+    entry is a bug in the lock file and raises ``KeyError``.
+    """
+    try:
+        return _revisions_from_file(lockfile).revision_for(dataset_path)
+    except KeyError:
+        raise KeyError(f"Dataset '{dataset_path}' is not pinned in {lockfile}") from None

eval_framework-0.7.2/src/eval_framework/tasks/frozen-hf-dataset-revisions.json

@@ -0,0 +1,3 @@
+{
+    "ybisk/piqa": "6b3aceb3276e5ab7e51895d73151a718690af38c"
+}

{eval_framework-0.6.4 → eval_framework-0.7.2}/src/eval_framework/tasks/hf-dataset-revisions.json

@@ -31,6 +31,5 @@
     "openai/openai_humaneval": "7dce6050a7d6d172f3cc5c32aa97f52fa1a2e544",
     "rajpurkar/squad": "7b6d24c440a36b6815f21b70d25016731768db1f",
     "rajpurkar/squad_v2": "3ffb306f725f7d2ce8394bc1873b24868140c412",
-    "tau/commonsense_qa": "94630fe30dad47192a8546eb75f094926d47e155",
-    "ybisk/piqa": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0"
+    "tau/commonsense_qa": "94630fe30dad47192a8546eb75f094926d47e155"
 }