eval-framework 0.6.4__tar.gz → 0.8.0__tar.gz

{eval_framework-0.6.4 → eval_framework-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.6.4
+Version: 0.8.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -215,7 +215,6 @@ Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.0,<6
-Requires-Dist: sacrebleu>=2.6.0,<3
 Requires-Dist: pycountry>=26.2.16,<27
 Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
@@ -231,20 +230,17 @@ Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.4.20
-Requires-Dist: wandb>=0.28.0,<1
+Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
-Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral] ; extra == 'all'
+Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,optional] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.5,<2 ; extra == 'mistral'
-Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
-Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.109.1,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -252,19 +248,15 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.5,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Dist: vllm>=0.22,<0.23 ; extra == 'vllm'
-Requires-Dist: torch>=2.5,<3 ; extra == 'vllm'
 Requires-Python: >=3.12, <3.13
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
 Provides-Extra: accelerate
 Provides-Extra: all
 Provides-Extra: api
 Provides-Extra: determined
-Provides-Extra: mistral
 Provides-Extra: openai
 Provides-Extra: optional
 Provides-Extra: transformers
-Provides-Extra: vllm
 Description-Content-Type: text/markdown
 
 <!-- Badges -->
@@ -317,12 +309,13 @@ pip install eval_framework
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
 - `determined` for running jobs via determined.
-- `mistral` for inference on Mistral models.
+- `openai` for inference against OpenAI-compatible HTTP endpoints.
 - `transformers` for inference using the transformers library.
-- `vllm` for inference via VLLM.
 
 As a short hand, the `all` extra installs all of the above.
 
+For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
+
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -523,8 +516,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.4 → eval_framework-0.8.0}/README.md

@@ -48,12 +48,13 @@ pip install eval_framework
 There are optional extras available to unlock specific features of the library:
 - `api` for inference using the aleph-alpha client.
 - `determined` for running jobs via determined.
-- `mistral` for inference on Mistral models.
+- `openai` for inference against OpenAI-compatible HTTP endpoints.
 - `transformers` for inference using the transformers library.
-- `vllm` for inference via VLLM.
 
 As a short hand, the `all` extra installs all of the above.
 
+For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
+
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -254,8 +255,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.4 → eval_framework-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.6.4"
+version = "0.8.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -22,7 +22,6 @@ dependencies = [
   "xmltodict>=1.0.4,<1.1",
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.0,<6",
-  "sacrebleu>=2.6.0,<3",
   "pycountry>=26.2.16,<27",
   "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
@@ -38,7 +37,7 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.4.20",
-  "wandb>=0.28.0,<1",
+  "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
   "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
@@ -55,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=1.109.1,<3",
+  "openai>=2.46.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -65,22 +64,15 @@ transformers = [
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
-vllm = [
-  "vllm>=0.22,<0.23",
-  "torch>=2.5,<3"
-]
-mistral = [
-  "mistral-common>=1.11.5,<2",
-  "huggingface-hub>=0.36.2,<0.37",
-  "eval_framework[vllm]",
-]
+
+
 # from template-formatting
 optional = [
   "transformers>=4.45.2,<5",
   "jinja2>=3.1.6,<4"
 ]
 all = [
-  "eval_framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral]"
+  "eval_framework[determined,api,openai,transformers,accelerate,optional]"
 ]
 
 [project.urls]
@@ -91,16 +83,16 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.2.0,<3",
+  "mypy>=2.3.0,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
   "types-pyyaml>=6.0.12.20260518,<7",
-  "types-python-dateutil>=2.9.0.20260518,<3",
-  "types-requests>=2.33.0.20260518,<3",
-  "plotly>=6.8.0,<7",
-  "ruff>=0.15.20",
+  "types-python-dateutil>=2.9.0.20260716,<3",
+  "types-requests>=2.33.0.20260712,<3",
+  "plotly>=6.9.0,<7",
+  "ruff>=0.15.22",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +101,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.28,<0.11.29"]
+requires = ["uv_build>=0.11.29,<0.11.30"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]
@@ -161,7 +153,6 @@ markers = [
     "cpu_slow: runs for a long time (on CPU)",
     "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
     "external_api: needs external services for execution",
-    "vllm: tests that specifically require vLLM functionality",
     "formatter_hash: formatter consistency tests using hash comparisons",
 ]
 filterwarnings = [

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/llm/models.py

@@ -18,11 +18,6 @@ if is_extra_installed(extra="transformers"):
         Qwen3_0_6B,
     )
 
-if is_extra_installed("mistral"):
-    from eval_framework.llm.mistral import MagistralVLLM  # noqa F401
 
 if is_extra_installed("openai"):
     from eval_framework.llm.openai import OpenAIModel  # noqa F401
-
-if is_extra_installed("vllm"):
-    from eval_framework.llm.vllm import VLLMRegistryModel, Qwen3_0_6B_VLLM, Qwen3_0_6B_VLLM_No_Thinking  # noqa F401

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/metrics/completion/language_checker.py

@@ -41,6 +41,9 @@ class LanguageConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         completion_language = response.get_completion_language()
         target_language = response.get_instruction_language()
         if completion_language == target_language == "":
@@ -57,6 +60,9 @@ class LanguageRawConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.raw_completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         raw_completion_language = response.get_raw_completion_language()
         target_language = response.get_instruction_language()
 

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/run.py

@@ -13,6 +13,7 @@ except ImportError:
 from eval_framework.context.local import LocalContext
 from eval_framework.main import main
 from eval_framework.tasks.task_loader import load_extra_tasks
+from eval_framework.tasks.utils import close_pools
 from eval_framework.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
@@ -392,19 +393,22 @@ def _run_single_task(kwargs: dict) -> None:
 def run_with_kwargs(kwargs: dict) -> None:
     task_suite_path = kwargs.pop("task_suite", None)
 
-    if task_suite_path is not None:
-        from eval_framework.suite import TaskSuite, run_suite
-
-        output_dir = kwargs.get("output_dir", "outputs")
-        log_level = kwargs.get("verbosity", 1)
-        setup_logging(output_dir, log_level=log_level)
-
-        suite = TaskSuite.load(task_suite_path)
-        logger.info(f"Loaded task suite '{suite.name}' from {task_suite_path}")
-        result = run_suite(suite, kwargs)
-        logger.info(f"Suite '{suite.name}' completed. Aggregates: {result.aggregates}")
-    else:
-        _run_single_task(kwargs)
+    try:
+        if task_suite_path is not None:
+            from eval_framework.suite import TaskSuite, run_suite
+
+            output_dir = kwargs.get("output_dir", "outputs")
+            log_level = kwargs.get("verbosity", 1)
+            setup_logging(output_dir, log_level=log_level)
+
+            suite = TaskSuite.load(task_suite_path)
+            logger.info(f"Loaded task suite '{suite.name}' from {task_suite_path}")
+            result = run_suite(suite, kwargs)
+            logger.info(f"Suite '{suite.name}' completed. Aggregates: {result.aggregates}")
+        else:
+            _run_single_task(kwargs)
+    finally:
+        close_pools()
 
 
 def run() -> None:

eval_framework-0.8.0/src/eval_framework/tasks/__init__.py

@@ -0,0 +1,6 @@
+# Register all tasks on import
+from .task_names import register_all_tasks
+
+register_all_tasks()
+
+del register_all_tasks

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/base.py

@@ -10,12 +10,10 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar
 
 import iso639
 from datasets import DatasetDict, DownloadConfig, load_dataset
-from huggingface_hub import HfApi
-from huggingface_hub.errors import RevisionNotFoundError
 from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
-from eval_framework.tasks.dataset_revisions import DatasetRevision
+from eval_framework.tasks.dataset_revisions import pinned_revision
 from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import Message, Role
 
@@ -93,7 +91,6 @@ class BaseTask[SubjectType](ABC):
     SAMPLE_SPLIT: str
     FEWSHOT_SPLIT: str
     SUBJECTS: list[SubjectType]
-    HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
     # The lock file this task resolves its pinned dataset revision from, keyed by ``DATASET_PATH``.
     # Each task sets this explicitly: point it at a lock file (e.g. ``HF_REVISIONS_LOCKFILE`` or a
@@ -118,15 +115,18 @@ class BaseTask[SubjectType](ABC):
         self.user_prompt_suffix: str | None = None
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
-        self._apply_hf_revision()
+        self.hf_revision: str | None = self._apply_hf_revision()
 
-    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> None:
-        # Precedence: CLI/config override > class HF_REVISION > task-dataset-revisions.json pin.
-        # Applied once at instance creation; not refreshed if the pin file changes mid-run.
+    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
+        # Precedence: CLI/config override > REVISION_LOCKFILE pin.
+        # Tasks without a Hugging Face dataset set REVISION_LOCKFILE to None and are not pinned.
         if custom_hf_revision:
-            self.HF_REVISION = custom_hf_revision
-        elif self.HF_REVISION is None and (pinned := DatasetRevision.pinned_revision(self.__class__.__name__)):
-            self.HF_REVISION = pinned
+            hf_revision = custom_hf_revision
+        elif self.REVISION_LOCKFILE is not None:
+            hf_revision = pinned_revision(self.REVISION_LOCKFILE, self.DATASET_PATH)
+        else:
+            hf_revision = None
+        return hf_revision
 
     @classmethod
     def with_overwrite(
@@ -148,7 +148,7 @@ class BaseTask[SubjectType](ABC):
             logger.info(f"Setting SUBJECTS to `{filtered_subjects}` for the task {instance.__class__.__name__}")
             instance.SUBJECTS = filtered_subjects  # type: ignore[assignment]
 
-        instance._apply_hf_revision(custom_hf_revision)
+        instance.hf_revision = instance._apply_hf_revision(custom_hf_revision)
 
         return instance
 
@@ -187,29 +187,14 @@ class BaseTask[SubjectType](ABC):
             return custom_subjects  # type: ignore[return-value]
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
-        # Check if the HF_REVISION is valid before loading the dataset
-        if self.HF_REVISION:
-            try:
-                _ = HfApi().dataset_info(repo_id=kwargs["path"], revision=self.HF_REVISION, timeout=100.0)
-            except Exception as e:
-                if isinstance(e, RevisionNotFoundError):
-                    raise e
-
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
         download_config = DownloadConfig(cache_dir=cache_dir, max_retries=5)
-        try:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=cache_dir,
-                download_config=download_config,
-            )
-        except Exception:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=f"{Path.home()}/.cache/eval-framework",
-            )
+        return load_dataset(
+            **kwargs,
+            revision=self.hf_revision,
+            cache_dir=cache_dir,
+            download_config=download_config,
+        )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
         dataset = {}

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/copa.py

@@ -17,6 +17,8 @@ class COPAEvalHarness(BaseTask[str]):
     This version uses samples from the validation split as evaluation examples (same as lm-eval-harness).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "COPAEvalHarness"
     DATASET_PATH = "aps/super_glue"
     SAMPLE_SPLIT = "validation"  # 100 examples (same split as lm-eval)
@@ -88,6 +90,7 @@ class COPA(COPAEvalHarness):
 
 
 class COPA_IDKEvalHarness(COPAEvalHarness):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "COPA_IDKEvalHarness"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/csqa.py

@@ -13,6 +13,8 @@ from eval_framework.tasks.utils import get_n_letters
 class CommonsenseQACloze(BaseTask[str]):
     """CommonsenseQA dataset: https://huggingface.co/datasets/tau/commonsense_qa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQACloze"
     DATASET_PATH = "tau/commonsense_qa"
     SAMPLE_SPLIT = "validation"
@@ -53,6 +55,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
     Scores loglikelihood over the full correct choice text; includes bits-per-byte.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAFullTextCloze"
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
 
@@ -65,6 +69,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
 class CommonsenseQAMC(CommonsenseQACloze):
     """Multiple-choice variant of CommonsenseQA where the model selects a letter (A-E)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAMC"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/drop.py

@@ -75,6 +75,8 @@ class DropCompletion(BaseTask[str]):
     Uses DROP F1 and exact match. Stop at new paragraph or repeated prefixes.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCompletion"
     DATASET_PATH = "EleutherAI/drop"
     SAMPLE_SPLIT = "validation"
@@ -163,6 +165,8 @@ class DropCompletion_OLMES(DropCompletion):
 class DropMC(BaseTask[str]):
     """Multiple-choice variant using allenai/drop-gen2mc (passage_original, question_original, choices, answerKey)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropMC"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"
@@ -233,6 +237,8 @@ class DropCloze(BaseTask[str]):
     Includes BitsPerByte on the correct choice.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCloze"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -22,6 +22,8 @@ logger = logging.getLogger(__name__)
 class GPQA(BaseTask[str]):
     """GPQA dataset: https://huggingface.co/datasets/Idavidrein/gpqa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GPQA"
     DATASET_PATH = "Idavidrein/gpqa"
     SAMPLE_SPLIT = "train"
@@ -172,6 +174,7 @@ class GPQA_IDK(GPQA):
 
 
 class GPQA_COT(GPQA):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GPQA_COT"
     RESPONSE_TYPE = ResponseType.COMPLETION
     METRICS = [AccuracyCompletion]

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -28,6 +28,8 @@ class HumanEvalMetricContext(BaseMetricContext):
 class HumanEval(BaseTask[str]):
     """HumanEval dataset: https://huggingface.co/datasets/openai/openai_humaneval/"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval"
     DATASET_PATH = "openai/openai_humaneval"
     SAMPLE_SPLIT = "test"
@@ -128,6 +130,7 @@ class HumanEval_OLMES(HumanEval):
 
 class HumanEvalInstruct(HumanEval):
     # See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/humaneval/humaneval_instruct.yaml
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Human Eval Instruct"
     CUE_PREFIX = "Here is the completed function:\n```python\n"
 

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -482,6 +482,8 @@ class MATH500(MATHReasoning):
 class MATH(MATHReasoning):
     """MATH dataset: https://huggingface.co/datasets/EleutherAI/hendrycks_math"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Math"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -559,6 +561,8 @@ class MATHMinervaEvalHarness(MATHReasoning):
     Metrics: Exact Match, Exact Match (Flex) via MathMinervaCompletion.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinervaEvalHarness"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -595,6 +599,8 @@ class MATHMinerva(MATHMinervaEvalHarness):
     "(The )Final Answer: The (final )answer is ...( I hope it is correct.)", where parentheses are optional.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinerva"
     METRICS = [MathMinervaCompletionRelaxed]
 
@@ -610,6 +616,8 @@ class MATH500Minerva(MATHMinerva):
     Uses HuggingFaceH4/MATH-500 which has a single 'default' config (no subject splits).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATH500Minerva"
     DATASET_PATH = "HuggingFaceH4/MATH-500"
     SAMPLE_SPLIT = "test"
@@ -622,6 +630,7 @@ class MATH500Minerva(MATHMinerva):
 
 
 class MATHLvl5(MATH):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Math Lvl 5"
 
     def _load_dataset(self, subject: SubjectType) -> None:

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -33,6 +33,7 @@ class MBPP(BaseTask[str]):
 
     NAME = "MBPP"
     DATASET_PATH = "google-research-datasets/mbpp"
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/medqa.py

@@ -17,6 +17,8 @@ from eval_framework.tasks.utils import get_n_letters
 class MedQACloze(BaseTask[str]):
     """MedQA cloze (loglikelihood over choice text)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQACloze"
     DATASET_PATH = "davidheineman/medqa-en"
     SAMPLE_SPLIT = "test"
@@ -53,6 +55,8 @@ class MedQACloze(BaseTask[str]):
 class MedQAMC(MedQACloze):
     """MedQA multiple choice (loglikelihood over A/B/C/D/...)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQAMC"
 
     def __init__(self, num_fewshot: int = 0) -> None:

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -82,11 +82,13 @@ class _NaturalQsOpenChoice_Base(BaseTask[str]):
 
 
 class NaturalQsOpenCloze(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenCloze"
     TASK_STYLER = ClozeStyle()
 
 
 class NaturalQsOpenMC(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenMC"
     TASK_STYLER = MCStyle()
 

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/piqa.py

@@ -9,18 +9,17 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
+from eval_framework.tasks.dataset_revisions import FROZEN_HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class PIQA(BaseTask[str]):
     """PIQA dataset: https://huggingface.co/datasets/ybisk/piqa"""
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA"
     DATASET_PATH = "ybisk/piqa"
-    HF_REVISION = "6b3aceb3276e5ab7e51895d73151a718690af38c"
     SAMPLE_SPLIT = "validation"  # 1838 examples (same split as lm-eval)
     FEWSHOT_SPLIT = "test"  # 3084 examples
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
@@ -54,7 +53,7 @@ class PIQA_OLMES(PIQA):
     loglikelihood over " A"/" B".
     """
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA_OLMES"
     SAMPLE_SPLIT = "train"  # Use train split (largest) to best match OLMES, which evaluates all splits
@@ -79,7 +78,7 @@ class PIQA_OLMES(PIQA):
 
 
 class PIQA_IDK(PIQA):
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
     NAME = "PIQA_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/sciq.py

@@ -95,6 +95,7 @@ class SCIQ_OLMES(SCIQ):
 
 
 class SCIQ_IDK(SCIQ):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "SciQ_IDK"
     METRICS = [
         AccuracyLoglikelihood,
@@ -137,6 +138,7 @@ class SCIQEvalHarness(SCIQ):
 
 
 class SCIQEvalHarness_IDK(SCIQEvalHarness):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "SciQ Eval Harness_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.4 → eval_framework-0.8.0}/src/eval_framework/tasks/benchmarks/social_iqa.py

@@ -151,6 +151,8 @@ class SocialIQACloze(BaseTask[str]):
     Social IQA cloze: loglikelihood over full answer text.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "SocialIQACloze"
     DATASET_PATH = SOCIAL_I_QA_DATASET_PATH
     SAMPLE_SPLIT = "validation"