eval-framework 0.6.3__tar.gz → 0.7.2__tar.gz

{eval_framework-0.6.3 → eval_framework-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.6.3
+Version: 0.7.2
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -215,7 +215,6 @@ Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.0,<6
-Requires-Dist: sacrebleu>=2.6.0,<3
 Requires-Dist: pycountry>=26.2.16,<27
 Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
@@ -231,7 +230,7 @@ Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.4.20
-Requires-Dist: wandb>=0.28.0,<1
+Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
@@ -241,10 +240,10 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.5,<2 ; extra == 'mistral'
+Requires-Dist: mistral-common>=1.11.6,<2 ; extra == 'mistral'
 Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.109.1,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -523,8 +522,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.3 → eval_framework-0.7.2}/README.md

@@ -254,8 +254,10 @@ If you use `eval-framework` in your research, please cite:
 
 ```bibtex
 @software{eval_framework,
+  author={Aleph Alpha Research},
   title={Aleph Alpha Eval Framework},
-  year={2025},
+  year={2026},
+  version = {x.y.z},
   url={https://github.com/Aleph-Alpha-Research/eval-framework}
 }
 ```

{eval_framework-0.6.3 → eval_framework-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.6.3"
+version = "0.7.2"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -22,7 +22,6 @@ dependencies = [
   "xmltodict>=1.0.4,<1.1",
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.0,<6",
-  "sacrebleu>=2.6.0,<3",
   "pycountry>=26.2.16,<27",
   "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
@@ -38,7 +37,7 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.4.20",
-  "wandb>=0.28.0,<1",
+  "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
   "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
@@ -55,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=1.109.1,<3",
+  "openai>=2.46.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -70,7 +69,7 @@ vllm = [
   "torch>=2.5,<3"
 ]
 mistral = [
-  "mistral-common>=1.11.5,<2",
+  "mistral-common>=1.11.6,<2",
   "huggingface-hub>=0.36.2,<0.37",
   "eval_framework[vllm]",
 ]
@@ -91,16 +90,16 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.2.0,<3",
+  "mypy>=2.3.0,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
   "types-pyyaml>=6.0.12.20260518,<7",
-  "types-python-dateutil>=2.9.0.20260518,<3",
-  "types-requests>=2.33.0.20260518,<3",
+  "types-python-dateutil>=2.9.0.20260716,<3",
+  "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.8.0,<7",
-  "ruff>=0.15.20",
+  "ruff>=0.15.22",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +108,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.28,<0.11.29"]
+requires = ["uv_build>=0.11.29,<0.11.30"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/context/determined.py

@@ -30,6 +30,7 @@ class TaskArgs(BaseModel):
     judge_model_args: dict[str, Any] = {}
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
     perturbation_config: PerturbationConfig | None = None
     repeats: int | None = None
 
@@ -108,6 +109,7 @@ class DeterminedContext(EvalContext):
             "task_subjects",
             "batch_size",
             "hf_revision",
+            "user_prompt_suffix",
             "judge_model_name",
             "judge_model_args",
             "perturbation_config",
@@ -121,6 +123,9 @@ class DeterminedContext(EvalContext):
         # Hyperparameters take precedence over core context
         llm_name = self.hparams.llm_name or self.llm_name
         judge_model_name = self.hparams.task_args.judge_model_name or self.judge_model_name
+        user_prompt_suffix = self.hparams.task_args.user_prompt_suffix
+        if user_prompt_suffix is None:
+            user_prompt_suffix = self.user_prompt_suffix
 
         llm_class = _load_model(llm_name, models_path=self.models_path)
         llm_judge_class: type[BaseLLM] | None = (
@@ -139,6 +144,7 @@ class DeterminedContext(EvalContext):
             task_name=self.hparams.task_args.task_name,
             task_subjects=self.hparams.task_args.task_subjects,
             hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
             perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
             output_dir=self.hparams.output_dir,
             llm_judge_class=llm_judge_class,

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/context/eval.py

@@ -57,6 +57,7 @@ class EvalContext(AbstractContextManager):
         task_name: str | None = None,
         task_subjects: list[str] | None = None,
         hf_revision: str | None = None,
+        user_prompt_suffix: str | None = None,
         output_dir: Path | None = None,
         wandb_project: str | None = None,
         wandb_entity: str | None = None,
@@ -86,6 +87,7 @@ class EvalContext(AbstractContextManager):
         self.task_name = task_name
         self.task_subjects = task_subjects
         self.hf_revision = hf_revision
+        self.user_prompt_suffix = user_prompt_suffix
         self.output_dir = output_dir
         self.wandb_project = wandb_project
         self.wandb_entity = wandb_entity

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/context/local.py

@@ -52,6 +52,7 @@ class LocalContext(EvalContext):
             task_name=self.task_name,
             task_subjects=self.task_subjects,
             hf_revision=self.hf_revision,
+            user_prompt_suffix=self.user_prompt_suffix,
             output_dir=self.output_dir,
             hf_upload_dir=self.hf_upload_dir,
             hf_upload_repo=self.hf_upload_repo,

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/metrics/completion/language_checker.py

@@ -41,6 +41,9 @@ class LanguageConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         completion_language = response.get_completion_language()
         target_language = response.get_instruction_language()
         if completion_language == target_language == "":
@@ -57,6 +60,9 @@ class LanguageRawConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
+        if not response.raw_completion:
+            return []  # No completion means no language to detect, so it is excluded from aggregation
+
         raw_completion_language = response.get_raw_completion_language()
         target_language = response.get_instruction_language()
 

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/response_generator.py

@@ -66,10 +66,14 @@ class ResponseGenerator:
                 config.num_fewshot,
                 config.task_subjects,
                 config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
         else:
             self.task = registry()[config.task_name].create(
-                config.num_fewshot, config.task_subjects, config.hf_revision
+                config.num_fewshot,
+                config.task_subjects,
+                config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
 
         self.response_type = self.task.get_response_type()
@@ -345,9 +349,10 @@ class ResponseGenerator:
             "llm_args",
             "perturbation_config",
             "repeats",
+            "user_prompt_suffix",
         ]
         for key in keys:
-            if loaded_metadata[key] != current_metadata[key]:
+            if loaded_metadata.get(key) != current_metadata[key]:
                 raise ValueError(f"Existing metadata does not match current metadata for {key}.")
 
     def __del__(self) -> None:

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/run.py

@@ -150,6 +150,13 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="A tag name, a branch name, or commit hash for the task HF dataset.",
     )
+    parser.add_argument(
+        "--user-prompt-suffix",
+        type=str,
+        required=False,
+        default=None,
+        help="Text appended verbatim to the evaluated user prompt before formatting.",
+    )
     parser.add_argument(
         "--judge-models",
         type=Path,
@@ -343,6 +350,7 @@ def _run_single_task(kwargs: dict) -> None:
         task_name=kwargs["task_name"],
         task_subjects=kwargs["task_subjects"],
         hf_revision=kwargs["hf_revision"],
+        user_prompt_suffix=kwargs.get("user_prompt_suffix"),
         output_dir=kwargs["output_dir"],
         wandb_project=kwargs["wandb_project"],
         wandb_entity=kwargs["wandb_entity"],

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/suite.py

@@ -34,6 +34,7 @@ _EVAL_CONFIG_FIELDS = {
     "batch_size",
     "task_subjects",
     "hf_revision",
+    "user_prompt_suffix",
 }
 
 _HYPERPARAM_FIELDS = _LLM_ARG_FIELDS | _EVAL_CONFIG_FIELDS
@@ -100,6 +101,7 @@ class TaskSuite(BaseModel):
     batch_size: int | None = None
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
 
     @model_validator(mode="after")
     def validate_suite(self) -> Self:

eval_framework-0.7.2/src/eval_framework/tasks/__init__.py

@@ -0,0 +1,6 @@
+# Register all tasks on import
+from .task_names import register_all_tasks
+
+register_all_tasks()
+
+del register_all_tasks

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/base.py

@@ -10,12 +10,10 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar
 
 import iso639
 from datasets import DatasetDict, DownloadConfig, load_dataset
-from huggingface_hub import HfApi
-from huggingface_hub.errors import RevisionNotFoundError
 from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
-from eval_framework.tasks.dataset_revisions import DatasetRevision
+from eval_framework.tasks.dataset_revisions import pinned_revision
 from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import Message, Role
 
@@ -93,7 +91,6 @@ class BaseTask[SubjectType](ABC):
     SAMPLE_SPLIT: str
     FEWSHOT_SPLIT: str
     SUBJECTS: list[SubjectType]
-    HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
     # The lock file this task resolves its pinned dataset revision from, keyed by ``DATASET_PATH``.
     # Each task sets this explicitly: point it at a lock file (e.g. ``HF_REVISIONS_LOCKFILE`` or a
@@ -115,23 +112,35 @@ class BaseTask[SubjectType](ABC):
 
     def __init__(self, num_fewshot: int = 0) -> None:
         self.num_fewshot = num_fewshot
+        self.user_prompt_suffix: str | None = None
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
-        self._apply_hf_revision()
+        self.hf_revision: str | None = self._apply_hf_revision()
 
-    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> None:
-        # Precedence: CLI/config override > class HF_REVISION > task-dataset-revisions.json pin.
-        # Applied once at instance creation; not refreshed if the pin file changes mid-run.
+    def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
+        # Precedence: CLI/config override > REVISION_LOCKFILE pin.
+        # Tasks without a Hugging Face dataset set REVISION_LOCKFILE to None and are not pinned.
         if custom_hf_revision:
-            self.HF_REVISION = custom_hf_revision
-        elif self.HF_REVISION is None and (pinned := DatasetRevision.pinned_revision(self.__class__.__name__)):
-            self.HF_REVISION = pinned
+            hf_revision = custom_hf_revision
+        elif self.REVISION_LOCKFILE is not None:
+            hf_revision = pinned_revision(self.REVISION_LOCKFILE, self.DATASET_PATH)
+        else:
+            hf_revision = None
+        return hf_revision
 
     @classmethod
     def with_overwrite(
-        cls, num_fewshot: int, *, custom_subjects: list[str] | None, custom_hf_revision: str | None
+        cls,
+        num_fewshot: int,
+        *,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
+        if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
+            raise ValueError("user_prompt_suffix is only supported for completion tasks.")
+        instance.user_prompt_suffix = user_prompt_suffix
 
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
         filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
@@ -139,7 +148,7 @@ class BaseTask[SubjectType](ABC):
             logger.info(f"Setting SUBJECTS to `{filtered_subjects}` for the task {instance.__class__.__name__}")
             instance.SUBJECTS = filtered_subjects  # type: ignore[assignment]
 
-        instance._apply_hf_revision(custom_hf_revision)
+        instance.hf_revision = instance._apply_hf_revision(custom_hf_revision)
 
         return instance
 
@@ -178,29 +187,14 @@ class BaseTask[SubjectType](ABC):
             return custom_subjects  # type: ignore[return-value]
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
-        # Check if the HF_REVISION is valid before loading the dataset
-        if self.HF_REVISION:
-            try:
-                _ = HfApi().dataset_info(repo_id=kwargs["path"], revision=self.HF_REVISION, timeout=100.0)
-            except Exception as e:
-                if isinstance(e, RevisionNotFoundError):
-                    raise e
-
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
         download_config = DownloadConfig(cache_dir=cache_dir, max_retries=5)
-        try:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=cache_dir,
-                download_config=download_config,
-            )
-        except Exception:
-            return load_dataset(
-                **kwargs,
-                revision=self.HF_REVISION,
-                cache_dir=f"{Path.home()}/.cache/eval-framework",
-            )
+        return load_dataset(
+            **kwargs,
+            revision=self.hf_revision,
+            cache_dir=cache_dir,
+            download_config=download_config,
+        )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
         dataset = {}
@@ -241,7 +235,7 @@ class BaseTask[SubjectType](ABC):
 
     def _get_messages(self, item: dict[str, Any]) -> list[Message]:
         example_messages = self._get_example_messages(item)
-        instruction_message = self._get_instruction_messages(item)
+        instruction_message = self._apply_user_prompt_suffix(self._get_instruction_messages(item))
         cue_text = self._get_cue_text(item)
         cue_message = [Message(role=Role.ASSISTANT, content=cue_text)] if cue_text else []
         messages = example_messages + instruction_message + cue_message
@@ -254,6 +248,18 @@ class BaseTask[SubjectType](ABC):
             return [Message(role=Role.SYSTEM, content=system_prompt_text)] + messages
         return messages
 
+    def _apply_user_prompt_suffix(self, instruction_messages: list[Message]) -> list[Message]:
+        """Append the configured suffix verbatim to the evaluated user turn."""
+        if self.user_prompt_suffix is None:
+            return instruction_messages
+
+        for message in reversed(instruction_messages):
+            if message.role == Role.USER:
+                message.content = f"{message.content}{self.user_prompt_suffix}"
+                return instruction_messages
+
+        raise ValueError("Cannot append user_prompt_suffix: evaluated instruction contains no user message.")
+
     def _get_instruction_messages(self, item: dict[str, Any]) -> list[Message]:
         return [Message(role=Role.USER, content=self._get_instruction_text(item))]
 

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/copa.py

@@ -17,6 +17,8 @@ class COPAEvalHarness(BaseTask[str]):
     This version uses samples from the validation split as evaluation examples (same as lm-eval-harness).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "COPAEvalHarness"
     DATASET_PATH = "aps/super_glue"
     SAMPLE_SPLIT = "validation"  # 100 examples (same split as lm-eval)
@@ -88,6 +90,7 @@ class COPA(COPAEvalHarness):
 
 
 class COPA_IDKEvalHarness(COPAEvalHarness):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "COPA_IDKEvalHarness"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/csqa.py

@@ -13,6 +13,8 @@ from eval_framework.tasks.utils import get_n_letters
 class CommonsenseQACloze(BaseTask[str]):
     """CommonsenseQA dataset: https://huggingface.co/datasets/tau/commonsense_qa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQACloze"
     DATASET_PATH = "tau/commonsense_qa"
     SAMPLE_SPLIT = "validation"
@@ -53,6 +55,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
     Scores loglikelihood over the full correct choice text; includes bits-per-byte.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAFullTextCloze"
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
 
@@ -65,6 +69,8 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
 class CommonsenseQAMC(CommonsenseQACloze):
     """Multiple-choice variant of CommonsenseQA where the model selects a letter (A-E)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAMC"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/drop.py

@@ -75,6 +75,8 @@ class DropCompletion(BaseTask[str]):
     Uses DROP F1 and exact match. Stop at new paragraph or repeated prefixes.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCompletion"
     DATASET_PATH = "EleutherAI/drop"
     SAMPLE_SPLIT = "validation"
@@ -163,6 +165,8 @@ class DropCompletion_OLMES(DropCompletion):
 class DropMC(BaseTask[str]):
     """Multiple-choice variant using allenai/drop-gen2mc (passage_original, question_original, choices, answerKey)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropMC"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"
@@ -233,6 +237,8 @@ class DropCloze(BaseTask[str]):
     Includes BitsPerByte on the correct choice.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCloze"
     DATASET_PATH = "allenai/drop-gen2mc"
     SAMPLE_SPLIT = "validation"

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -22,6 +22,8 @@ logger = logging.getLogger(__name__)
 class GPQA(BaseTask[str]):
     """GPQA dataset: https://huggingface.co/datasets/Idavidrein/gpqa"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GPQA"
     DATASET_PATH = "Idavidrein/gpqa"
     SAMPLE_SPLIT = "train"
@@ -172,6 +174,7 @@ class GPQA_IDK(GPQA):
 
 
 class GPQA_COT(GPQA):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GPQA_COT"
     RESPONSE_TYPE = ResponseType.COMPLETION
     METRICS = [AccuracyCompletion]

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -28,6 +28,8 @@ class HumanEvalMetricContext(BaseMetricContext):
 class HumanEval(BaseTask[str]):
     """HumanEval dataset: https://huggingface.co/datasets/openai/openai_humaneval/"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval"
     DATASET_PATH = "openai/openai_humaneval"
     SAMPLE_SPLIT = "test"
@@ -128,6 +130,7 @@ class HumanEval_OLMES(HumanEval):
 
 class HumanEvalInstruct(HumanEval):
     # See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/humaneval/humaneval_instruct.yaml
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Human Eval Instruct"
     CUE_PREFIX = "Here is the completed function:\n```python\n"
 

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/math_reasoning.py

@@ -482,6 +482,8 @@ class MATH500(MATHReasoning):
 class MATH(MATHReasoning):
     """MATH dataset: https://huggingface.co/datasets/EleutherAI/hendrycks_math"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Math"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -559,6 +561,8 @@ class MATHMinervaEvalHarness(MATHReasoning):
     Metrics: Exact Match, Exact Match (Flex) via MathMinervaCompletion.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinervaEvalHarness"
     DATASET_PATH = "EleutherAI/hendrycks_math"
     SAMPLE_SPLIT = "test"
@@ -595,6 +599,8 @@ class MATHMinerva(MATHMinervaEvalHarness):
     "(The )Final Answer: The (final )answer is ...( I hope it is correct.)", where parentheses are optional.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATHMinerva"
     METRICS = [MathMinervaCompletionRelaxed]
 
@@ -610,6 +616,8 @@ class MATH500Minerva(MATHMinerva):
     Uses HuggingFaceH4/MATH-500 which has a single 'default' config (no subject splits).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MATH500Minerva"
     DATASET_PATH = "HuggingFaceH4/MATH-500"
     SAMPLE_SPLIT = "test"
@@ -622,6 +630,7 @@ class MATH500Minerva(MATHMinerva):
 
 
 class MATHLvl5(MATH):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "Math Lvl 5"
 
     def _load_dataset(self, subject: SubjectType) -> None:

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -33,6 +33,7 @@ class MBPP(BaseTask[str]):
 
     NAME = "MBPP"
     DATASET_PATH = "google-research-datasets/mbpp"
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/medqa.py

@@ -17,6 +17,8 @@ from eval_framework.tasks.utils import get_n_letters
 class MedQACloze(BaseTask[str]):
     """MedQA cloze (loglikelihood over choice text)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQACloze"
     DATASET_PATH = "davidheineman/medqa-en"
     SAMPLE_SPLIT = "test"
@@ -53,6 +55,8 @@ class MedQACloze(BaseTask[str]):
 class MedQAMC(MedQACloze):
     """MedQA multiple choice (loglikelihood over A/B/C/D/...)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "MedQAMC"
 
     def __init__(self, num_fewshot: int = 0) -> None:

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -82,11 +82,13 @@ class _NaturalQsOpenChoice_Base(BaseTask[str]):
 
 
 class NaturalQsOpenCloze(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenCloze"
     TASK_STYLER = ClozeStyle()
 
 
 class NaturalQsOpenMC(_NaturalQsOpenChoice_Base):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "NaturalQsOpenMC"
     TASK_STYLER = MCStyle()
 

{eval_framework-0.6.3 → eval_framework-0.7.2}/src/eval_framework/tasks/benchmarks/piqa.py

@@ -9,18 +9,17 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
+from eval_framework.tasks.dataset_revisions import FROZEN_HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class PIQA(BaseTask[str]):
     """PIQA dataset: https://huggingface.co/datasets/ybisk/piqa"""
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA"
     DATASET_PATH = "ybisk/piqa"
-    HF_REVISION = "6b3aceb3276e5ab7e51895d73151a718690af38c"
     SAMPLE_SPLIT = "validation"  # 1838 examples (same split as lm-eval)
     FEWSHOT_SPLIT = "test"  # 3084 examples
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
@@ -54,7 +53,7 @@ class PIQA_OLMES(PIQA):
     loglikelihood over " A"/" B".
     """
 
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
 
     NAME = "PIQA_OLMES"
     SAMPLE_SPLIT = "train"  # Use train split (largest) to best match OLMES, which evaluates all splits
@@ -79,7 +78,7 @@ class PIQA_OLMES(PIQA):
 
 
 class PIQA_IDK(PIQA):
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+    REVISION_LOCKFILE = FROZEN_HF_REVISIONS_LOCKFILE
     NAME = "PIQA_IDK"
     METRICS = [
         AccuracyLoglikelihood,