eval-framework 0.5.3__tar.gz → 0.6.4__tar.gz

{eval_framework-0.5.3 → eval_framework-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.5.3
+Version: 0.6.4
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -217,7 +217,7 @@ Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.0,<6
 Requires-Dist: sacrebleu>=2.6.0,<3
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.9.4,<4
+Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
@@ -231,7 +231,7 @@ Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.4.20
-Requires-Dist: wandb>=0.27.2,<1
+Requires-Dist: wandb>=0.28.0,<1
 Requires-Dist: boto3>=1.43.19,<2
 Requires-Dist: numpy>=2.2.6
 Requires-Dist: antlr4-python3-runtime==4.11.0
@@ -240,11 +240,11 @@ Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,optional,mistral] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
-Requires-Dist: tensorboard==2.20.0 ; extra == 'determined'
-Requires-Dist: mistral-common>=1.11.3,<2 ; extra == 'mistral'
+Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
+Requires-Dist: mistral-common>=1.11.5,<2 ; extra == 'mistral'
 Requires-Dist: huggingface-hub>=0.36.2,<0.37 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.62,<3 ; extra == 'openai'
+Requires-Dist: openai>=1.109.1,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.5.3 → eval_framework-0.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.5.3"
+version = "0.6.4"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -24,7 +24,7 @@ dependencies = [
   "datasets>=5.0.0,<6",
   "sacrebleu>=2.6.0,<3",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.9.4,<4",
+  "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
@@ -38,7 +38,7 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.4.20",
-  "wandb>=0.27.2,<1",
+  "wandb>=0.28.0,<1",
   "boto3>=1.43.19,<2",
   "numpy>=2.2.6",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
@@ -51,11 +51,11 @@ dependencies = [
 # Model-specific extras
 determined = [
   "determined>=0.38.1,<0.39",
-  "tensorboard==2.20.0"
+  "tensorboard==2.21.0"
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=1.62,<3",
+  "openai>=1.109.1,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -70,7 +70,7 @@ vllm = [
   "torch>=2.5,<3"
 ]
 mistral = [
-  "mistral-common>=1.11.3,<2",
+  "mistral-common>=1.11.5,<2",
   "huggingface-hub>=0.36.2,<0.37",
   "eval_framework[vllm]",
 ]
@@ -91,7 +91,7 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.1.0,<3",
+  "mypy>=2.2.0,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
@@ -109,7 +109,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.24,<0.11.25"]
+requires = ["uv_build>=0.11.28,<0.11.29"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]
@@ -145,7 +145,6 @@ known-third-party = ["wandb"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]
-"tests/tests_eval_framework/tasks/benchmarks/test_mmlu_de.py" = ["E501"]
 
 [tool.mypy]
 plugins = "pydantic.mypy"

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/context/determined.py

@@ -30,6 +30,7 @@ class TaskArgs(BaseModel):
     judge_model_args: dict[str, Any] = {}
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
     perturbation_config: PerturbationConfig | None = None
     repeats: int | None = None
 
@@ -108,6 +109,7 @@ class DeterminedContext(EvalContext):
             "task_subjects",
             "batch_size",
             "hf_revision",
+            "user_prompt_suffix",
             "judge_model_name",
             "judge_model_args",
             "perturbation_config",
@@ -121,6 +123,9 @@ class DeterminedContext(EvalContext):
         # Hyperparameters take precedence over core context
         llm_name = self.hparams.llm_name or self.llm_name
         judge_model_name = self.hparams.task_args.judge_model_name or self.judge_model_name
+        user_prompt_suffix = self.hparams.task_args.user_prompt_suffix
+        if user_prompt_suffix is None:
+            user_prompt_suffix = self.user_prompt_suffix
 
         llm_class = _load_model(llm_name, models_path=self.models_path)
         llm_judge_class: type[BaseLLM] | None = (
@@ -139,6 +144,7 @@ class DeterminedContext(EvalContext):
             task_name=self.hparams.task_args.task_name,
             task_subjects=self.hparams.task_args.task_subjects,
             hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
             perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
             output_dir=self.hparams.output_dir,
             llm_judge_class=llm_judge_class,

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/context/eval.py

@@ -57,6 +57,7 @@ class EvalContext(AbstractContextManager):
         task_name: str | None = None,
         task_subjects: list[str] | None = None,
         hf_revision: str | None = None,
+        user_prompt_suffix: str | None = None,
         output_dir: Path | None = None,
         wandb_project: str | None = None,
         wandb_entity: str | None = None,
@@ -86,6 +87,7 @@ class EvalContext(AbstractContextManager):
         self.task_name = task_name
         self.task_subjects = task_subjects
         self.hf_revision = hf_revision
+        self.user_prompt_suffix = user_prompt_suffix
         self.output_dir = output_dir
         self.wandb_project = wandb_project
         self.wandb_entity = wandb_entity

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/context/local.py

@@ -52,6 +52,7 @@ class LocalContext(EvalContext):
             task_name=self.task_name,
             task_subjects=self.task_subjects,
             hf_revision=self.hf_revision,
+            user_prompt_suffix=self.user_prompt_suffix,
             output_dir=self.output_dir,
             hf_upload_dir=self.hf_upload_dir,
             hf_upload_repo=self.hf_upload_repo,

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/metrics/completion/code_assertion.py

@@ -15,7 +15,7 @@ class CodeCompletionAssertion(BaseMetric[Completion]):
         # this will always be a list, if return is "" this will be an empty list
         code = response.completion
         try:
-            output = run_python_code(code, image="python:3.12-slim")
+            output = run_python_code(code, image="python:3.12-slim", runtime_configs={"mem_limit": "512m"})
         except SandboxTimeoutError:
             # The submitted code timed out (e.g. an infinite loop) -- a failing sample, not an infra
             # problem.

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/response_generator.py

@@ -66,10 +66,14 @@ class ResponseGenerator:
                 config.num_fewshot,
                 config.task_subjects,
                 config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
         else:
             self.task = registry()[config.task_name].create(
-                config.num_fewshot, config.task_subjects, config.hf_revision
+                config.num_fewshot,
+                config.task_subjects,
+                config.hf_revision,
+                user_prompt_suffix=config.user_prompt_suffix,
             )
 
         self.response_type = self.task.get_response_type()
@@ -345,9 +349,10 @@ class ResponseGenerator:
             "llm_args",
             "perturbation_config",
             "repeats",
+            "user_prompt_suffix",
         ]
         for key in keys:
-            if loaded_metadata[key] != current_metadata[key]:
+            if loaded_metadata.get(key) != current_metadata[key]:
                 raise ValueError(f"Existing metadata does not match current metadata for {key}.")
 
     def __del__(self) -> None:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/run.py

@@ -150,6 +150,13 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help="A tag name, a branch name, or commit hash for the task HF dataset.",
     )
+    parser.add_argument(
+        "--user-prompt-suffix",
+        type=str,
+        required=False,
+        default=None,
+        help="Text appended verbatim to the evaluated user prompt before formatting.",
+    )
     parser.add_argument(
         "--judge-models",
         type=Path,
@@ -343,6 +350,7 @@ def _run_single_task(kwargs: dict) -> None:
         task_name=kwargs["task_name"],
         task_subjects=kwargs["task_subjects"],
         hf_revision=kwargs["hf_revision"],
+        user_prompt_suffix=kwargs.get("user_prompt_suffix"),
         output_dir=kwargs["output_dir"],
         wandb_project=kwargs["wandb_project"],
         wandb_entity=kwargs["wandb_entity"],

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/suite.py

@@ -34,6 +34,7 @@ _EVAL_CONFIG_FIELDS = {
     "batch_size",
     "task_subjects",
     "hf_revision",
+    "user_prompt_suffix",
 }
 
 _HYPERPARAM_FIELDS = _LLM_ARG_FIELDS | _EVAL_CONFIG_FIELDS
@@ -100,6 +101,7 @@ class TaskSuite(BaseModel):
     batch_size: int | None = None
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
+    user_prompt_suffix: str | None = None
 
     @model_validator(mode="after")
     def validate_suite(self) -> Self:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/base.py

@@ -95,6 +95,12 @@ class BaseTask[SubjectType](ABC):
     SUBJECTS: list[SubjectType]
     HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
+    # The lock file this task resolves its pinned dataset revision from, keyed by ``DATASET_PATH``.
+    # Each task sets this explicitly: point it at a lock file (e.g. ``HF_REVISIONS_LOCKFILE`` or a
+    # frozen one), or ``None`` to opt out of pinning. Deliberately not defaulted so it is never
+    # inherited implicitly (a subclass in another package would otherwise resolve the wrong file).
+    REVISION_LOCKFILE: Path | None
+
     # Words in _get_instruction_text() not to be perturbed. List of words is case insensitive. No special characters
     # or whitespace should be included.
     PERTURBATION_UNMODIFIABLE_WORDS: list[str] | None
@@ -109,6 +115,7 @@ class BaseTask[SubjectType](ABC):
 
     def __init__(self, num_fewshot: int = 0) -> None:
         self.num_fewshot = num_fewshot
+        self.user_prompt_suffix: str | None = None
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
         self._apply_hf_revision()
@@ -123,9 +130,17 @@ class BaseTask[SubjectType](ABC):
 
     @classmethod
     def with_overwrite(
-        cls, num_fewshot: int, *, custom_subjects: list[str] | None, custom_hf_revision: str | None
+        cls,
+        num_fewshot: int,
+        *,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
+        if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
+            raise ValueError("user_prompt_suffix is only supported for completion tasks.")
+        instance.user_prompt_suffix = user_prompt_suffix
 
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
         filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
@@ -235,7 +250,7 @@ class BaseTask[SubjectType](ABC):
 
     def _get_messages(self, item: dict[str, Any]) -> list[Message]:
         example_messages = self._get_example_messages(item)
-        instruction_message = self._get_instruction_messages(item)
+        instruction_message = self._apply_user_prompt_suffix(self._get_instruction_messages(item))
         cue_text = self._get_cue_text(item)
         cue_message = [Message(role=Role.ASSISTANT, content=cue_text)] if cue_text else []
         messages = example_messages + instruction_message + cue_message
@@ -248,6 +263,18 @@ class BaseTask[SubjectType](ABC):
             return [Message(role=Role.SYSTEM, content=system_prompt_text)] + messages
         return messages
 
+    def _apply_user_prompt_suffix(self, instruction_messages: list[Message]) -> list[Message]:
+        """Append the configured suffix verbatim to the evaluated user turn."""
+        if self.user_prompt_suffix is None:
+            return instruction_messages
+
+        for message in reversed(instruction_messages):
+            if message.role == Role.USER:
+                message.content = f"{message.content}{self.user_prompt_suffix}"
+                return instruction_messages
+
+        raise ValueError("Cannot append user_prompt_suffix: evaluated instruction contains no user message.")
+
     def _get_instruction_messages(self, item: dict[str, Any]) -> list[Message]:
         return [Message(role=Role.USER, content=self._get_instruction_text(item))]
 

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/arc.py

@@ -9,12 +9,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class ARC(BaseTask[str]):
     """ARC dataset: https://huggingface.co/datasets/allenai/ai2_arc"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC"
     DATASET_PATH = "allenai/ai2_arc"
     SAMPLE_SPLIT = "test"
@@ -55,6 +58,8 @@ class ARC_OLMES(ARC):
     loglikelihood over " A"/" B"/ etc.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
@@ -76,6 +81,7 @@ class ARC_OLMES(ARC):
 
 
 class ARC_IDK(ARC):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "ARC_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/arc_de.py

@@ -6,12 +6,15 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
 class ARC_DE(BaseTask[str]):
     """ARC-DE dataset: https://huggingface.co/datasets/LeoLM/ArcChallenge_de"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "ARC German"
     DATASET_PATH = "LeoLM/ArcChallenge_de"
     SAMPLE_SPLIT = "test"

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/bigcodebench.py

@@ -16,6 +16,7 @@ from eval_framework.tasks.base import (
     Sample,
     SubjectType,
 )
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import (
     BIG_CODE_BENCH_PACKAGE_MAPPING,
     CallableSerializer,
@@ -123,6 +124,8 @@ class BigCodeBench_OLMES(BigCodeBench):
     then compute pass@1 over the 5 samples per problem (post-process if needed).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "BigCodeBench_OLMES"
     SAMPLE_SPLIT = "v0.1.2"
     FEWSHOT_SPLIT = "v0.1.2"

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/copa.py

@@ -8,6 +8,7 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -51,6 +52,8 @@ class COPA_OLMES(COPAEvalHarness):
     space-prefixed labels (" A.", " B."); loglikelihood over " A"/" B".
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "COPA_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/csqa.py

@@ -6,6 +6,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -86,6 +87,8 @@ class CommonsenseQAMC_OLMES(CommonsenseQAMC):
     CommonsenseQA MC with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "CommonsenseQAMC_OLMES"
     SAMPLE_SPLIT = "train"  # Use train split (largest) to best match OLMES, which evaluates all splits
     FEWSHOT_SPLIT = "train"

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/drop.py

@@ -10,6 +10,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 )
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 
@@ -139,6 +140,8 @@ class DropCompletion(BaseTask[str]):
 class DropCompletion_OLMES(DropCompletion):
     """DropCompletion matching OLMES, using train split for fewshot and max tokens 100."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropCompletion_OLMES"
     FEWSHOT_SPLIT = "train"
 
@@ -210,6 +213,8 @@ class DropMC_OLMES(DropMC):
     DropMC with OLMES-style prompt: space before each label in the prompt (" A.", " B.", ...).
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "DropMC_OLMES"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/global_mmlu.py

@@ -9,6 +9,7 @@ from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.tasks.base import RANDOM_SEED, BaseTask, Language, ResponseType
 from eval_framework.tasks.benchmarks.mmlu import MMLU_SUBJECTS
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 GLOBAL_MMLU_LANGUAGES = ["fr", "de", "es", "it", "pt", "ar"]
@@ -469,6 +470,8 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
     https://github.com/aisingapore/SEA-HELM/blob/main/seahelm_tasks/knowledge/global_mmlu/abstract_algebra/config.yaml
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GlobalMMLU"
     DATASET_PATH = "CohereLabs/Global-MMLU"
     SAMPLE_SPLIT = "test"
@@ -534,6 +537,7 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
 
 
 class GlobalMMLU_German(GlobalMMLU):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GlobalMMLU_German"
     SUBJECTS = [("de", subject) for subject in MMLU_SUBJECTS]
     LANGUAGE = Language.DEU

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/goldenswag.py

@@ -8,12 +8,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.benchmarks.hellaswag import HELLASWAG
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class GOLDENSWAG(HELLASWAG):
     """GoldenSwag dataset: https://huggingface.co/datasets/PleIAs/GoldenSwag
     available data set sections: validation"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GoldenSwag"
     DATASET_PATH = "PleIAs/GoldenSwag"
     SAMPLE_SPLIT = "validation"
@@ -21,6 +24,7 @@ class GOLDENSWAG(HELLASWAG):
 
 
 class GOLDENSWAG_IDK(GOLDENSWAG):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GoldenSwag_IDK"
     METRICS = [
         AccuracyLoglikelihood,

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -13,6 +13,7 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.utils import get_n_letters
 
 logger = logging.getLogger(__name__)
@@ -126,6 +127,8 @@ class GPQA_OLMES(GPQA):
     (" A.", " B.", " C.", " D."); loglikelihood over " A"/" B"/" C"/" D".
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "GPQA_OLMES"
 
     def _get_possible_completions_marked(self, item: dict[str, Any]) -> tuple[list[str], int]:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/gsm8k.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion, AccuracyCompletionOLMES
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
 
 logger = logging.getLogger(__name__)
@@ -155,6 +156,7 @@ class GSM8K(GSM8KEvalHarness):
 
 
 class GSM8K_OLMES(GSM8K):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GSM8K_OLMES"
     METRICS = [AccuracyCompletionOLMES]
 
@@ -219,6 +221,7 @@ class GSM8K_OLMES(GSM8K):
 
 
 class GSM8KBPB(GSM8K_OLMES):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "GSM8KBPB"
     TASK_STYLER = BPBStyle(cue_text="Answer:", leading_space_continuations=False)
 

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/hellaswag.py

@@ -10,12 +10,15 @@ from eval_framework.metrics.loglikelihood.confidence_weighted_accuracy import Co
 from eval_framework.metrics.loglikelihood.dcs import DistributionalCorrectnessScore
 from eval_framework.metrics.loglikelihood.ternary import TernaryScore
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class HELLASWAG(BaseTask[str]):
     """Hellaswag dataset: https://huggingface.co/datasets/Rowan/hellaswag
     available data set sections: train, validation, test"""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "HellaSwag"
     DATASET_PATH = "Rowan/hellaswag"
     SAMPLE_SPLIT = "validation"
@@ -49,6 +52,7 @@ class HELLASWAG(BaseTask[str]):
 
 
 class HELLASWAG_OLMES(HELLASWAG):
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
     NAME = "HellaSwag_OLMES"
     SAMPLE_SPLIT = "train"
 

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -4,6 +4,7 @@ from eval_framework.metrics.completion.code_assertion import CodeCompletionAsser
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
 from eval_framework.shared.types import BaseMetricContext
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType, Sample
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 CODE_TO_EXECUTE = """
 {start_of_code}
@@ -84,6 +85,8 @@ class HumanEvalBPB(HumanEval):
     Reports bits-per-byte on the reference completion.
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval BPB"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [BitsPerByteLoglikelihood]
@@ -107,6 +110,8 @@ class HumanEval_OLMES(HumanEval):
         llm_args: {sampling_params: {temperature: 0.6, top_p: 0.6}}
     """
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "Human Eval OLMES"
 
     def __init__(self, num_fewshot: int = 3) -> None:

{eval_framework-0.5.3 → eval_framework-0.6.4}/src/eval_framework/tasks/benchmarks/ifeval.py

@@ -3,11 +3,14 @@ from typing import Any
 from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
 from eval_framework.metrics.completion.language_checker import LanguageRawConsistencyChecker
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
+from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
 class IFEval(BaseTask[str]):
     """IFEval: Instruction Following Eval (https://arxiv.org/pdf/2311.07911)."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "IFEval"
     DATASET_PATH = "google/IFEval"
     SAMPLE_SPLIT = "train"
@@ -73,6 +76,8 @@ class IFEvalFiSv(IFEval):
 class IFEvalDe(IFEval):
     """German version of the Instruction Following Evaluation (IFEval) benchmark."""
 
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
     NAME = "IFEval German"
     DATASET_PATH = "jzhang86/de_ifeval"
     SUBJECTS = [NO_SUBJECT]