eval-framework 0.9.1__tar.gz → 0.10.0__tar.gz

{eval_framework-0.9.1 → eval_framework-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.9.1
+Version: 0.10.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -208,6 +208,8 @@ Classifier: Operating System :: OS Independent
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
@@ -216,12 +218,11 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.1,<4
+Requires-Dist: nltk>=3.10.2,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
-Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
 Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
@@ -232,7 +233,7 @@ Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
-Requires-Dist: numpy>=2.5.1
+Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
@@ -248,7 +249,7 @@ Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
 Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
-Requires-Python: >=3.12, <3.13
+Requires-Python: >=3.12, <3.15
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework
 Provides-Extra: accelerate
 Provides-Extra: all

{eval_framework-0.9.1 → eval_framework-0.10.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [project]
 name = "eval-framework"
-version = "0.9.1"
+version = "0.10.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
-requires-python = ">=3.12,<3.13"
+requires-python = ">=3.12,<3.15"
 authors = [
   { name = "Aleph Alpha Research" }
 ]
@@ -13,6 +13,8 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
   "Programming Language :: Python :: 3 :: Only",
   "Topic :: Software Development :: Libraries",
   "Typing :: Typed",
@@ -23,12 +25,11 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.1,<4",
+  "nltk>=3.10.2,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
-  "spacy>=3.8.14,<4",
   "jsonschema>=4.26.0,<5",
   "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
@@ -39,7 +40,7 @@ dependencies = [
   "python-iso639>=2026.7.23",
   "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
-  "numpy>=2.5.1",
+  "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
@@ -89,10 +90,10 @@ dev = [
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
   "types-pyyaml>=6.0.12.20260724,<7",
-  "types-python-dateutil>=2.9.0.20260716,<3",
+  "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.1",
+  "ruff>=0.16.2",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -137,6 +138,16 @@ known-third-party = ["wandb"]
 
 [tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]
+# Prompt strings are too long for:
+"tests/tests_eval_framework/tasks/benchmarks/test_csqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gsm8k_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hendrycks_math_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mbpp_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_piqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_siqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_winogrande_ellamind.py" = ["E501"]
 
 [tool.mypy]
 plugins = "pydantic.mypy"
@@ -147,7 +158,7 @@ files = ["src", "utils"]
 [tool.pytest.ini_options]
 testpaths = ["./tests"]
 minversion = 8.0
-addopts = "-p 'no:legacypath' --doctest-modules"
+addopts = "-p 'no:legacypath' --doctest-modules -n auto"
 markers = [
     "gpu: needs a GPU runner, otherwise test can not be run",
     "cpu_slow: runs for a long time (on CPU)",

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/context/determined.py

@@ -12,7 +12,6 @@ from eval_framework.context.eval import EvalContext
 from eval_framework.context.local import _load_model
 from eval_framework.llm.base import BaseLLM
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.perturbation import PerturbationConfig
 from eval_framework.tasks.registry import validate_task_name
 from eval_framework.tasks.task_loader import load_extra_tasks
 
@@ -31,7 +30,6 @@ class TaskArgs(BaseModel):
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     user_prompt_suffix: str | None = None
-    perturbation_config: PerturbationConfig | None = None
     repeats: int | None = None
 
 
@@ -48,7 +46,7 @@ class Hyperparameters(BaseModel):
     description: str | None = None
     task_args: TaskArgs
     llm_args: dict[str, Any] | None = {}
-    extra_task_modules: list[str] | None = None
+    extra_tasks_dir: str | None = None
     delete_output_dir_after_upload: bool | None = None
 
 
@@ -67,19 +65,18 @@ class DeterminedContext(EvalContext):
             raise RuntimeError("Failed to retrieve cluster info.")
 
         # Load extra tasks if specified first
-        extra_task_modules = info.trial.hparams.get("extra_task_modules", None)
-        if extra_task_modules:
-            name = "extra_task_modules"
+        extra_tasks_dir = info.trial.hparams.get("extra_tasks_dir", None)
+        if extra_tasks_dir:
+            name = "extra_tasks_dir"
             val_cli = getattr(self, name, None)
-            val_hparams = extra_task_modules
-            if val_hparams:
-                if val_cli and val_hparams and val_cli != val_hparams:
-                    logger.info(
-                        f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters:"
-                        f"({val_hparams}). If it fails due to duplicate task names, remove the CLI argument and"
-                        "consolidate as a determined hyperparameter instead."
-                    )
-                load_extra_tasks(val_hparams)
+            val_hparams = extra_tasks_dir
+            if val_cli and val_hparams and val_cli != val_hparams:
+                logger.info(
+                    f"CLI argument {name} ({val_cli}) is being overridden by hyperparameters:"
+                    f"({val_hparams}). If it fails due to duplicate task names, remove the CLI argument and"
+                    "consolidate as a determined hyperparameter instead."
+                )
+            load_extra_tasks(val_hparams)
 
         self.hparams = Hyperparameters(**info.trial.hparams)
 
@@ -112,7 +109,6 @@ class DeterminedContext(EvalContext):
             "user_prompt_suffix",
             "judge_model_name",
             "judge_model_args",
-            "perturbation_config",
             "repeats",
         ]:
             val_cli = getattr(self, name, None)
@@ -145,7 +141,6 @@ class DeterminedContext(EvalContext):
             task_subjects=self.hparams.task_args.task_subjects,
             hf_revision=self.hparams.task_args.hf_revision or self.hf_revision,
             user_prompt_suffix=user_prompt_suffix,
-            perturbation_config=self.hparams.task_args.perturbation_config or self.perturbation_config,
             output_dir=self.hparams.output_dir,
             llm_judge_class=llm_judge_class,
             judge_model_args=self.hparams.task_args.judge_model_args or self.judge_model_args,

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/context/eval.py

@@ -9,7 +9,6 @@ from typing import Any
 import eval_framework
 from eval_framework.llm.base import BaseLLM
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.perturbation import PerturbationConfig
 
 
 def import_models(models_file: PathLike | str) -> dict[str, type[BaseLLM]]:
@@ -72,9 +71,6 @@ class EvalContext(AbstractContextManager):
         judge_model_args: dict[str, Any] | None = None,
         batch_size: int | None = None,
         description: str | None = None,
-        perturbation_type: str | None = None,
-        perturbation_probability: float | None = None,
-        perturbation_seed: int | None = None,
         randomize_judge_order: bool = False,
         delete_output_dir_after_upload: bool | None = None,
         repeats: int | None = None,
@@ -105,17 +101,6 @@ class EvalContext(AbstractContextManager):
         self.randomize_judge_order = randomize_judge_order
         self.delete_output_dir_after_upload = delete_output_dir_after_upload
         self.repeats = repeats
-        if perturbation_type or perturbation_probability is not None:
-            perturbation = {
-                "type": perturbation_type,
-                "probability": perturbation_probability,
-                "seed": perturbation_seed,
-            }
-            self.perturbation_config: PerturbationConfig | None = PerturbationConfig(
-                **{k: v for k, v in perturbation.items() if v is not None}
-            )
-        else:
-            self.perturbation_config = None
 
         self.config: EvalConfig | None = None
 

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/context/local.py

@@ -48,7 +48,6 @@ class LocalContext(EvalContext):
             num_samples=self.num_samples,
             max_tokens=self.max_tokens,
             num_fewshot=self.num_fewshot,
-            perturbation_config=self.perturbation_config,
             task_name=self.task_name,
             task_subjects=self.task_subjects,
             hf_revision=self.hf_revision,

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/metrics/completion/code_assertion.py

@@ -2,7 +2,7 @@ from llm_sandbox.exceptions import SandboxTimeoutError
 
 from eval_framework.metrics.base import BaseMetric, MetricResult
 from eval_framework.shared.types import Completion
-from eval_framework.tasks.utils import run_python_code
+from eval_framework.tasks.utils import DockerReturnedEmptyOutput, run_python_code
 
 
 class CodeCompletionAssertion(BaseMetric[Completion]):
@@ -16,7 +16,7 @@ class CodeCompletionAssertion(BaseMetric[Completion]):
         code = response.completion
         try:
             output = run_python_code(code, image="python:3.12-slim", runtime_configs={"mem_limit": "512m"})
-        except SandboxTimeoutError:
+        except (SandboxTimeoutError, DockerReturnedEmptyOutput):
             # The submitted code timed out (e.g. an infinite loop) -- a failing sample, not an infra
             # problem.
             import traceback

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/metrics/completion/minerva_math_utils.py

@@ -350,8 +350,17 @@ def strip_string_hendrycks(string: str) -> str:
     return string
 
 
+# MATH gold answers stay under ~80 chars.
+_PLAUSIBLE_ANSWER_LEN = 128
+
+
 def is_equiv_minerva(x1: str, x2: str, timeout_seconds: int = 5) -> bool:
     """Sympy-based equivalence (Minerva)."""
+    # extract_answers can fall back to the whole completion; sympy grinds on such
+    # 1000+ char strings until the timeout. A side far longer than the other can't
+    # be an equivalent final answer, so refuse it before parsing.
+    if max(len(x1), len(x2)) > max(_PLAUSIBLE_ANSWER_LEN, 3 * min(len(x1), len(x2))):
+        return False
 
     def _timeout_handler(signum: Any, frame: Any) -> None:
         raise TimeoutError()

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/response_generator.py

@@ -26,7 +26,7 @@ from eval_framework.shared.types import (
     Loglikelihood,
     RawLoglikelihood,
 )
-from eval_framework.tasks.base import Language, ResponseType, Sample
+from eval_framework.tasks.base import RANDOM_SEED, Language, ResponseType, Sample
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.utils import raise_errors
 from eval_framework.utils.constants import RED, RESET
@@ -60,21 +60,13 @@ class ResponseGenerator:
         self.num_samples = config.num_samples
         self.save_intermediate_results = config.save_intermediate_results
 
-        if config.perturbation_config is not None:
-            self.task = registry()[config.task_name].create_perturbation(
-                config.perturbation_config,
-                config.num_fewshot,
-                config.task_subjects,
-                config.hf_revision,
-                user_prompt_suffix=config.user_prompt_suffix,
-            )
-        else:
-            self.task = registry()[config.task_name].create(
-                config.num_fewshot,
-                config.task_subjects,
-                config.hf_revision,
-                user_prompt_suffix=config.user_prompt_suffix,
-            )
+        self.task = registry()[config.task_name].create(
+            config.num_fewshot,
+            config.task_subjects,
+            config.hf_revision,
+            user_prompt_suffix=config.user_prompt_suffix,
+            seed=RANDOM_SEED,
+        )
 
         self.response_type = self.task.get_response_type()
 
@@ -339,7 +331,6 @@ class ResponseGenerator:
             "num_samples",
             "llm_name",
             "llm_args",
-            "perturbation_config",
             "repeats",
             "user_prompt_suffix",
         ]

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/run.py

@@ -42,11 +42,11 @@ def parse_args() -> argparse.Namespace:
         help="The path to the Python module file containing model classes.",
     )
     parser.add_argument(
-        "--extra-task-modules",
-        nargs="*",
-        default=[],
+        "--extra-tasks-dir",
+        type=Path,
+        default=None,
         required=False,
-        help="List of files and folders containing additional task definitions.",
+        help="Directory containing additional task plugin modules to load.",
     )
     parser.add_argument(
         "--llm-name",
@@ -102,37 +102,6 @@ def parse_args() -> argparse.Namespace:
         help="Randomize the order of answers presented to the LLM judge to mitigate position bias.",
     )
 
-    # Perturbation arguments
-    parser.add_argument(
-        "--perturbation-type",
-        type=str,
-        required=False,
-        choices=[
-            "editor",
-            "permute",
-            "replace",
-            "delete",
-            "uppercase",
-        ],
-        help=(
-            "The type of perturbation to apply. Note that this may not make sense for some prompts, for example those "
-            "containing math and code."
-        ),
-    )
-    parser.add_argument(
-        "--perturbation-probability",
-        type=float,
-        required=False,
-        default=None,
-        help="The probability of applying a perturbation to each word or character (between 0.0 and 1.0).",
-    )
-    parser.add_argument(
-        "--perturbation-seed",
-        type=int,
-        required=False,
-        default=42,
-        help="Random seed controlling perturbations.",
-    )
     parser.add_argument(
         "--task-subjects",
         type=str,
@@ -313,12 +282,6 @@ def parse_args() -> argparse.Namespace:
 
     args.judge_model_args = judge_model_args
 
-    # if args.extra_task_modules:
-    #     # Convert the comma-separated string into a list
-    #     args.extra_task_modules = [file_or_dir.strip() for file_or_dir in args.extra_task_modules.split(",")]
-    # else:
-    #     args.extra_task_modules = None
-
     if args.task_suite and args.task_name:
         parser.error("--task-suite and --task-name are mutually exclusive.")
 
@@ -336,8 +299,8 @@ def _run_single_task(kwargs: dict) -> None:
     now = datetime.datetime.now()
     logger.info(f"starting time: {now}")
 
-    if kwargs.get("extra_task_modules"):
-        load_extra_tasks(kwargs["extra_task_modules"])
+    if kwargs.get("extra_tasks_dir"):
+        load_extra_tasks(kwargs["extra_tasks_dir"])
 
     context_name = kwargs.pop("context")
 
@@ -366,9 +329,6 @@ def _run_single_task(kwargs: dict) -> None:
         judge_model_args=kwargs["judge_model_args"],
         batch_size=kwargs["batch_size"],
         description=kwargs["description"],
-        perturbation_type=kwargs["perturbation_type"],
-        perturbation_probability=kwargs["perturbation_probability"],
-        perturbation_seed=kwargs["perturbation_seed"],
         randomize_judge_order=kwargs["randomize_judge_order"],
         delete_output_dir_after_upload=kwargs["delete_output_dir_after_upload"],
         # save_logs=kwargs["save_logs"],

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/run_direct.py

@@ -32,9 +32,6 @@ if __name__ == "__main__":
         "hf_upload_dir": "",
         "description": "",
         "batch_size": 1,
-        "perturbation_type": None,
-        "perturbation_probability": None,
-        "perturbation_seed": None,
         "save_logs": True,
     }
     run_with_kwargs(kwargs)

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/tasks/base.py

@@ -129,10 +129,6 @@ class BaseTask[SubjectType](Task):
     # inherited implicitly (a subclass in another package would otherwise resolve the wrong file).
     REVISION_LOCKFILE: Path | None
 
-    # Words in _get_instruction_text() not to be perturbed. List of words is case insensitive. No special characters
-    # or whitespace should be included.
-    PERTURBATION_UNMODIFIABLE_WORDS: list[str] | None
-
     # The language (or languages) tested by the benchmark. Accepts a single string, a dictionary specifying
     # language by subtopic, or `None` (for tasks not specific to a single language).
     LANGUAGE: Language | dict[str, Language] | dict[str, tuple[Language, Language]] | None
@@ -147,6 +143,7 @@ class BaseTask[SubjectType](Task):
         self.stop_sequences: list[str] | None = None
         self.max_tokens: int | None = None
         self.hf_revision: str | None = self._apply_hf_revision()
+        self.rnd: random.Random | None = None
 
     def _apply_hf_revision(self, custom_hf_revision: str | None = None) -> str | None:
         # Precedence: CLI/config override > REVISION_LOCKFILE pin.
@@ -167,72 +164,27 @@ class BaseTask[SubjectType](Task):
         custom_subjects: list[str] | None,
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
+        seed: int | None = RANDOM_SEED,
     ) -> Self:
         instance = cls(num_fewshot=num_fewshot)
         if user_prompt_suffix is not None and instance.get_response_type() != ResponseType.COMPLETION:
             raise ValueError("user_prompt_suffix is only supported for completion tasks.")
         instance.user_prompt_suffix = user_prompt_suffix
-
+        instance.rnd = random.Random(seed)
         # If custom subjects were provided during initialization, they take precedence over the class-level SUBJECTS.
-        filtered_subjects = instance._filter_task_subjects(custom_subjects=custom_subjects)
-        if filtered_subjects:
-            logger.info(f"Setting SUBJECTS to `{filtered_subjects}` for the task {instance.__class__.__name__}")
-            instance.SUBJECTS = filtered_subjects  # type: ignore[assignment]
+        if custom_subjects:
+            filtered_subjects = resolve_overwrite_subjects(
+                custom_subjects=custom_subjects,
+                accepted_subjects=instance.SUBJECTS,
+                task_name=instance.display_name(),
+            )
+            logger.info(f"Setting SUBJECTS to `{filtered_subjects}` for the task {instance.display_name()}")
+            instance.SUBJECTS = filtered_subjects
 
         instance.hf_revision = instance._apply_hf_revision(custom_hf_revision)
 
         return instance
 
-    def _filter_task_subjects(self, custom_subjects: list[str] | None) -> list[str] | list[tuple] | None:
-        """Process custom subjects passed from EvalConfig. Check and returns restricted task subjects if specified."""
-        if not custom_subjects:
-            return None
-
-        assert hasattr(self, "SUBJECTS") and len(self.SUBJECTS) > 0
-        if isinstance(self.SUBJECTS[0], tuple):
-            # subjects are specified as comma-separated strings but tuple positions may each hold a
-            # different type (e.g. tuple[str, int, str]). Infer the expected type per position from an
-            # actual subject and cast each part back to it, so it compares equal to the real values
-            # below instead of just their str() form. "*" is a wildcard sentinel and stays a string.
-            num_items = len(self.SUBJECTS[0])
-            position_types = [type(self.SUBJECTS[0][i]) for i in range(num_items)]
-
-            def cast(raw: str, i: int) -> Any:
-                raw = raw.strip()
-                return raw if raw == "*" else position_types[i](raw)
-
-            filters = []
-            for custom_subject in custom_subjects:
-                parts = custom_subject.split(",")
-                assert len(parts) == num_items, (
-                    f"Subject '{custom_subject}' has {len(parts)} parts, expected {num_items} for "
-                    f"task {self.display_name()}"
-                )
-                filters.append(tuple(cast(part, i) for i, part in enumerate(parts)))
-
-            # check if all parts of custom subjects exists (* is a wildcard)
-            legal_values = [
-                set([s[i] for s in self.SUBJECTS if isinstance(s, tuple)] + ["*"]) for i in range(num_items)
-            ]
-
-            for tpl in filters:
-                for i, v in enumerate(tpl):
-                    assert v in legal_values[i], f"Subject part {v} not found in task {self.__class__.__name__}"
-
-            # filter task subjects. * is a supported wildcard for a specific item in a tuple, e.g. "DE_DE, *"
-            chosen_subjects: list[tuple] = []
-            for subject in self.SUBJECTS:
-                subject_tuple = subject if isinstance(subject, tuple) else tuple(str(subject).split(","))
-                for filter in filters:
-                    if all(filter[i] == "*" or filter[i] == subject_tuple[i] for i in range(num_items)):
-                        chosen_subjects.append(subject_tuple)
-                        break
-            return chosen_subjects
-        else:
-            for cs in custom_subjects:
-                assert cs in self.SUBJECTS, f"Subject {cs} not found in task {self.__class__.__name__}"
-            return custom_subjects
-
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")
         download_config = DownloadConfig(cache_dir=cache_dir, max_retries=5)
@@ -244,8 +196,8 @@ class BaseTask[SubjectType](Task):
         )
 
     def _shuffle_splits(self, hf_dataset: DatasetDict) -> dict[str, Any]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         dataset = {}
-        self.rnd = random.Random(RANDOM_SEED)
 
         for split, data in hf_dataset.items():
             if split not in [self.SAMPLE_SPLIT, self.FEWSHOT_SPLIT]:
@@ -413,6 +365,7 @@ class BaseTask[SubjectType](Task):
         return None
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        assert self.rnd is not None, "Task RNG is unseeded; build tasks via `with_overwrite`."
         if self.FEWSHOT_SPLIT == self.SAMPLE_SPLIT:
             # If the fewshot and sample splits are the same, we risk including the current eval item
             # as a fewshot example (leaking the answer). To prevent this, sample one extra example,
@@ -546,3 +499,51 @@ class BaseTask[SubjectType](Task):
 
     def display_name(self) -> str:
         return self.NAME
+
+
+def _subject_parts(subject: object) -> tuple[str, ...]:
+    """A subject as its stringified parts: tuple subjects keep their fields, scalars are a single part.
+
+    Comparing string forms means tuple fields of any type work (e.g. tuple[str, int, str]) without
+    having to parse the CLI token into each field's native type.
+    """
+    return tuple(str(field) for field in subject) if isinstance(subject, tuple) else (str(subject),)
+
+
+def resolve_overwrite_subjects[SubjectType](
+    custom_subjects: list[str], accepted_subjects: list[SubjectType], task_name: str
+) -> list[SubjectType]:
+    """Restrict `accepted_subjects` to the ones requested via --task-subjects.
+
+    A subject matches a token when their parts agree position by position, where "*" matches any one
+    part -- so "DE_DE,*" selects every German subject and "*" selects every scalar subject.
+    """
+    if not accepted_subjects:
+        raise ValueError(f"Task {task_name} has no SUBJECTS defined")
+
+    filters = {token: tuple(part.strip() for part in token.split(",")) for token in custom_subjects}
+
+    # Select and validate in one pass. Every filter is tried against every subject (no early exit), so
+    # a filter still counts as used when its subjects were already selected by an earlier filter --
+    # e.g. "a,*" after "a,1" is valid, not unused.
+    chosen_subjects: list[SubjectType] = []
+    used_filters: set[str] = set()
+    for subject in accepted_subjects:
+        fields = _subject_parts(subject)
+        matching = [
+            token
+            for token, parts in filters.items()
+            if len(parts) == len(fields) and all(p in ("*", f) for p, f in zip(parts, fields))
+        ]
+        if matching:
+            chosen_subjects.append(subject)
+            used_filters.update(matching)
+
+    for token in custom_subjects:
+        if token not in used_filters:
+            raise ValueError(
+                f"Subject '{token}' not found in task {task_name}. Subjects are matched by their "
+                f"string form, so check number and enum formatting."
+            )
+
+    return chosen_subjects

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/tasks/benchmarks/arc.py

@@ -25,7 +25,6 @@ class ARC(BaseTask[str]):
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
     SUBJECTS = ["ARC-Easy", "ARC-Challenge"]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question"] + get_n_letters(5)
     LANGUAGE = Language.ENG
 
     def __init__(self, num_fewshot: int = 0) -> None:

{eval_framework-0.9.1 → eval_framework-0.10.0}/src/eval_framework/tasks/benchmarks/arc_de.py

@@ -22,7 +22,6 @@ class ARC_DE(BaseTask[str]):
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
     SUBJECTS = [NO_SUBJECT]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Frage"] + get_n_letters(5)
     LANGUAGE = Language.DEU
 
     def __init__(self, num_fewshot: int = 0) -> None: