eval-framework 0.9.0__tar.gz → 0.9.1__tar.gz

{eval_framework-0.9.0 → eval_framework-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.9.0
+Version: 0.9.1
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -450,11 +450,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.9.0 → eval_framework-0.9.1}/README.md

@@ -189,11 +189,13 @@ from eval_framework.main import main
 from eval_framework.tasks.eval_config import EvalConfig
 from template_formatting.formatter import HFFormatter
 
+
 # Define your model
 class MyHuggingFaceModel(HFLLM):
     LLM_NAME = "microsoft/DialoGPT-medium"
     DEFAULT_FORMATTER = partial(HFFormatter, "microsoft/DialoGPT-medium")
 
+
 if __name__ == "__main__":
     # Initialize your model
     llm = MyHuggingFaceModel()

{eval_framework-0.9.0 → eval_framework-0.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.9.0"
+version = "0.9.1"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.9.0 → eval_framework-0.9.1}/src/eval_framework/tasks/base.py

@@ -190,11 +190,27 @@ class BaseTask[SubjectType](Task):
 
         assert hasattr(self, "SUBJECTS") and len(self.SUBJECTS) > 0
         if isinstance(self.SUBJECTS[0], tuple):
-            # subjects are specified as strings but we need tuples
-            filters = [tuple(item.strip() for item in subject.split(",")) for subject in custom_subjects]
+            # subjects are specified as comma-separated strings but tuple positions may each hold a
+            # different type (e.g. tuple[str, int, str]). Infer the expected type per position from an
+            # actual subject and cast each part back to it, so it compares equal to the real values
+            # below instead of just their str() form. "*" is a wildcard sentinel and stays a string.
+            num_items = len(self.SUBJECTS[0])
+            position_types = [type(self.SUBJECTS[0][i]) for i in range(num_items)]
+
+            def cast(raw: str, i: int) -> Any:
+                raw = raw.strip()
+                return raw if raw == "*" else position_types[i](raw)
+
+            filters = []
+            for custom_subject in custom_subjects:
+                parts = custom_subject.split(",")
+                assert len(parts) == num_items, (
+                    f"Subject '{custom_subject}' has {len(parts)} parts, expected {num_items} for "
+                    f"task {self.display_name()}"
+                )
+                filters.append(tuple(cast(part, i) for i, part in enumerate(parts)))
 
             # check if all parts of custom subjects exists (* is a wildcard)
-            num_items = len(self.SUBJECTS[0])
             legal_values = [
                 set([s[i] for s in self.SUBJECTS if isinstance(s, tuple)] + ["*"]) for i in range(num_items)
             ]
@@ -204,18 +220,18 @@ class BaseTask[SubjectType](Task):
                     assert v in legal_values[i], f"Subject part {v} not found in task {self.__class__.__name__}"
 
             # filter task subjects. * is a supported wildcard for a specific item in a tuple, e.g. "DE_DE, *"
-            chosen_subjects = []
+            chosen_subjects: list[tuple] = []
             for subject in self.SUBJECTS:
                 subject_tuple = subject if isinstance(subject, tuple) else tuple(str(subject).split(","))
                 for filter in filters:
                     if all(filter[i] == "*" or filter[i] == subject_tuple[i] for i in range(num_items)):
                         chosen_subjects.append(subject_tuple)
                         break
-            return chosen_subjects  # type: ignore[return-value]
+            return chosen_subjects
         else:
             for cs in custom_subjects:
                 assert cs in self.SUBJECTS, f"Subject {cs} not found in task {self.__class__.__name__}"
-            return custom_subjects  # type: ignore[return-value]
+            return custom_subjects
 
     def _load_hf_dataset(self, **kwargs: Any) -> Any:
         cache_dir: str = os.environ.get("HF_DATASET_CACHE_DIR", f"{Path.home()}/.cache/huggingface/datasets")

{eval_framework-0.9.0 → eval_framework-0.9.1}/src/eval_framework/tasks/registry.py

@@ -223,20 +223,15 @@ class _Eager(EvalFactory):
 
 
 class Registry:
-    """A registry for tasks with support for lazy loading.
-
-    Task names are hashed based on the upper-case name, to avoid issues with
-    ambiguous naming.
-    """
+    """A registry for Tasks"""
 
     def __init__(self) -> None:
-        # TODO: Lookup only with upper names
-        self._registry: dict[str, tuple[str, EvalFactory]] = dict()
+        self._registry: dict[str, EvalFactory] = dict()
 
     def __iter__(self) -> Iterator[str]:
         """Iterate over all task names in the registry."""
-        for name, _ in self._registry.values():
-            yield name
+        for factory in self._registry.values():
+            yield factory.id()
 
     def task_names(self) -> list[str]:
         """The names of all registered tasks."""
@@ -244,7 +239,8 @@ class Registry:
 
     def items(self) -> Iterator[tuple[str, EvalFactory]]:
         """Iterate over `(task name, EvalFactory)` pairs in the registry."""
-        yield from self._registry.values()
+        for factory in self._registry.values():
+            yield factory.id(), factory
 
     @staticmethod
     def _task_key(name: str, /) -> str:
@@ -262,26 +258,25 @@ class Registry:
     def __getitem__(self, name: str, /) -> EvalFactory:
         task_key = self._task_key(name)
         try:
-            _, factory = self._registry[task_key]
+            return self._registry[task_key]
         except KeyError:
             raise KeyError(f"Task not found: {name=} with task_key {task_key=}")
 
-        return factory
-
-    def __setitem__(self, name: str, factory: EvalFactory) -> None:
-        task_key = self._task_key(name)
+    def add(self, factory: EvalFactory) -> None:
+        """Register a factory under the key derived from its ``id()``."""
+        task_key = self._task_key(factory.id())
         if task_key in self._registry:
             raise ValueError(f"Cannot register duplicate task with key: {task_key}")
 
-        self._registry[task_key] = (name, factory)
+        self._registry[task_key] = factory
 
     def register(self, task: type[BaseTask]) -> str:
         """The class name is used as the task name."""
         if not issubclass(task, BaseTask):
             raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-        name = task.__name__
-        self[name] = _Eager(task)
-        return name
+        factory = _Eager(task)
+        self.add(factory)
+        return factory.id()
 
     def register_lazy(self, class_path: str, /) -> None:
         """Register a task by its dotted class path, without importing its module."""
@@ -291,7 +286,7 @@ class Registry:
                 "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
             )
         base_module, class_name = class_path.rsplit(".", maxsplit=1)
-        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+        self.add(_Lazy(class_name=class_name, module=base_module))
 
 
 _REGISTRY = Registry()
@@ -335,6 +330,7 @@ def register_task(task: type[BaseTask]) -> str:
     return registry().register(task)
 
 
-def register_lazy_task(class_path: str, /) -> None:
+def register_lazy_task(class_path: str, /, registry: Registry | None = None) -> None:
     """Register a task by its dotted class path, without importing its module."""
-    registry().register_lazy(class_path)
+    r = registry if registry is not None else _REGISTRY
+    r.register_lazy(class_path)

eval_framework-0.9.1/src/eval_framework/tasks/task_names.py

@@ -0,0 +1,213 @@
+from enum import Enum
+
+from eval_framework.tasks.base import BaseTask
+from eval_framework.tasks.registry import Registry, register_lazy_task
+from eval_framework.tasks.registry import registry as global_registry
+
+
+class TaskNameEnum(Enum):
+    @property
+    def value(self) -> type[BaseTask]:
+        return super().value
+
+
+def register_all_tasks(registry: Registry | None = None) -> None:
+    """Register all the benchmark tasks with the eval framework
+
+    Uses global registry by default.
+    """
+    registry = registry if registry is not None else global_registry()
+
+    register_math_reasoning_tasks(registry=registry)
+    register_arc_tasks(registry=registry)
+    register_arc_de_tasks(registry=registry)
+    register_bigcodebench_tasks(registry=registry)
+    register_copa_tasks(registry=registry)
+    register_goldenswag_tasks(registry=registry)
+    register_gpqa_tasks(registry=registry)
+    register_gsm8k_tasks(registry=registry)
+    register_hellaswag_tasks(registry=registry)
+    register_humaneval_tasks(registry=registry)
+    register_ifeval_tasks(registry=registry)
+    register_multipl_e_tasks(registry=registry)
+    register_mbpp_tasks(registry=registry)
+    register_mmlu_tasks(registry=registry)
+    register_mmlu_pro_tasks(registry=registry)
+    register_global_mmlu_tasks(registry=registry)
+    register_piqa_tasks(registry=registry)
+    register_sciq_tasks(registry=registry)
+    register_squad_tasks(registry=registry)
+    register_winogrande_tasks(registry=registry)
+    register_csqa_tasks(registry=registry)
+    register_drop_tasks(registry=registry)
+    register_naturalqs_open_tasks(registry=registry)
+    register_social_iqa_tasks(registry=registry)
+    register_medqa_tasks(registry=registry)
+
+
+def register_arc_tasks(registry: Registry) -> None:
+    """Register arc benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_OLMES", registry=registry)
+
+
+def register_hellaswag_tasks(registry: Registry) -> None:
+    """Register hellaswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_OLMES", registry=registry)
+
+
+def register_piqa_tasks(registry: Registry) -> None:
+    """Register piqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_OLMES", registry=registry)
+
+
+def register_gpqa_tasks(registry: Registry) -> None:
+    """Register gpqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT", registry=registry)
+
+
+def register_gsm8k_tasks(registry: Registry) -> None:
+    """Register gsm8k benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB", registry=registry)
+
+
+def register_math_reasoning_tasks(registry: Registry) -> None:
+    """Register math_reasoning benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2026", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL", registry=registry)
+
+
+def register_mmlu_tasks(registry: Registry) -> None:
+    """Register mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT", registry=registry)
+
+
+def register_humaneval_tasks(registry: Registry) -> None:
+    """Register humaneval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEvalBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEval_OLMES", registry=registry)
+
+
+def register_mbpp_tasks(registry: Registry) -> None:
+    """Register mbpp benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus", registry=registry)
+
+
+def register_bigcodebench_tasks(registry: Registry) -> None:
+    """Register bigcodebench benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench_OLMES", registry=registry)
+
+
+def register_arc_de_tasks(registry: Registry) -> None:
+    """Register arc_de benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE", registry=registry)
+
+
+def register_copa_tasks(registry: Registry) -> None:
+    """Register copa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.copa.COPA_OLMES", registry=registry)
+
+
+def register_goldenswag_tasks(registry: Registry) -> None:
+    """Register goldenswag benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK", registry=registry)
+
+
+def register_ifeval_tasks(registry: Registry) -> None:
+    """Register ifeval benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEval", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe", registry=registry)
+
+
+def register_multipl_e_tasks(registry: Registry) -> None:
+    """Register multipl_e benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalSh", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPCpp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJava", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPPhp", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPRs", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh", registry=registry)
+
+
+def register_mmlu_pro_tasks(registry: Registry) -> None:
+    """Register mmlu_pro benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT", registry=registry)
+
+
+def register_global_mmlu_tasks(registry: Registry) -> None:
+    """Register global_mmlu benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German", registry=registry)
+
+
+def register_sciq_tasks(registry: Registry) -> None:
+    """Register sciq benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES", registry=registry)
+
+
+def register_squad_tasks(registry: Registry) -> None:
+    """Register squad benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT", registry=registry)
+
+
+def register_winogrande_tasks(registry: Registry) -> None:
+    """Register winogrande benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze", registry=registry)
+
+
+def register_csqa_tasks(registry: Registry) -> None:
+    """Register csqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES", registry=registry)
+
+
+def register_drop_tasks(registry: Registry) -> None:
+    """Register drop benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropMC_OLMES", registry=registry)
+
+
+def register_naturalqs_open_tasks(registry: Registry) -> None:
+    """Register naturalqs_open benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpen", registry=registry)
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES", registry=registry)
+
+
+def register_social_iqa_tasks(registry: Registry) -> None:
+    """Register social_iqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES", registry=registry)
+
+
+def register_medqa_tasks(registry: Registry) -> None:
+    """Register medqa benchmark tasks."""
+    register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES", registry=registry)

eval_framework-0.9.0/src/eval_framework/tasks/task_names.py

@@ -1,82 +0,0 @@
-from enum import Enum
-
-from eval_framework.tasks.base import BaseTask
-from eval_framework.tasks.registry import register_lazy_task
-
-
-class TaskNameEnum(Enum):
-    @property
-    def value(self) -> type[BaseTask]:
-        return super().value
-
-
-def register_all_tasks() -> None:
-    """Register all the benchmark tasks with the eval framework."""
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2024")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2025")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.AIME2026")
-    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC")
-    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_IDK")
-    register_lazy_task("eval_framework.tasks.benchmarks.arc.ARC_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.arc_de.ARC_DE")
-    register_lazy_task("eval_framework.tasks.benchmarks.bigcodebench.BigCodeBench_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.copa.COPA_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG")
-    register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK")
-    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT")
-    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.GSM8KReasoning")
-    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG")
-    register_lazy_task("eval_framework.tasks.benchmarks.hellaswag.HELLASWAG_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEvalBPB")
-    register_lazy_task("eval_framework.tasks.benchmarks.humaneval.HumanEval_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEval")
-    register_lazy_task("eval_framework.tasks.benchmarks.ifeval.IFEvalDe")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATH500")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinerva_OLMES_NONL")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalCpp")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJava")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalJs")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalPhp")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalRs")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEHumanEvalSh")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPCpp")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJava")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPJs")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPPhp")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPRs")
-    register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh")
-    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB")
-    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus")
-    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.FullTextMMLU")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_IDK")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT")
-    register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT")
-    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU")
-    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German")
-    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA")
-    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_IDK")
-    register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
-    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
-    register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")
-    register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.drop.DropMC_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpen")
-    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES")
-    register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES")