eval-framework 0.8.7__tar.gz → 0.8.11__tar.gz

{eval_framework-0.8.7 → eval_framework-0.8.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.7
+Version: 0.8.11
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -216,14 +216,14 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.0,<4
+Requires-Dist: nltk>=3.10.1,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
 Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
-Requires-Dist: mysql-connector-python>=9.7.0,<10
+Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.39
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.50.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.52.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.7 → eval_framework-0.8.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.7"
+version = "0.8.11"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -23,14 +23,14 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.0,<4",
+  "nltk>=3.10.1,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
   "spacy>=3.8.14,<4",
   "jsonschema>=4.26.0,<5",
-  "mysql-connector-python>=9.7.0,<10", # required for sql-related tasks
+  "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.39",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.50.0,<3",
+  "openai>=2.52.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -92,7 +92,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.0",
+  "ruff>=0.16.1",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [

{eval_framework-0.8.7 → eval_framework-0.8.11}/src/eval_framework/suite.py

@@ -18,7 +18,7 @@ from eval_framework.context.local import _load_model
 from eval_framework.result_processors.result_processor import generate_output_dir
 from eval_framework.run import _run_single_task
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.registry import is_registered
+from eval_framework.tasks.registry import registry
 
 logger = logging.getLogger(__name__)
 
@@ -108,7 +108,7 @@ class TaskSuite(BaseModel):
         if isinstance(self.tasks, str):
             if self.name is None:
                 self.name = self.tasks
-            if not is_registered(self.tasks):
+            if self.tasks not in registry():
                 raise ValueError(f"Task '{self.tasks}' is not registered.")
         elif not self.tasks:
             raise ValueError(f"TaskSuite '{self.name}': 'tasks' must not be empty.")

{eval_framework-0.8.7 → eval_framework-0.8.11}/src/eval_framework/tasks/hf-dataset-revisions.json

@@ -23,7 +23,6 @@
     "google-research-datasets/nq_open": "5dd9790a83002ad084ddeb7c420dc716852c6f28",
     "google/IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84",
     "jzhang86/de_ifeval": "4f52d847003b3c83cc282e9d296853a24b34b19a",
-    "mandarjoshi/trivia_qa": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
     "math-ai/aime25": "563bb8404243c5f09de6ec262f2db674fe5bce9b",
     "math-ai/aime26": "79037aebdb6580008fb960d17cb21fd3099083e3",
     "nuprl/MultiPL-E": "28441b6024e71d4a1c1c0f6bf171c935cd5a43f2",

{eval_framework-0.8.7 → eval_framework-0.8.11}/src/eval_framework/tasks/registry.py

@@ -255,10 +255,6 @@ class Registry:
 
         return factory
 
-    def add(self, task: type[BaseTask]) -> None:
-        task_key = self._task_key(task.NAME)
-        self._registry[task_key] = (task.NAME, _Eager(task))
-
     def __setitem__(self, name: str, factory: EvalFactory) -> None:
         task_key = self._task_key(name)
         if task_key in self._registry:
@@ -266,6 +262,24 @@ class Registry:
 
         self._registry[task_key] = (name, factory)
 
+    def register(self, task: type[BaseTask]) -> str:
+        """The class name is used as the task name."""
+        if not issubclass(task, BaseTask):
+            raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
+        name = task.__name__
+        self[name] = _Eager(task)
+        return name
+
+    def register_lazy(self, class_path: str, /) -> None:
+        """Register a task by its dotted class path, without importing its module."""
+        if "." not in class_path:
+            raise ValueError(
+                f"Invalid class path `{class_path}`. This needs to be a global path like "
+                "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
+            )
+        base_module, class_name = class_path.rsplit(".", maxsplit=1)
+        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+
 
 _REGISTRY = Registry()
 
@@ -298,7 +312,7 @@ def is_registered(name: str, /) -> bool:
 
 def validate_task_name(name: str) -> str:
     """Pydantic-style validator for task names."""
-    if not is_registered(name):
+    if name not in registry():
         raise ValueError(f"Task not registered: {name}")
     return name
 
@@ -313,28 +327,9 @@ def get_task(name: str, /) -> type[BaseTask]:
 
 def register_task(task: type[BaseTask]) -> str:
     """The class name is used as the task name."""
-    if not issubclass(task, BaseTask):
-        raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-    name = task.__name__
-    _REGISTRY[name] = _Eager(task)
-    return name
+    return registry().register(task)
 
 
 def register_lazy_task(class_path: str, /) -> None:
-    """Register a task without importing it.
-
-    Lazily register a task without importing the module.
-
-    Args:
-        class_path: The full path to the task class. For example,
-            `eval_framework.tasks.benchmarks.mmlu.MMLU`.
-        extras: Any extra dependencies of `eval_framework` that need to be installed for this task.
-    """
-    if "." not in class_path:
-        raise ValueError(
-            f"Invalid class path `{class_path}`. This needs to be a global path like "
-            "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
-        )
-
-    base_module, class_name = class_path.rsplit(".", maxsplit=1)
-    _REGISTRY[class_name] = _Lazy(class_name=class_name, module=base_module)
+    """Register a task by its dotted class path, without importing its module."""
+    registry().register_lazy(class_path)

{eval_framework-0.8.7 → eval_framework-0.8.11}/src/eval_framework/tasks/task_loader.py

@@ -8,7 +8,8 @@ from types import ModuleType
 from typing import Any
 
 from eval_framework.tasks.base import BaseTask
-from eval_framework.tasks.registry import is_registered, register_task
+from eval_framework.tasks.registry import Registry
+from eval_framework.tasks.registry import registry as global_registry
 
 logger = logging.getLogger(__name__)
 
@@ -46,14 +47,13 @@ def import_file(f: str | os.PathLike, /) -> Any:
     return user_module
 
 
-def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
+def load_extra_tasks(module_paths: Sequence[str | os.PathLike], registry: Registry | None = None) -> None:
     """Dynamically load and register user-defined tasks from a list of files or directories.
 
-    Each .py file found will be imported, and any BaseTask subclass will be registered
-    in the TaskName enum for use by name.
-    Provides clear error messages for missing/invalid files or import errors.
+    Each .py file found is imported, and any BaseTask subclass is registered
     """
     assert not (isinstance(module_paths, str)), "module_paths must be a sequence of strings / os.PathLike objects"
+    registry = registry if registry is not None else global_registry()
     for file_path in find_all_python_files(*module_paths):
         user_module = import_file(file_path)
 
@@ -63,7 +63,7 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                 if not hasattr(obj, "NAME"):
                     logger.info(f"[User Task Loader] Skipping {obj.__module__} - no NAME attribute present.")
                 else:
-                    if is_registered(obj.NAME):
+                    if obj.NAME in registry:
                         # two classes with the same NAME attribute
                         logger.info(obj.__module__)
 
@@ -77,5 +77,5 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                     else:
                         # if there is no duplicate name conflict then register the new task
                         class_obj = getattr(user_module, name)
-                        register_task(class_obj)
+                        registry.register(class_obj)
                         logger.info(f"[User Task Loader] Registered task: {class_obj.NAME}")

{eval_framework-0.8.7 → eval_framework-0.8.11}/src/eval_framework/tasks/task_names.py

@@ -71,8 +71,6 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TriviaQA_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")
     register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES")
@@ -81,9 +79,3 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES")
-    try:
-        # Importing the companion registers the additional tasks from the module.
-        # This is mostly for convenience for internal use-cases
-        import eval_framework_companion  # noqa
-    except ImportError:
-        pass

eval_framework-0.8.7/src/eval_framework/tasks/benchmarks/triviaqa.py

@@ -1,78 +0,0 @@
-import random
-from typing import Any
-
-from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
-from eval_framework.metrics.completion.f1 import F1, F1SquadNormalized
-from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
-
-
-class TRIVIAQA(BaseTask[str]):
-    """Trivia QA dataset: https://huggingface.co/datasets/mandarjoshi/trivia_qa"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA"
-    DATASET_PATH = "mandarjoshi/trivia_qa"
-    SAMPLE_SPLIT = "validation"
-    FEWSHOT_SPLIT = "train"
-    RESPONSE_TYPE = ResponseType.COMPLETION
-    METRICS = [AccuracyCompletion, F1]
-    SUBJECTS = ["rc.wikipedia.nocontext"]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
-    LANGUAGE = Language.ENG
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = ["\n"]
-        self.max_tokens = 400  # the max length of the ground truth is 282 characters while the average is ~16
-        self.rnd_choice_shuffle = random.Random()
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        prompt = f"Question: {item['question'].strip()}\nAnswer:"
-        return prompt
-
-    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        target = self._get_ground_truth(item)[0]
-        assert target is not None
-        assert isinstance(target, str)
-        return f" {target}"
-
-    def _get_ground_truth(self, item: dict[str, Any]) -> list[str]:
-        return item["answer"]["aliases"]
-
-    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
-        return completion_text.strip().rstrip(".")
-
-
-class TriviaQA_MA(TRIVIAQA):
-    """TriviaQA with the exact system prompt used in MA training"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA_MA"
-    SUBJECTS = ["rc.wikipedia"]
-    UNANSWERABLE_STR = "unanswerable"
-
-    METRICS = [AccuracyCompletion, F1, F1SquadNormalized]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer", "Context", "unanswerable"]
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = []
-        self.max_tokens = 27_000
-
-    def _get_context_text(self, item: dict[str, Any]) -> str:
-        return "\n\n".join(item["entity_pages"]["wiki_context"])
-
-    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
-        return (
-            "You are a helpful assistant and will answer the user's questions carefully, "
-            "logically, accurately and well-reasoned.\n"
-            "Use the given context to answer the question faithfully. Answer only if the "
-            f"answer is present in the given context, otherwise respond with '{self.UNANSWERABLE_STR}' "
-            "if the answer is not present in the context."
-        )
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return f"Context:\n{self._get_context_text(item)}\n\nQuestion:\n{item['question'].strip()}\n"