eval-framework 0.8.11__tar.gz → 0.9.0__tar.gz

{eval_framework-0.8.11 → eval_framework-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.11
+Version: 0.9.0
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -226,7 +226,7 @@ Requires-Dist: jsonschema>=4.26.0,<5
 Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
-Requires-Dist: llm-sandbox[docker]==0.3.39
+Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.52.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.11 → eval_framework-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.11"
+version = "0.9.0"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -33,7 +33,7 @@ dependencies = [
   "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.14.0,<2",
-  "llm-sandbox[docker]==0.3.39",
+  "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.52.0,<3",
+  "openai>=2.53.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/response_generator.py

@@ -179,7 +179,7 @@ class ResponseGenerator:
         :param should_preempt_callable: function to check if preempt is called
         :return: list of responses, preempted
         """
-        logger.info(f"{RED}[ Running task {self.task.NAME} against model ------------ ]{RESET}")
+        logger.info(f"{RED}[ Running task {self.task.display_name()} against model ------------ ]{RESET}")
         self.start_time, monotonic_start = time.time(), time.monotonic()
         run_fn = self._generative_output_type_selector()
         self._verify_loaded_metadata_compatibility()

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/tasks/base.py

@@ -2,7 +2,7 @@ import logging
 import os
 import random
 import traceback
-from abc import ABC
+from abc import ABC, abstractmethod
 from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
@@ -86,7 +86,37 @@ SubjectType = TypeVar("SubjectType")
 logger = logging.getLogger(__name__)
 
 
-class BaseTask[SubjectType](ABC):
+class Task(ABC):
+    """The contract a caller relies on to run an evaluation"""
+
+    @abstractmethod
+    def iterate_samples(self, num_samples: int | None = None) -> Iterable[Sample]: ...
+
+    @abstractmethod
+    def generate_completions(
+        self,
+        llm: "BaseLLM",
+        samples: list[Sample],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        fail_on_error: bool = True,
+    ) -> list[Completion]:
+        """Run ``llm`` over ``samples`` and return their completions."""
+
+    @abstractmethod
+    def get_metadata(self) -> dict[str, str | list[str]]:
+        """Descriptive metadata about the eval for result reporting."""
+
+    @abstractmethod
+    def get_response_type(self) -> ResponseType: ...
+
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable display name. Is allowed to have special characters and whitespaces."""
+        ...
+
+
+class BaseTask[SubjectType](Task):
     NAME: str
     DATASET_PATH: str
     SAMPLE_SPLIT: str
@@ -497,3 +527,6 @@ class BaseTask[SubjectType](ABC):
     def METRICS(cls) -> list[type["BaseMetric"]]:
         """For backwards compatibility."""
         return cls.get_metrics()
+
+    def display_name(self) -> str:
+        return self.NAME

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/tasks/benchmarks/gpqa.py

@@ -232,3 +232,8 @@ class GPQA_COT(GPQA):
         # index 1 selects the letter
         answer_key = choices[correct_answer_position][1]
         return answer_key
+
+
+class GPQA_DIAMOND_COT(GPQA_COT):
+    NAME = "GPQA_DIAMOND_COT"
+    SUBJECTS = ["gpqa_diamond"]

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/tasks/markdown_doc.py

@@ -47,7 +47,7 @@ def markdown_doc(
     buf.write(f"- Module: `{module}`\n\n")
 
     if http_path:
-        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
+        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n")
     else:
         assert example_messages is not None, "a task without a dataset link must supply an example sample"
         for split, size in (split_sizes or {}).items():

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/tasks/registry.py

@@ -18,7 +18,6 @@ __all__ = [
     "EvalFactory",
     "Registry",
     "with_registry",
-    "get_task",
     "is_registered",
     "validate_task_name",
     "registered_task_names",
@@ -36,8 +35,8 @@ class EvalFactory(ABC):
     """
 
     @abstractmethod
-    def task_class(self) -> type[BaseTask]:
-        """Return the task class, importing it on first access if necessary."""
+    def id(self) -> str:
+        "Canonical key used to register this benchmark"
 
     @property
     @abstractmethod
@@ -75,13 +74,10 @@ class EvalFactory(ABC):
         user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
+    @abstractmethod
     def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
         """Render the eval's documentation as markdown."""
-        try:
-            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
-        except (TypeError, ValueError, AssertionError):
-            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
-        return task.markdown_doc(formatters)
+        ...
 
 
 class _Lazy(EvalFactory):
@@ -104,6 +100,9 @@ class _Lazy(EvalFactory):
     def source_module(self) -> str:
         return self._module
 
+    def id(self) -> str:
+        return self._class_name
+
     def task_class(self) -> type[BaseTask]:
         if self._loaded is None:
             module = importlib.import_module(self._module)
@@ -152,6 +151,13 @@ class _Lazy(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
+
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -163,8 +169,8 @@ class _Eager(EvalFactory):
     def source_module(self) -> str:
         return self._task.__module__
 
-    def task_class(self) -> type[BaseTask]:
-        return self._task
+    def id(self) -> str:
+        return self._task.__name__
 
     def create(
         self,
@@ -173,7 +179,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        return self.task_class().with_overwrite(
+        return self._task.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
             custom_hf_revision=custom_hf_revision,
@@ -188,7 +194,7 @@ class _Eager(EvalFactory):
         custom_hf_revision: str | None,
         user_prompt_suffix: str | None = None,
     ) -> BaseTask:
-        perturbation_task_class = create_perturbation_class(self.task_class(), perturbation_config)
+        perturbation_task_class = create_perturbation_class(self._task, perturbation_config)
         return perturbation_task_class.with_overwrite(
             num_fewshot=num_fewshot,
             custom_subjects=custom_subjects,
@@ -198,15 +204,22 @@ class _Eager(EvalFactory):
 
     def response_type(self) -> ResponseType:
         """The eval's response type"""
-        return self.task_class().get_response_type()
+        return self._task.get_response_type()
 
     def metrics(self) -> list[type["BaseMetric"]]:
         """The eval's metrics"""
-        return self.task_class().get_metrics()
+        return self._task.get_metrics()
 
     def display_name(self) -> str:
         """The eval's human-readable display name (the task's ``NAME``)."""
-        return self.task_class().NAME
+        return self._task.NAME
+
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
 
 
 class Registry:
@@ -317,14 +330,6 @@ def validate_task_name(name: str) -> str:
     return name
 
 
-def get_task(name: str, /) -> type[BaseTask]:
-    """Return a registered task for a given name.
-
-    Note: This method will import any lazily registered task.
-    """
-    return _REGISTRY[name].task_class()
-
-
 def register_task(task: type[BaseTask]) -> str:
     """The class name is used as the task name."""
     return registry().register(task)

{eval_framework-0.8.11 → eval_framework-0.9.0}/src/eval_framework/tasks/task_names.py

@@ -24,6 +24,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG")
     register_lazy_task("eval_framework.tasks.benchmarks.goldenswag.GOLDENSWAG_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT")
     register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8K_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.gsm8k.GSM8KBPB")
     register_lazy_task("eval_framework.tasks.benchmarks.math_reasoning.MATHMinervaBPB")