eval-framework 0.8.4__tar.gz → 0.8.7__tar.gz

{eval_framework-0.8.4 → eval_framework-0.8.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.4
+Version: 0.8.7
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -214,7 +214,7 @@ Classifier: Typing :: Typed
 Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
-Requires-Dist: datasets>=5.0.0,<6
+Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
 Requires-Dist: nltk>=3.10.0,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.48.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.50.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.4 → eval_framework-0.8.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.4"
+version = "0.8.7"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -21,7 +21,7 @@ dependencies = [
   "pyyaml>=6.0.3,<7",
   "xmltodict>=1.0.4,<1.1",
   "pydantic>=2.13.4,<3",
-  "datasets>=5.0.0,<6",
+  "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
   "nltk>=3.10.0,<4",
   "python-dotenv>=1.2.2,<2",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.48.0,<3",
+  "openai>=2.50.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]

{eval_framework-0.8.4 → eval_framework-0.8.7}/src/eval_framework/evaluation_generator.py

@@ -68,8 +68,7 @@ class EvaluationGenerator:
             metric: BaseMetric
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 if llm_judge is None:
-                    assert self.config.llm_judge_class is not None, "The llm_judge_class must be defined in the config."
-                    llm_judge = self.config.llm_judge_class(**self.config.judge_model_args)
+                    llm_judge = self.config.llm_judge()
                 metric = metric_class(
                     llm_judge=llm_judge,
                     randomize_order=self.config.randomize_judge_order,

{eval_framework-0.8.4 → eval_framework-0.8.7}/src/eval_framework/tasks/eval_config.py

@@ -3,14 +3,13 @@ import json
 from pathlib import Path
 from typing import Annotated, Any
 
-from pydantic import AfterValidator, BeforeValidator, Field, field_serializer, field_validator, model_validator
+from pydantic import BeforeValidator, Field, field_serializer, field_validator, model_validator
 
 from eval_framework.base_config import BaseConfig
 from eval_framework.llm.base import BaseLLM
 from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
-from eval_framework.tasks.base import BaseTask
 from eval_framework.tasks.perturbation import PerturbationConfig
-from eval_framework.tasks.registry import get_task, registry, validate_task_name
+from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import ROOT_DIR
 
 # Keys that don't impact actual evaluation results and should be excluded from config dumps for hashing purposes.
@@ -44,7 +43,7 @@ class EvalConfig(BaseConfig):
     num_samples: Annotated[int | None, Field(ge=1)] = 10  # Allows None or int
     max_tokens: int | None = None
     perturbation_config: PerturbationConfig | None = None
-    task_name: Annotated[str, AfterValidator(validate_task_name)]
+    task_name: str
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     user_prompt_suffix: str | None = None
@@ -66,9 +65,9 @@ class EvalConfig(BaseConfig):
     fail_on_error: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
-    @property
-    def task_class(self) -> type[BaseTask]:
-        return get_task(self.task_name)
+    def llm_judge(self) -> BaseLLM:
+        assert self.llm_judge_class is not None, "The llm_judge_class must be defined in the config."
+        return self.llm_judge_class(**self.judge_model_args)
 
     @field_serializer("output_dir")
     def serialize_output_dir(self, value: Path) -> str:
@@ -136,6 +135,12 @@ class EvalConfig(BaseConfig):
             return value.__name__
         return None
 
+    @field_serializer("judge_model_args")
+    def serialize_judge_model_args(self, value: dict[str, Any]) -> dict[str, Any]:
+        value.pop("api_key", None)
+        value.pop("base_url", None)
+        return value
+
     def model_json_dump(self) -> str:
         model_dump = self.model_dump(mode="json")
         return json.dumps(model_dump, sort_keys=True)

{eval_framework-0.8.4 → eval_framework-0.8.7}/src/eval_framework/tasks/registry.py

@@ -2,11 +2,12 @@ import contextlib
 import importlib
 import re
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Iterator
+from collections.abc import Generator, Iterator, Sequence
 from typing import TYPE_CHECKING, Any
 
 from eval_framework.tasks.base import BaseTask, ResponseType
 from eval_framework.tasks.perturbation import PerturbationConfig, create_perturbation_class
+from template_formatting.formatter import BaseFormatter
 
 if TYPE_CHECKING:
     from eval_framework.metrics.base import BaseMetric
@@ -55,10 +56,6 @@ class EvalFactory(ABC):
     def display_name(self) -> str:
         """Human-readable display name. Is allowed to have special characters and whitespaces."""
 
-    @abstractmethod
-    def dataset_path(self) -> str | None:
-        """Identifier of the eval's data source (e.g. a HuggingFace repo id), or None if it has none."""
-
     @abstractmethod
     def create(
         self,
@@ -78,6 +75,14 @@ class EvalFactory(ABC):
         user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        """Render the eval's documentation as markdown."""
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
+
 
 class _Lazy(EvalFactory):
     """
@@ -147,9 +152,6 @@ class _Lazy(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
-    def dataset_path(self) -> str | None:
-        return getattr(self.task_class(), "DATASET_PATH", None)
-
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -206,9 +208,6 @@ class _Eager(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
-    def dataset_path(self) -> str | None:
-        return getattr(self.task_class(), "DATASET_PATH", None)
-
 
 class Registry:
     """A registry for tasks with support for lazy loading.
@@ -226,6 +225,10 @@ class Registry:
         for name, _ in self._registry.values():
             yield name
 
+    def task_names(self) -> list[str]:
+        """The names of all registered tasks."""
+        return list(self)
+
     def items(self) -> Iterator[tuple[str, EvalFactory]]:
         """Iterate over `(task name, EvalFactory)` pairs in the registry."""
         yield from self._registry.values()

eval_framework-0.8.4/src/eval_framework/utils/generate_task_docs.py

@@ -1,153 +0,0 @@
-import argparse
-import os
-from pathlib import Path
-
-import tqdm
-
-from eval_framework.tasks.registry import registered_task_names, registry
-from eval_framework.tasks.task_loader import load_extra_tasks
-from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter
-
-DEFAULT_OUTPUT_DOCS_DIRECTORY = Path("docs/tasks")
-
-EXCLUDED_TASKS: list[str] = []
-
-# Base URL for the main repository to ensure links work even in external/companion repos
-REPO_URL = "https://github.com/Aleph-Alpha-Research/eval-framework/blob/main"
-
-
-def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
-    """Parse command line arguments for the script."""
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--add-prompt-examples",
-        action="store_true",
-        default=False,
-        required=False,
-        help="Unused. Only there for backwards compatibility",
-    )
-    parser.add_argument(
-        "--exclude-tasks",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of task names to exclude from documentation generation.",
-    )
-    parser.add_argument(
-        "--extra-task-modules",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of files and folders containing additional task definitions.",
-    )
-    parser.add_argument(
-        "--formatter",
-        nargs="*",
-        type=str,
-        required=False,
-        default=["ConcatFormatter", "Llama3Formatter"],
-        help="Specify which formatter to use for formatting the task samples. "
-        "If not explicitly specified, default formatters will be used.",
-    )
-    parser.add_argument(
-        "--only-tasks",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of task names to generate documentation for. If empty, all tasks will be processed.",
-    )
-    return parser.parse_args(args=cli_args)
-
-
-def generate_docs_for_task(output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter]) -> None:
-    """Generate documentation for a specific task."""
-    task_class = registry()[task_name].task_class()
-
-    try:
-        task = task_class(num_fewshot=1)
-    except (TypeError, ValueError, AssertionError):
-        task = task_class(num_fewshot=0)
-
-    (output_docs_directory / f"{task_name}.md").write_text(task.markdown_doc(formatters), encoding="utf-8")
-
-
-def generate_readme_list(output_docs_directory: Path, total_tasks: int) -> None:
-    """Generate a README file listing all tasks with total count."""
-
-    with open(f"{output_docs_directory}/README.md", "w") as f:
-        f.write(
-            "# Task documentation\n\n"
-            "This directory contains the generated documentation for all benchmark tasks available in the package.\n\n"
-            f"**Total number of tasks: {total_tasks}**\n\n"
-            "The documentation can be generated or updated with "
-            "`uv run -m eval_framework.utils.generate_task_docs`.\n\n"
-            "NOTE: This is an automatically generated file. Any manual modifications will not be preserved when "
-            "the file is updated.\n\n"
-        )
-
-        f.write("## List of tasks\n\n")
-        # sort files alphabetically and ignore README.md
-        for file in sorted(os.listdir(output_docs_directory)):
-            if file.endswith(".md") and file != "README.md":
-                task_name = file[:-3]
-                f.write(f"- [{task_name}]({task_name}.md)\n")
-
-
-def generate_all_docs(args: argparse.Namespace, output_docs_directory: Path) -> None:
-    # Load extra tasks if specified
-    if args.extra_task_modules:
-        print(f"Loading extra tasks from: {args.extra_task_modules}")
-        load_extra_tasks(args.extra_task_modules)
-
-    # List the tasks to process
-    filtered_tasks = []
-    for task_name in registered_task_names():
-        if args.only_tasks and task_name not in args.only_tasks:
-            continue
-        if task_name in args.exclude_tasks or task_name in EXCLUDED_TASKS:
-            continue
-        filtered_tasks.append(task_name)
-    filtered_tasks.sort()
-
-    print(f"Found {len(filtered_tasks)} tasks to process: {', '.join([task_name for task_name in filtered_tasks])}")
-
-    # List the formatters to use
-    supported_formatters = {f.__class__.__name__: f for f in [ConcatFormatter(), Llama3Formatter()]}
-    formatters = []
-    for f in args.formatter:
-        if f in supported_formatters:
-            formatters.append(supported_formatters[f])
-        else:
-            raise ValueError(f"Unsupported formatter: {f}")
-
-    # Create the output directory if it does not exist
-    os.makedirs(output_docs_directory, exist_ok=True)
-
-    for task_name in tqdm.tqdm(filtered_tasks, desc="Generating documentation for tasks"):
-        try:
-            generate_docs_for_task(
-                output_docs_directory=output_docs_directory,
-                task_name=task_name,
-                formatters=formatters,
-            )
-
-        except Exception as e:
-            print("---")
-            print(f"failed generating documentation for task {task_name}: {e}")
-            file_path = f"{output_docs_directory}/{task_name}.md"
-            if os.path.exists(file_path):
-                os.remove(file_path)
-            print("---")
-
-    # Pass the total number of processed tasks to the README generator
-    generate_readme_list(output_docs_directory=output_docs_directory, total_tasks=len(filtered_tasks))
-
-
-if __name__ == "__main__":
-    print("Generating task documentation...")
-    args = parse_args()
-    generate_all_docs(args, output_docs_directory=DEFAULT_OUTPUT_DOCS_DIRECTORY)