eval-framework 0.8.4__tar.gz → 0.8.11__tar.gz

{eval_framework-0.8.4 → eval_framework-0.8.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.4
+Version: 0.8.11
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -214,16 +214,16 @@ Classifier: Typing :: Typed
 Requires-Dist: pyyaml>=6.0.3,<7
 Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
-Requires-Dist: datasets>=5.0.0,<6
+Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.0,<4
+Requires-Dist: nltk>=3.10.1,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
 Requires-Dist: langdetect>=1.0.9,<2
 Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
-Requires-Dist: mysql-connector-python>=9.7.0,<10
+Requires-Dist: mysql-connector-python>=26.7.0,<27
 Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.39
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.48.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.52.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.4 → eval_framework-0.8.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.4"
+version = "0.8.11"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -21,16 +21,16 @@ dependencies = [
   "pyyaml>=6.0.3,<7",
   "xmltodict>=1.0.4,<1.1",
   "pydantic>=2.13.4,<3",
-  "datasets>=5.0.0,<6",
+  "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.0,<4",
+  "nltk>=3.10.1,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
   "langdetect>=1.0.9,<2", # required by the original ifeval implementation
   "spacy>=3.8.14,<4",
   "jsonschema>=4.26.0,<5",
-  "mysql-connector-python>=9.7.0,<10", # required for sql-related tasks
+  "mysql-connector-python>=26.7.0,<27", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
   "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.39",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.48.0,<3",
+  "openai>=2.52.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -92,7 +92,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.0",
+  "ruff>=0.16.1",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/evaluation_generator.py

@@ -68,8 +68,7 @@ class EvaluationGenerator:
             metric: BaseMetric
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 if llm_judge is None:
-                    assert self.config.llm_judge_class is not None, "The llm_judge_class must be defined in the config."
-                    llm_judge = self.config.llm_judge_class(**self.config.judge_model_args)
+                    llm_judge = self.config.llm_judge()
                 metric = metric_class(
                     llm_judge=llm_judge,
                     randomize_order=self.config.randomize_judge_order,

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/suite.py

@@ -18,7 +18,7 @@ from eval_framework.context.local import _load_model
 from eval_framework.result_processors.result_processor import generate_output_dir
 from eval_framework.run import _run_single_task
 from eval_framework.tasks.eval_config import EvalConfig
-from eval_framework.tasks.registry import is_registered
+from eval_framework.tasks.registry import registry
 
 logger = logging.getLogger(__name__)
 
@@ -108,7 +108,7 @@ class TaskSuite(BaseModel):
         if isinstance(self.tasks, str):
             if self.name is None:
                 self.name = self.tasks
-            if not is_registered(self.tasks):
+            if self.tasks not in registry():
                 raise ValueError(f"Task '{self.tasks}' is not registered.")
         elif not self.tasks:
             raise ValueError(f"TaskSuite '{self.name}': 'tasks' must not be empty.")

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/tasks/eval_config.py

@@ -3,14 +3,13 @@ import json
 from pathlib import Path
 from typing import Annotated, Any
 
-from pydantic import AfterValidator, BeforeValidator, Field, field_serializer, field_validator, model_validator
+from pydantic import BeforeValidator, Field, field_serializer, field_validator, model_validator
 
 from eval_framework.base_config import BaseConfig
 from eval_framework.llm.base import BaseLLM
 from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
-from eval_framework.tasks.base import BaseTask
 from eval_framework.tasks.perturbation import PerturbationConfig
-from eval_framework.tasks.registry import get_task, registry, validate_task_name
+from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import ROOT_DIR
 
 # Keys that don't impact actual evaluation results and should be excluded from config dumps for hashing purposes.
@@ -44,7 +43,7 @@ class EvalConfig(BaseConfig):
     num_samples: Annotated[int | None, Field(ge=1)] = 10  # Allows None or int
     max_tokens: int | None = None
     perturbation_config: PerturbationConfig | None = None
-    task_name: Annotated[str, AfterValidator(validate_task_name)]
+    task_name: str
     task_subjects: list[str] | None = None
     hf_revision: str | None = None
     user_prompt_suffix: str | None = None
@@ -66,9 +65,9 @@ class EvalConfig(BaseConfig):
     fail_on_error: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
-    @property
-    def task_class(self) -> type[BaseTask]:
-        return get_task(self.task_name)
+    def llm_judge(self) -> BaseLLM:
+        assert self.llm_judge_class is not None, "The llm_judge_class must be defined in the config."
+        return self.llm_judge_class(**self.judge_model_args)
 
     @field_serializer("output_dir")
     def serialize_output_dir(self, value: Path) -> str:
@@ -136,6 +135,12 @@ class EvalConfig(BaseConfig):
             return value.__name__
         return None
 
+    @field_serializer("judge_model_args")
+    def serialize_judge_model_args(self, value: dict[str, Any]) -> dict[str, Any]:
+        value.pop("api_key", None)
+        value.pop("base_url", None)
+        return value
+
     def model_json_dump(self) -> str:
         model_dump = self.model_dump(mode="json")
         return json.dumps(model_dump, sort_keys=True)

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/tasks/hf-dataset-revisions.json

@@ -23,7 +23,6 @@
     "google-research-datasets/nq_open": "5dd9790a83002ad084ddeb7c420dc716852c6f28",
     "google/IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84",
     "jzhang86/de_ifeval": "4f52d847003b3c83cc282e9d296853a24b34b19a",
-    "mandarjoshi/trivia_qa": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
     "math-ai/aime25": "563bb8404243c5f09de6ec262f2db674fe5bce9b",
     "math-ai/aime26": "79037aebdb6580008fb960d17cb21fd3099083e3",
     "nuprl/MultiPL-E": "28441b6024e71d4a1c1c0f6bf171c935cd5a43f2",

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/tasks/registry.py

@@ -2,11 +2,12 @@ import contextlib
 import importlib
 import re
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Iterator
+from collections.abc import Generator, Iterator, Sequence
 from typing import TYPE_CHECKING, Any
 
 from eval_framework.tasks.base import BaseTask, ResponseType
 from eval_framework.tasks.perturbation import PerturbationConfig, create_perturbation_class
+from template_formatting.formatter import BaseFormatter
 
 if TYPE_CHECKING:
     from eval_framework.metrics.base import BaseMetric
@@ -55,10 +56,6 @@ class EvalFactory(ABC):
     def display_name(self) -> str:
         """Human-readable display name. Is allowed to have special characters and whitespaces."""
 
-    @abstractmethod
-    def dataset_path(self) -> str | None:
-        """Identifier of the eval's data source (e.g. a HuggingFace repo id), or None if it has none."""
-
     @abstractmethod
     def create(
         self,
@@ -78,6 +75,14 @@ class EvalFactory(ABC):
         user_prompt_suffix: str | None = None,
     ) -> BaseTask: ...
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        """Render the eval's documentation as markdown."""
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None)
+        return task.markdown_doc(formatters)
+
 
 class _Lazy(EvalFactory):
     """
@@ -147,9 +152,6 @@ class _Lazy(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
-    def dataset_path(self) -> str | None:
-        return getattr(self.task_class(), "DATASET_PATH", None)
-
 
 class _Eager(EvalFactory):
     """Wraps an already-imported task class."""
@@ -206,9 +208,6 @@ class _Eager(EvalFactory):
         """The eval's human-readable display name (the task's ``NAME``)."""
         return self.task_class().NAME
 
-    def dataset_path(self) -> str | None:
-        return getattr(self.task_class(), "DATASET_PATH", None)
-
 
 class Registry:
     """A registry for tasks with support for lazy loading.
@@ -226,6 +225,10 @@ class Registry:
         for name, _ in self._registry.values():
             yield name
 
+    def task_names(self) -> list[str]:
+        """The names of all registered tasks."""
+        return list(self)
+
     def items(self) -> Iterator[tuple[str, EvalFactory]]:
         """Iterate over `(task name, EvalFactory)` pairs in the registry."""
         yield from self._registry.values()
@@ -252,10 +255,6 @@ class Registry:
 
         return factory
 
-    def add(self, task: type[BaseTask]) -> None:
-        task_key = self._task_key(task.NAME)
-        self._registry[task_key] = (task.NAME, _Eager(task))
-
     def __setitem__(self, name: str, factory: EvalFactory) -> None:
         task_key = self._task_key(name)
         if task_key in self._registry:
@@ -263,6 +262,24 @@ class Registry:
 
         self._registry[task_key] = (name, factory)
 
+    def register(self, task: type[BaseTask]) -> str:
+        """The class name is used as the task name."""
+        if not issubclass(task, BaseTask):
+            raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
+        name = task.__name__
+        self[name] = _Eager(task)
+        return name
+
+    def register_lazy(self, class_path: str, /) -> None:
+        """Register a task by its dotted class path, without importing its module."""
+        if "." not in class_path:
+            raise ValueError(
+                f"Invalid class path `{class_path}`. This needs to be a global path like "
+                "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
+            )
+        base_module, class_name = class_path.rsplit(".", maxsplit=1)
+        self[class_name] = _Lazy(class_name=class_name, module=base_module)
+
 
 _REGISTRY = Registry()
 
@@ -295,7 +312,7 @@ def is_registered(name: str, /) -> bool:
 
 def validate_task_name(name: str) -> str:
     """Pydantic-style validator for task names."""
-    if not is_registered(name):
+    if name not in registry():
         raise ValueError(f"Task not registered: {name}")
     return name
 
@@ -310,28 +327,9 @@ def get_task(name: str, /) -> type[BaseTask]:
 
 def register_task(task: type[BaseTask]) -> str:
     """The class name is used as the task name."""
-    if not issubclass(task, BaseTask):
-        raise ValueError(f"Can only register subclasses of BaseTask, got {task}")
-    name = task.__name__
-    _REGISTRY[name] = _Eager(task)
-    return name
+    return registry().register(task)
 
 
 def register_lazy_task(class_path: str, /) -> None:
-    """Register a task without importing it.
-
-    Lazily register a task without importing the module.
-
-    Args:
-        class_path: The full path to the task class. For example,
-            `eval_framework.tasks.benchmarks.mmlu.MMLU`.
-        extras: Any extra dependencies of `eval_framework` that need to be installed for this task.
-    """
-    if "." not in class_path:
-        raise ValueError(
-            f"Invalid class path `{class_path}`. This needs to be a global path like "
-            "`eval_framework.tasks.benchmarks.mmlu.MMLU`): "
-        )
-
-    base_module, class_name = class_path.rsplit(".", maxsplit=1)
-    _REGISTRY[class_name] = _Lazy(class_name=class_name, module=base_module)
+    """Register a task by its dotted class path, without importing its module."""
+    registry().register_lazy(class_path)

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/tasks/task_loader.py

@@ -8,7 +8,8 @@ from types import ModuleType
 from typing import Any
 
 from eval_framework.tasks.base import BaseTask
-from eval_framework.tasks.registry import is_registered, register_task
+from eval_framework.tasks.registry import Registry
+from eval_framework.tasks.registry import registry as global_registry
 
 logger = logging.getLogger(__name__)
 
@@ -46,14 +47,13 @@ def import_file(f: str | os.PathLike, /) -> Any:
     return user_module
 
 
-def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
+def load_extra_tasks(module_paths: Sequence[str | os.PathLike], registry: Registry | None = None) -> None:
     """Dynamically load and register user-defined tasks from a list of files or directories.
 
-    Each .py file found will be imported, and any BaseTask subclass will be registered
-    in the TaskName enum for use by name.
-    Provides clear error messages for missing/invalid files or import errors.
+    Each .py file found is imported, and any BaseTask subclass is registered
     """
     assert not (isinstance(module_paths, str)), "module_paths must be a sequence of strings / os.PathLike objects"
+    registry = registry if registry is not None else global_registry()
     for file_path in find_all_python_files(*module_paths):
         user_module = import_file(file_path)
 
@@ -63,7 +63,7 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                 if not hasattr(obj, "NAME"):
                     logger.info(f"[User Task Loader] Skipping {obj.__module__} - no NAME attribute present.")
                 else:
-                    if is_registered(obj.NAME):
+                    if obj.NAME in registry:
                         # two classes with the same NAME attribute
                         logger.info(obj.__module__)
 
@@ -77,5 +77,5 @@ def load_extra_tasks(module_paths: Sequence[str | os.PathLike]) -> None:
                     else:
                         # if there is no duplicate name conflict then register the new task
                         class_obj = getattr(user_module, name)
-                        register_task(class_obj)
+                        registry.register(class_obj)
                         logger.info(f"[User Task Loader] Registered task: {class_obj.NAME}")

{eval_framework-0.8.4 → eval_framework-0.8.11}/src/eval_framework/tasks/task_names.py

@@ -71,8 +71,6 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA")
-    register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TriviaQA_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")
     register_lazy_task("eval_framework.tasks.benchmarks.csqa.CommonsenseQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.drop.DropCompletion_OLMES")
@@ -81,9 +79,3 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.medqa.MedQAMC_OLMES")
-    try:
-        # Importing the companion registers the additional tasks from the module.
-        # This is mostly for convenience for internal use-cases
-        import eval_framework_companion  # noqa
-    except ImportError:
-        pass

eval_framework-0.8.4/src/eval_framework/tasks/benchmarks/triviaqa.py

@@ -1,78 +0,0 @@
-import random
-from typing import Any
-
-from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
-from eval_framework.metrics.completion.f1 import F1, F1SquadNormalized
-from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
-from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
-
-
-class TRIVIAQA(BaseTask[str]):
-    """Trivia QA dataset: https://huggingface.co/datasets/mandarjoshi/trivia_qa"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA"
-    DATASET_PATH = "mandarjoshi/trivia_qa"
-    SAMPLE_SPLIT = "validation"
-    FEWSHOT_SPLIT = "train"
-    RESPONSE_TYPE = ResponseType.COMPLETION
-    METRICS = [AccuracyCompletion, F1]
-    SUBJECTS = ["rc.wikipedia.nocontext"]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
-    LANGUAGE = Language.ENG
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = ["\n"]
-        self.max_tokens = 400  # the max length of the ground truth is 282 characters while the average is ~16
-        self.rnd_choice_shuffle = random.Random()
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        prompt = f"Question: {item['question'].strip()}\nAnswer:"
-        return prompt
-
-    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        target = self._get_ground_truth(item)[0]
-        assert target is not None
-        assert isinstance(target, str)
-        return f" {target}"
-
-    def _get_ground_truth(self, item: dict[str, Any]) -> list[str]:
-        return item["answer"]["aliases"]
-
-    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
-        return completion_text.strip().rstrip(".")
-
-
-class TriviaQA_MA(TRIVIAQA):
-    """TriviaQA with the exact system prompt used in MA training"""
-
-    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
-
-    NAME = "TriviaQA_MA"
-    SUBJECTS = ["rc.wikipedia"]
-    UNANSWERABLE_STR = "unanswerable"
-
-    METRICS = [AccuracyCompletion, F1, F1SquadNormalized]
-    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer", "Context", "unanswerable"]
-
-    def __init__(self, num_fewshot: int = 0) -> None:
-        super().__init__(num_fewshot)
-        self.stop_sequences = []
-        self.max_tokens = 27_000
-
-    def _get_context_text(self, item: dict[str, Any]) -> str:
-        return "\n\n".join(item["entity_pages"]["wiki_context"])
-
-    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
-        return (
-            "You are a helpful assistant and will answer the user's questions carefully, "
-            "logically, accurately and well-reasoned.\n"
-            "Use the given context to answer the question faithfully. Answer only if the "
-            f"answer is present in the given context, otherwise respond with '{self.UNANSWERABLE_STR}' "
-            "if the answer is not present in the context."
-        )
-
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return f"Context:\n{self._get_context_text(item)}\n\nQuestion:\n{item['question'].strip()}\n"

eval_framework-0.8.4/src/eval_framework/utils/generate_task_docs.py

@@ -1,153 +0,0 @@
-import argparse
-import os
-from pathlib import Path
-
-import tqdm
-
-from eval_framework.tasks.registry import registered_task_names, registry
-from eval_framework.tasks.task_loader import load_extra_tasks
-from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter
-
-DEFAULT_OUTPUT_DOCS_DIRECTORY = Path("docs/tasks")
-
-EXCLUDED_TASKS: list[str] = []
-
-# Base URL for the main repository to ensure links work even in external/companion repos
-REPO_URL = "https://github.com/Aleph-Alpha-Research/eval-framework/blob/main"
-
-
-def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
-    """Parse command line arguments for the script."""
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--add-prompt-examples",
-        action="store_true",
-        default=False,
-        required=False,
-        help="Unused. Only there for backwards compatibility",
-    )
-    parser.add_argument(
-        "--exclude-tasks",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of task names to exclude from documentation generation.",
-    )
-    parser.add_argument(
-        "--extra-task-modules",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of files and folders containing additional task definitions.",
-    )
-    parser.add_argument(
-        "--formatter",
-        nargs="*",
-        type=str,
-        required=False,
-        default=["ConcatFormatter", "Llama3Formatter"],
-        help="Specify which formatter to use for formatting the task samples. "
-        "If not explicitly specified, default formatters will be used.",
-    )
-    parser.add_argument(
-        "--only-tasks",
-        nargs="*",
-        type=str,
-        default=[],
-        required=False,
-        help="List of task names to generate documentation for. If empty, all tasks will be processed.",
-    )
-    return parser.parse_args(args=cli_args)
-
-
-def generate_docs_for_task(output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter]) -> None:
-    """Generate documentation for a specific task."""
-    task_class = registry()[task_name].task_class()
-
-    try:
-        task = task_class(num_fewshot=1)
-    except (TypeError, ValueError, AssertionError):
-        task = task_class(num_fewshot=0)
-
-    (output_docs_directory / f"{task_name}.md").write_text(task.markdown_doc(formatters), encoding="utf-8")
-
-
-def generate_readme_list(output_docs_directory: Path, total_tasks: int) -> None:
-    """Generate a README file listing all tasks with total count."""
-
-    with open(f"{output_docs_directory}/README.md", "w") as f:
-        f.write(
-            "# Task documentation\n\n"
-            "This directory contains the generated documentation for all benchmark tasks available in the package.\n\n"
-            f"**Total number of tasks: {total_tasks}**\n\n"
-            "The documentation can be generated or updated with "
-            "`uv run -m eval_framework.utils.generate_task_docs`.\n\n"
-            "NOTE: This is an automatically generated file. Any manual modifications will not be preserved when "
-            "the file is updated.\n\n"
-        )
-
-        f.write("## List of tasks\n\n")
-        # sort files alphabetically and ignore README.md
-        for file in sorted(os.listdir(output_docs_directory)):
-            if file.endswith(".md") and file != "README.md":
-                task_name = file[:-3]
-                f.write(f"- [{task_name}]({task_name}.md)\n")
-
-
-def generate_all_docs(args: argparse.Namespace, output_docs_directory: Path) -> None:
-    # Load extra tasks if specified
-    if args.extra_task_modules:
-        print(f"Loading extra tasks from: {args.extra_task_modules}")
-        load_extra_tasks(args.extra_task_modules)
-
-    # List the tasks to process
-    filtered_tasks = []
-    for task_name in registered_task_names():
-        if args.only_tasks and task_name not in args.only_tasks:
-            continue
-        if task_name in args.exclude_tasks or task_name in EXCLUDED_TASKS:
-            continue
-        filtered_tasks.append(task_name)
-    filtered_tasks.sort()
-
-    print(f"Found {len(filtered_tasks)} tasks to process: {', '.join([task_name for task_name in filtered_tasks])}")
-
-    # List the formatters to use
-    supported_formatters = {f.__class__.__name__: f for f in [ConcatFormatter(), Llama3Formatter()]}
-    formatters = []
-    for f in args.formatter:
-        if f in supported_formatters:
-            formatters.append(supported_formatters[f])
-        else:
-            raise ValueError(f"Unsupported formatter: {f}")
-
-    # Create the output directory if it does not exist
-    os.makedirs(output_docs_directory, exist_ok=True)
-
-    for task_name in tqdm.tqdm(filtered_tasks, desc="Generating documentation for tasks"):
-        try:
-            generate_docs_for_task(
-                output_docs_directory=output_docs_directory,
-                task_name=task_name,
-                formatters=formatters,
-            )
-
-        except Exception as e:
-            print("---")
-            print(f"failed generating documentation for task {task_name}: {e}")
-            file_path = f"{output_docs_directory}/{task_name}.md"
-            if os.path.exists(file_path):
-                os.remove(file_path)
-            print("---")
-
-    # Pass the total number of processed tasks to the README generator
-    generate_readme_list(output_docs_directory=output_docs_directory, total_tasks=len(filtered_tasks))
-
-
-if __name__ == "__main__":
-    print("Generating task documentation...")
-    args = parse_args()
-    generate_all_docs(args, output_docs_directory=DEFAULT_OUTPUT_DOCS_DIRECTORY)