eval-framework 0.10.0__tar.gz → 0.10.4__tar.gz

{eval_framework-0.10.0 → eval_framework-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.10.0
+Version: 0.10.4
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -218,7 +218,7 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.2,<4
+Requires-Dist: nltk>=3.10.3,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
@@ -231,8 +231,8 @@ Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
-Requires-Dist: wandb>=0.28.1,<1
-Requires-Dist: boto3>=1.43.19,<2
+Requires-Dist: wandb>=0.28.2,<1
+Requires-Dist: boto3>=1.43.72,<2
 Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
@@ -241,7 +241,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=3.1.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -315,8 +315,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

{eval_framework-0.10.0 → eval_framework-0.10.4}/README.md

@@ -53,8 +53,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

eval_framework-0.10.4/pyproject.toml

@@ -0,0 +1,162 @@
+[project]
+name = "eval-framework"
+version = "0.10.4"
+description = "Evaluation Framework"
+readme = "README.md"
+requires-python = ">=3.12,<3.15"
+classifiers = [
+    "Operating System :: OS Independent",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pyyaml>=6.0.3,<7",
+    "xmltodict>=1.0.4,<1.1",
+    "pydantic>=2.13.4,<3",
+    "datasets>=5.0.1,<6",
+    "pycountry>=26.2.16,<27",
+    "nltk>=3.10.3,<4",
+    "python-dotenv>=1.2.2,<2",
+    "lingua-language-detector>=2.2.0,<3",
+    "google-crc32c>=1.8.0,<2",
+    "langdetect>=1.0.9,<2",
+    "jsonschema>=4.26.0,<5",
+    "mysql-connector-python>=26.7.0,<27",
+    "psycopg2-binary>=2.9.12,<3",
+    "sympy>=1.14.0,<2",
+    "llm-sandbox[docker]==0.3.44",
+    "jsonlines>=4,<5",
+    "lxml>=6.1.1,<7",
+    "python-iso639>=2026.7.23",
+    "wandb>=0.28.2,<1",
+    "boto3>=1.43.72,<2",
+    "numpy>=2.5.2",
+    "antlr4-python3-runtime==4.11.0",
+    "scipy>=1.18.0,<2",
+]
+
+[project.license]
+file = "LICENSE"
+
+[[project.authors]]
+name = "Aleph Alpha Research"
+
+[project.optional-dependencies]
+determined = [
+    "determined>=0.38.1,<0.39",
+    "tensorboard==2.21.0",
+]
+api = ["aleph-alpha-client>=11.5.1"]
+openai = [
+    "openai>=3.1.0,<4",
+    "tiktoken>=0.13.0,<1",
+    "transformers>=4.45.2,<5",
+]
+transformers = [
+    "transformers>=4.45.2,<5",
+    "torch>=2.13.0,<3",
+    "accelerate>=1.14.0,<2",
+]
+accelerate = ["accelerate"]
+optional = [
+    "transformers>=4.45.2,<5",
+    "jinja2>=3.1.6,<4",
+]
+all = ["eval_framework[determined,api,openai,transformers,accelerate,optional]"]
+
+[project.urls]
+repository = "https://github.com/Aleph-Alpha-Research/eval-framework"
+
+[project.scripts]
+eval_framework = "eval_framework.run:run"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.3.0,<3",
+    "pytest>=9.1.1,<10",
+    "pytest-mock>=3.15.1",
+    "pytest-xdist>=3.8.0,<4",
+    "pytest-sugar>1.1,<2",
+    "types-pyyaml>=6.0.12.20260724,<7",
+    "types-python-dateutil>=2.9.0.20260807,<3",
+    "types-requests>=2.33.0.20260712,<3",
+    "plotly>=6.9.0,<7",
+    "ruff>=0.16.3",
+    "pip-licenses>=5.5.5",
+]
+flash-attn = [
+    "flash-attn>=2.8.3.post1,<2.9",
+    "torch",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.12.6"]
+build-backend = "uv_build"
+
+[tool.uv]
+override-dependencies = ["requests>=2.32,<3"]
+
+[tool.uv.build-backend]
+module-name = [
+    "eval_framework",
+    "template_formatting",
+]
+
+[[tool.uv.extra-build-dependencies.flash-attn]]
+requirement = "torch"
+match-runtime = true
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "UP",
+    "I",
+]
+
+[tool.ruff.lint.isort]
+known-third-party = ["wandb"]
+
+[tool.ruff.lint.extend-per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/tests_eval_framework/tasks/benchmarks/test_csqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gsm8k_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hendrycks_math_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mbpp_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_piqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_siqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_winogrande_ellamind.py" = ["E501"]
+
+[tool.mypy]
+plugins = "pydantic.mypy"
+disallow_untyped_defs = true
+ignore_missing_imports = true
+files = [
+    "src",
+    "utils",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["./tests"]
+minversion = 8.0
+addopts = "-p 'no:legacypath' --doctest-modules -n auto"
+markers = [
+    "gpu: needs a GPU runner, otherwise test can not be run",
+    "cpu_slow: runs for a long time (on CPU)",
+    "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
+    "external_api: needs external services for execution",
+    "formatter_hash: formatter consistency tests using hash comparisons",
+]
+filterwarnings = ["ignore::DeprecationWarning:datasets.utils._dill:"]
+env = ["WANDB_MODE = disabled"]

eval_framework-0.10.0/pyproject.toml → eval_framework-0.10.4/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.0"
+version = "0.10.4"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -25,7 +25,7 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.2,<4",
+  "nltk>=3.10.3,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
@@ -38,8 +38,8 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
-  "wandb>=0.28.1,<1",
-  "boto3>=1.43.19,<2",
+  "wandb>=0.28.2,<1",
+  "boto3>=1.43.72,<2",
   "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,7 +55,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.53.0,<3",
+  "openai>=3.1.0,<4",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -93,7 +93,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.2",
+  "ruff>=0.16.3",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.32,<0.11.33"]
+requires = ["uv_build>=0.12.5,<0.12.6"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.10.0 → eval_framework-0.10.4}/src/eval_framework/tasks/base.py

@@ -2,7 +2,6 @@ import logging
 import os
 import random
 import traceback
-from abc import ABC, abstractmethod
 from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
@@ -10,11 +9,11 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar
 
 import iso639
 from datasets import DatasetDict, DownloadConfig, load_dataset
-from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
 from eval_framework.tasks.dataset_revisions import pinned_revision
 from eval_framework.tasks.markdown_doc import markdown_doc as render_markdown_doc
+from eval_framework.tasks.task import EvalFactory, ResponseType, Sample, Task
 from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import BaseFormatter, Message, Role
 
@@ -26,11 +25,6 @@ RANDOM_SEED = 42
 NO_SUBJECT = "no_subject"
 
 
-class ResponseType(Enum):
-    COMPLETION = "completion"
-    LOGLIKELIHOODS = "loglikelihoods"
-
-
 class TaskStyle(Enum):
     MULTIPLE_CHOICE = "multiple_choice"
     CLOZE = "cloze"
@@ -71,51 +65,11 @@ for language in iso639.ALL_LANGUAGES:
 Language: type[Enum] = Language.add_members(languages)  # type: ignore[no-redef]
 
 
-class Sample(BaseModel):
-    model_config = ConfigDict(extra="forbid")
-    id: int
-    subject: str
-    messages: list[Message]
-    ground_truth: str | list[str] | None
-    possible_completions: list[str] | None
-    context: BaseMetricContext | list[BaseMetricContext] | None = None
-
-
 SubjectType = TypeVar("SubjectType")
 
 logger = logging.getLogger(__name__)
 
 
-class Task(ABC):
-    """The contract a caller relies on to run an evaluation"""
-
-    @abstractmethod
-    def iterate_samples(self, num_samples: int | None = None) -> Iterable[Sample]: ...
-
-    @abstractmethod
-    def generate_completions(
-        self,
-        llm: "BaseLLM",
-        samples: list[Sample],
-        stop_sequences: list[str] | None = None,
-        max_tokens: int | None = None,
-        fail_on_error: bool = True,
-    ) -> list[Completion]:
-        """Run ``llm`` over ``samples`` and return their completions."""
-
-    @abstractmethod
-    def get_metadata(self) -> dict[str, str | list[str]]:
-        """Descriptive metadata about the eval for result reporting."""
-
-    @abstractmethod
-    def get_response_type(self) -> ResponseType: ...
-
-    @abstractmethod
-    def display_name(self) -> str:
-        """Human-readable display name. Is allowed to have special characters and whitespaces."""
-        ...
-
-
 class BaseTask[SubjectType](Task):
     NAME: str
     DATASET_PATH: str
@@ -292,7 +246,6 @@ class BaseTask[SubjectType](Task):
 
         return render_markdown_doc(
             name=self.NAME,
-            module=type(self).__module__,
             dataset_path=dataset_path,
             sample_split=getattr(self, "SAMPLE_SPLIT", None),
             fewshot_split=getattr(self, "FEWSHOT_SPLIT", None),
@@ -547,3 +500,52 @@ def resolve_overwrite_subjects[SubjectType](
             )
 
     return chosen_subjects
+
+
+class Eager(EvalFactory):
+    """Wraps an already-imported task class."""
+
+    def __init__(self, task: type[BaseTask]) -> None:
+        self._task = task
+
+    def id(self) -> str:
+        return self._task.__name__
+
+    def create(
+        self,
+        num_fewshot: int,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
+        seed: int | None = None,
+    ) -> BaseTask:
+        return self._task.with_overwrite(
+            num_fewshot=num_fewshot,
+            custom_subjects=custom_subjects,
+            custom_hf_revision=custom_hf_revision,
+            user_prompt_suffix=user_prompt_suffix,
+            seed=seed,
+        )
+
+    def response_type(self) -> ResponseType:
+        """The eval's response type"""
+        return self._task.get_response_type()
+
+    def metrics(self) -> list[type["BaseMetric"]]:
+        """The eval's metrics"""
+        return self._task.get_metrics()
+
+    def subjects(self) -> list[Any]:
+        """The eval's subjects"""
+        return self._task.SUBJECTS
+
+    def display_name(self) -> str:
+        """The eval's human-readable display name (the task's ``NAME``)."""
+        return self._task.NAME
+
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        try:
+            task = self.create(num_fewshot=1, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
+        except (TypeError, ValueError, AssertionError):
+            task = self.create(num_fewshot=0, custom_subjects=None, custom_hf_revision=None, seed=RANDOM_SEED)
+        return task.markdown_doc(formatters)

{eval_framework-0.10.0 → eval_framework-0.10.4}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -104,6 +104,25 @@ class HumanEvalBPB(HumanEval):
         return [gt] if gt else None
 
 
+class HumanEvalBPB_V2(HumanEvalBPB):
+    """
+    HumanEvalBPB variant that wraps the prompt and canonical solution in markdown code fences
+    instead of prefixing the solution with a leading space, so the loglikelihood boundary
+    starts on a new line.
+    """
+
+    NAME = "Human Eval BPB V2"
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
+        return item["canonical_solution"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return "```python\n" + item["prompt"].rstrip() + "\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return item["canonical_solution"].rstrip() + "\n```"
+
+
 class HumanEval_OLMES(HumanEval):
     """HumanEval OLMES variant replicating codex_humaneval:3shot::olmo3:n32:v2 from oe_eval.
 
@@ -122,10 +141,10 @@ class HumanEval_OLMES(HumanEval):
         self.max_tokens = 1024
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return "```python\n" + item["prompt"]
+        return "```python\n" + item["prompt"].rstrip()
 
     def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        return item["canonical_solution"] + "```"
+        return "\n" + item["canonical_solution"] + "```"
 
 
 class HumanEvalInstruct(HumanEval):
@@ -147,3 +166,13 @@ class HumanEvalInstruct(HumanEval):
 
     def _get_cue_text(self, item: dict[str, Any]) -> str:
         return self.CUE_PREFIX + item["prompt"].lstrip()
+
+
+class HumanEval_NL(HumanEval_OLMES):
+    NAME = "Human Eval Newline OLMES"
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        return "```python\n" + item["prompt"].rstrip() + "\n"
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return item["canonical_solution"].lstrip("\n") + "```"

{eval_framework-0.10.0 → eval_framework-0.10.4}/src/eval_framework/tasks/benchmarks/humaneval_ellamind.py

@@ -5,8 +5,8 @@ https://huggingface.co/datasets/ellamind/humaneval-multilingual
 
 from typing import Any
 
-from eval_framework.tasks.base import BaseTask, Language
-from eval_framework.tasks.benchmarks.humaneval import HumanEval_OLMES
+from eval_framework.tasks.base import BaseTask, Language, Sample
+from eval_framework.tasks.benchmarks.humaneval import HumanEval_NL, HumanEval_OLMES
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 from eval_framework.tasks.task_style import BPBStyle
 
@@ -29,10 +29,6 @@ class HumanEvalDE_OLMES(HumanEval_OLMES):
     SUBJECTS = ["deu"]
     LANGUAGE = Language.DEU
 
-    def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        # Ensure that the code completion starts on a new line.
-        return "```python\n" + item["prompt"].rstrip() + "\n"
-
 
 class HumanEvalDE_BPB_OLMES(BaseTask[str]):
     """German HumanEval - BPB format (loglikelihood of the canonical solution).
@@ -63,3 +59,44 @@ class HumanEvalDE_BPB_OLMES(BaseTask[str]):
 
     def _get_correct_index(self, item: dict[str, Any]) -> int:
         return 0
+
+    def _create_samples(self, item: dict[str, Any], index: int, subject: str) -> list[Sample]:
+        """Creates one or more samples from a single dataset item. Default implementation returns single sample."""
+        messages = self._get_messages(item)
+        messages[-1].content = messages[-1].content.rstrip("\n")
+        return [
+            Sample(
+                id=index,
+                subject=str(subject),
+                messages=messages,
+                ground_truth=self._get_ground_truth(item),
+                possible_completions=self._get_possible_completions(item),
+                context=self._get_context(item),
+            )
+        ]
+
+
+class HumanEvalDE_BPB_OLMES_V2(HumanEvalDE_BPB_OLMES):
+    """HumanEvalDE_BPB_OLMES variant that wraps the prompt and canonical solution in markdown code
+    fences, mirroring the HumanEvalDE_OLMES completion prompt exactly.
+    """
+
+    NAME = "HumanEvalDE_BPB_OLMES V2"
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        # Ensure that the code completion starts on a new line.
+        return "```python\n" + item["prompt"].rstrip()
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        return "\n" + item["canonical_solution"].rstrip() + "\n```"
+
+
+class HumanEvalDE_NL(HumanEval_NL):
+    NAME = "HumanEvalDE_NL"
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
+    DATASET_PATH = "ellamind/humaneval-multilingual"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+    SUBJECTS = ["deu"]
+    LANGUAGE = Language.DEU

{eval_framework-0.10.0 → eval_framework-0.10.4}/src/eval_framework/tasks/eval_config.py

@@ -135,9 +135,7 @@ class EvalConfig(BaseConfig):
 
     @field_serializer("judge_model_args")
     def serialize_judge_model_args(self, value: dict[str, Any]) -> dict[str, Any]:
-        value.pop("api_key", None)
-        value.pop("base_url", None)
-        return value
+        return {k: v for k, v in value.items() if k not in ("api_key", "base_url")}
 
     def model_json_dump(self) -> str:
         model_dump = self.model_dump(mode="json")

eval_framework-0.10.4/src/eval_framework/tasks/lazy.py

@@ -0,0 +1,56 @@
+from collections.abc import Callable, Sequence
+from typing import TYPE_CHECKING, Any
+
+from eval_framework.tasks.task import EvalFactory, ResponseType, Task
+from template_formatting.formatter import BaseFormatter
+
+if TYPE_CHECKING:
+    from eval_framework.metrics.base import BaseMetric
+
+
+class Lazy(EvalFactory):
+    """An ``EvalFactory`` that defers producing the real factory until first use.
+
+    Holds an ``id`` and a ``load`` thunk returning the wrapped ``EvalFactory``; every
+    other call is delegated to that factory, which is built (and memoized) on first
+    access. This keeps ``Lazy`` independent of any concrete task type — the thunk owns
+    whatever loading (e.g. importing a module) the wrapped factory requires.
+    """
+
+    def __init__(self, id: str, load: Callable[[], EvalFactory]) -> None:
+        self._id = id
+        self._load = load
+        self._factory: EvalFactory | None = None
+
+    def _loaded_factory(self) -> EvalFactory:
+        if self._factory is None:
+            self._factory = self._load()
+        return self._factory
+
+    def id(self) -> str:
+        return self._id
+
+    def create(
+        self,
+        num_fewshot: int,
+        custom_subjects: list[str] | None,
+        custom_hf_revision: str | None,
+        user_prompt_suffix: str | None = None,
+        seed: int | None = None,
+    ) -> Task:
+        return self._loaded_factory().create(num_fewshot, custom_subjects, custom_hf_revision, user_prompt_suffix, seed)
+
+    def response_type(self) -> ResponseType:
+        return self._loaded_factory().response_type()
+
+    def metrics(self) -> list[type["BaseMetric"]]:
+        return self._loaded_factory().metrics()
+
+    def subjects(self) -> list[Any]:
+        return self._loaded_factory().subjects()
+
+    def display_name(self) -> str:
+        return self._loaded_factory().display_name()
+
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        return self._loaded_factory().markdown_doc(formatters)

{eval_framework-0.10.0 → eval_framework-0.10.4}/src/eval_framework/tasks/markdown_doc.py

@@ -8,7 +8,6 @@ from template_formatting.formatter import BaseFormatter, Message
 def markdown_doc(
     *,
     name: str,
-    module: str,
     dataset_path: str | None,
     sample_split: str | None,
     fewshot_split: str | None,
@@ -44,8 +43,6 @@ def markdown_doc(
         buf.write(f"LANGUAGE = {language!r}".strip() + "\n")
     buf.write("````\n\n")
 
-    buf.write(f"- Module: `{module}`\n\n")
-
     if http_path:
         buf.write(f"- Link to dataset: [{http_path}]({http_path})\n")
     else: