eval-framework 0.8.1__tar.gz → 0.8.3__tar.gz

{eval_framework-0.8.1 → eval_framework-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.1
+Version: 0.8.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -229,10 +229,10 @@ Requires-Dist: sympy>=1.13.1,<2
 Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
-Requires-Dist: python-iso639>=2026.4.20
+Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
-Requires-Dist: numpy>=2.2.6
+Requires-Dist: numpy>=2.5.1
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.48.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -528,6 +528,8 @@ If you use `eval-framework` in your research, please cite:
 
 This project is licensed under the [Apache License 2.0](LICENSE).
 
+The constituent tasks' datasets can be subject to specific, more restrictive license terms by third parties. By using `eval-framework` you agree to adhere to and be bound by such license terms in the individual case.
+
 <br><br>
 ---
 

{eval_framework-0.8.1 → eval_framework-0.8.3}/README.md

@@ -267,6 +267,8 @@ If you use `eval-framework` in your research, please cite:
 
 This project is licensed under the [Apache License 2.0](LICENSE).
 
+The constituent tasks' datasets can be subject to specific, more restrictive license terms by third parties. By using `eval-framework` you agree to adhere to and be bound by such license terms in the individual case.
+
 <br><br>
 ---
 

{eval_framework-0.8.1 → eval_framework-0.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.1"
+version = "0.8.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -36,10 +36,10 @@ dependencies = [
   "llm-sandbox[docker]==0.3.39",
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
-  "python-iso639>=2026.4.20",
+  "python-iso639>=2026.7.23",
   "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
-  "numpy>=2.2.6",
+  "numpy>=2.5.1",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.46.0,<3",
+  "openai>=2.48.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -88,11 +88,11 @@ dev = [
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
-  "types-pyyaml>=6.0.12.20260518,<7",
+  "types-pyyaml>=6.0.12.20260724,<7",
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.15.22",
+  "ruff>=0.16.0",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -101,7 +101,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.29,<0.11.30"]
+requires = ["uv_build>=0.11.32,<0.11.33"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/evaluation_generator.py

@@ -20,7 +20,7 @@ from eval_framework.tasks.base import ResponseType
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import RED, RESET
-from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag
 
 logger = logging.getLogger(__name__)
 
@@ -79,7 +79,6 @@ class EvaluationGenerator:
             metric.fail_on_error = self.config.fail_on_error
 
             logger.info(f"Starting calculation of {metric.NAME}")
-            safe_tqdm_write(f"INFO: Calculating {metric.NAME}")
             for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()):
                 if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing:
                     continue
@@ -117,7 +116,6 @@ class EvaluationGenerator:
                         self.result_processor.save_metrics_result(result)
 
             logger.info(f"Completed calculation of {metric.NAME}")
-            safe_tqdm_write(f"INFO: Completed {metric.NAME}")
 
         if not self.save_intermediate_results:
             self.result_processor.save_metrics_results(results)

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/metrics/completion/language_checker.py

@@ -41,9 +41,6 @@ class LanguageConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
-        if not response.completion:
-            return []  # No completion means no language to detect, so it is excluded from aggregation
-
         completion_language = response.get_completion_language()
         target_language = response.get_instruction_language()
         if completion_language == target_language == "":
@@ -60,9 +57,6 @@ class LanguageRawConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
-        if not response.raw_completion:
-            return []  # No completion means no language to detect, so it is excluded from aggregation
-
         raw_completion_language = response.get_raw_completion_language()
         target_language = response.get_instruction_language()
 

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/response_generator.py

@@ -30,7 +30,7 @@ from eval_framework.tasks.base import Language, ResponseType, Sample
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.utils import raise_errors
 from eval_framework.utils.constants import RED, RESET
-from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag
 
 logger = logging.getLogger(__name__)
 
@@ -232,9 +232,7 @@ class ResponseGenerator:
             if not samples_batch:
                 return
             if len(samples_batch) > 1:
-                log_msg = "Processing batch..."
-                logger.info(log_msg)  # For log files
-                safe_tqdm_write(log_msg)  # For console display with tqdm
+                logger.info("Processing batch...")
 
             responses_batch = generative_output_function(samples_batch)
             responses.extend(responses_batch)
@@ -268,17 +266,13 @@ class ResponseGenerator:
                 sample_index = i + 1
 
                 if sample.id in subject_response_id_mapping.get(sample.subject, []):
-                    log_msg = (
+                    logger.info(
                         f"Task: {self.response_type.value}{subject} - Sample: {sample_index} - skipping, already done."
                     )
-                    logger.info(log_msg)  # For log files
-                    safe_tqdm_write(log_msg)  # For console display with tqdm
                     pbar.update(1)
                     continue
 
-                log_msg = f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}"
-                logger.info(log_msg)  # For log files
-                safe_tqdm_write(log_msg)  # For console display with tqdm
+                logger.info(f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}")
                 pbar.set_postfix_str(f"Sample {sample_index}/{total_num_samples}")
                 pbar.update(1)
 
@@ -289,9 +283,7 @@ class ResponseGenerator:
                     samples_batch = []
 
                 if should_preempt_callable():
-                    log_msg = "Preempt"
-                    logger.info(log_msg)  # For log files
-                    safe_tqdm_write(log_msg)  # For console display with tqdm
+                    logger.info("Preempt")
                     if not self.save_intermediate_results:
                         self.result_processor.save_responses(responses)
                     return responses, True

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/tasks/base.py

@@ -3,7 +3,7 @@ import os
 import random
 import traceback
 from abc import ABC
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Self, TypeVar
@@ -14,8 +14,9 @@ from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
 from eval_framework.tasks.dataset_revisions import pinned_revision
+from eval_framework.tasks.markdown_doc import markdown_doc as render_markdown_doc
 from eval_framework.tasks.utils import classproperty, raise_errors
-from template_formatting.formatter import Message, Role
+from template_formatting.formatter import BaseFormatter, Message, Role
 
 if TYPE_CHECKING:
     from eval_framework.llm.base import BaseLLM
@@ -280,6 +281,35 @@ class BaseTask[SubjectType](ABC):
                         done = True
                         break
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        """Render this task's documentation as markdown."""
+        dataset_path = getattr(self, "DATASET_PATH", None)
+        example_messages = split_sizes = possible_completions = ground_truth = None
+        if dataset_path is None:
+            sample = next(iter(self.iterate_samples(1)))
+            example_messages = sample.messages
+            split_sizes = {split: len(self.dataset[split]) for split in self.dataset}
+            possible_completions = sample.possible_completions
+            ground_truth = sample.ground_truth
+
+        return render_markdown_doc(
+            name=self.NAME,
+            module=type(self).__module__,
+            dataset_path=dataset_path,
+            sample_split=getattr(self, "SAMPLE_SPLIT", None),
+            fewshot_split=getattr(self, "FEWSHOT_SPLIT", None),
+            response_type=self.get_response_type().name,
+            metrics=[m.__name__ for m in self.get_metrics()],
+            subjects=getattr(self, "SUBJECTS", None),
+            language=getattr(self, "LANGUAGE", None),
+            num_fewshot=self.num_fewshot,
+            formatters=formatters,
+            example_messages=example_messages,
+            split_sizes=split_sizes,
+            possible_completions=possible_completions,
+            ground_truth=ground_truth,
+        )
+
     def _create_samples(self, item: dict[str, Any], index: int, subject: str) -> list[Sample]:
         """Creates one or more samples from a single dataset item. Default implementation returns single sample."""
         return [

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -10,6 +10,7 @@ from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglik
 from eval_framework.shared.types import BaseMetricContext
 from eval_framework.tasks.base import BaseTask, Language, ResponseType, Sample
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
+from eval_framework.tasks.task_style import BPBStyle
 
 logger = logging.getLogger(__name__)
 
@@ -314,3 +315,90 @@ class MBPP_OLMES(MBPP):
         mbpp_ground_truth = str(sample.ground_truth)
         code = self._code_expander(extracted_code, mbpp_ground_truth)
         return code
+
+
+class MBPP_EvalPlus(MBPP):
+    """Version that faithfully follows the EvalPlus prompt format, which OLMES only approximated."""
+
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
+    NAME = "MBPP_EvalPlus"
+    FEWSHOT_SPLIT = "test"
+
+    def __init__(self, num_fewshot: int = 3) -> None:
+        super().__init__(num_fewshot)
+        if num_fewshot != 3:
+            logger.warning(f"MBPP_EvalPlus supports only 3-shot, got {num_fewshot}")
+        self.stop_sequences = ["```", "\nassert", "\nPlease provide"]
+
+    def _get_instruction_text(self, item: dict[str, Any]) -> str:
+        text = item["text"] if "text" in item else item["prompt"]
+        test = item["test_list"][0]
+        return (
+            "Please provide a self-contained Python script that solves the following problem"
+            f" in a markdown code block:\n```\n{text.strip()}\n{test}\n```\n"
+        )
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return (
+            "Below is a Python script with a self-contained function that solves the problem"
+            " and passes corresponding tests:\n```python"
+        )
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        # The cue ends at ```python (no trailing newline, since it is stripped as the final eval
+        # message); the fewshot target adds the newline before the code explicitly.
+        return self._get_cue_text(item) + "\n" + item["code"] + "\n```"
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return list(_OLMES_FEWSHOT_EXAMPLES)
+
+    def post_process_generated_completion(self, completion_text: str, sample: Sample) -> str:  # type: ignore[override]
+        extracted_code = completion_text + "\n"
+        mbpp_ground_truth = str(sample.ground_truth)
+        code = self._code_expander(extracted_code, mbpp_ground_truth)
+        return code
+
+
+class MBPP_BPB_EvalPlus(BaseTask[str]):
+    """BPB (loglikelihood) counterpart of MBPP_EvalPlus."""
+
+    REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
+
+    NAME = "MBPP_BPB_EvalPlus"
+    DATASET_PATH = "google-research-datasets/mbpp"
+    SAMPLE_SPLIT = "test"
+    FEWSHOT_SPLIT = "test"
+    SUBJECTS = ["full"]
+    LANGUAGE = Language.ENG
+    TASK_STYLER = BPBStyle(
+        question_prefix="",
+        trailing_newline=False,
+        cue_text=(
+            "Below is a Python script with a self-contained function that solves the problem"
+            f" and passes corresponding tests:\n{BEGIN}"
+        ),
+        leading_space_continuations=False,
+    )
+
+    def __init__(self, num_fewshot: int = 3) -> None:
+        super().__init__(num_fewshot)
+        if num_fewshot != 3:
+            logger.warning(f"MBPP_BPB_EvalPlus supports only 3-shot, got {num_fewshot}")
+
+    def _get_raw_question(self, item: dict[str, Any]) -> str:
+        text = item["text"] if "text" in item else item["prompt"]
+        test = item["test_list"][0]
+        return (
+            "Please provide a self-contained Python script that solves the following problem"
+            f" in a markdown code block:\n```\n{text.strip()}\n{test}\n```\n"
+        )
+
+    def _get_choices(self, item: dict[str, Any]) -> list[str]:
+        return ["\n" + item["code"] + f"\n{END}"]
+
+    def _get_correct_index(self, item: dict[str, Any]) -> int:
+        return 0
+
+    def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
+        return list(_OLMES_FEWSHOT_EXAMPLES)

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/tasks/benchmarks/squad.py

@@ -11,7 +11,7 @@ from huggingface_hub.errors import RevisionNotFoundError
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 from eval_framework.metrics.completion.f1 import F1, F1SquadNormalized
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
-from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, SubjectType
+from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
@@ -252,7 +252,7 @@ class SQuAD2_MA(SQUAD2):
     def __init__(self, num_fewshot: int = 0) -> None:
         super().__init__(num_fewshot)
         self.stop_sequences = []
-        self.max_tokens = None
+        self.max_tokens = 10_000
 
     def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
         return (
@@ -261,11 +261,46 @@ class SQuAD2_MA(SQUAD2):
             "Use the given context to answer the question faithfully. Answer only if the "
             f"answer is present in the given context, otherwise respond with '{self.UNANSWERABLE_STR}' "
             "if the answer is not present in the context."
+            "Always begin your answer with 'Final answer:'"
         )
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         return f"Context:\n{item['context']}\n\nQuestion:\n{item['question']}\n"
 
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        """Clean up the generated answer."""
+        # Remove common prefixes and clean whitespace
+        cleaned = completion_text.strip()
+        common_prefixes = ["Answer", "Final answer"]
+        # A list comprehension (not a generator) is required here: list.extend
+        # consumes lazily, so a generator over the list being extended never
+        # terminates and grows the list unboundedly until OOM.
+        common_prefixes.extend([f"**{prefix}**" for prefix in common_prefixes])
+        common_prefixes.reverse()
+
+        # Search for the last occurrence of any common prefix, and take only what's after it.
+        for prefix in common_prefixes:
+            idx = cleaned.rfind(prefix + ":")
+            if idx != -1:
+                cleaned = cleaned[idx + len(prefix) + 1 :].strip()
+                break
+        return cleaned
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> list[str]:
+        text_ = item["answers"]["text"]
+        ground_truth_for_unanswerable = [
+            self.UNANSWERABLE_STR,
+            self.UNANSWERABLE_STR + " ",
+            self.UNANSWERABLE_STR.capitalize(),
+        ]
+        ground_truths = text_ if text_ else ground_truth_for_unanswerable
+        return ground_truths
+
+
+class SQuAD2_MA_NO_SYSPROMPT(SQuAD2_MA):
+    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
+        return ""
+
 
 class SQuAD_OLMES(SQUAD):
     """SQuAD variant matching OLMES implementation."""

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/tasks/benchmarks/triviaqa.py

@@ -60,7 +60,7 @@ class TriviaQA_MA(TRIVIAQA):
     def __init__(self, num_fewshot: int = 0) -> None:
         super().__init__(num_fewshot)
         self.stop_sequences = []
-        self.max_tokens = None
+        self.max_tokens = 27_000
 
     def _get_context_text(self, item: dict[str, Any]) -> str:
         return "\n\n".join(item["entity_pages"]["wiki_context"])

eval_framework-0.8.3/src/eval_framework/tasks/markdown_doc.py

@@ -0,0 +1,81 @@
+from collections.abc import Sequence
+from io import StringIO
+from typing import Any
+
+from template_formatting.formatter import BaseFormatter, Message
+
+
+def markdown_doc(
+    *,
+    name: str,
+    module: str,
+    dataset_path: str | None,
+    sample_split: str | None,
+    fewshot_split: str | None,
+    response_type: str,
+    metrics: Sequence[str],
+    subjects: Any,
+    language: Any,
+    num_fewshot: int,
+    formatters: Sequence[BaseFormatter],
+    example_messages: list[Message] | None,
+    split_sizes: dict[str, int] | None,
+    possible_completions: str | list[str] | None,
+    ground_truth: str | list[str] | None,
+) -> str:
+    """Render a task's documentation as markdown"""
+    buf = StringIO()
+    buf.write(f"# {name}\n\n")
+    http_path = f"https://huggingface.co/datasets/{dataset_path}" if dataset_path else None
+
+    buf.write("````\n")  # fence with 4 thicks because some prompts have code blocks with 3 thicks
+    buf.write(f"NAME = {name}".strip() + "\n")
+    if dataset_path is not None:
+        buf.write(f"DATASET_PATH = {dataset_path}".strip() + "\n")
+    if sample_split is not None:
+        buf.write(f"SAMPLE_SPLIT = {sample_split}".strip() + "\n")
+    if fewshot_split is not None:
+        buf.write(f"FEWSHOT_SPLIT = {fewshot_split}".strip() + "\n")
+    buf.write(f"RESPONSE_TYPE = {response_type}".strip() + "\n")
+    buf.write(f"METRICS = [{', '.join(metrics)}]".strip() + "\n")
+    if subjects is not None:
+        buf.write(f"SUBJECTS = {subjects!r}".strip() + "\n")
+    if language is not None:
+        buf.write(f"LANGUAGE = {language!r}".strip() + "\n")
+    buf.write("````\n\n")
+
+    buf.write(f"- Module: `{module}`\n\n")
+
+    if http_path:
+        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
+    else:
+        assert example_messages is not None, "a task without a dataset link must supply an example sample"
+        for split, size in (split_sizes or {}).items():
+            buf.write(f"- `{split}` has {size} samples\n\n")
+
+        for formatter in formatters:
+            buf.write(f"## Example prompt with {formatter.__class__.__name__} ({num_fewshot}-shot)\n\n")
+            formatted_sample = formatter.format(example_messages, output_mode="string")
+            buf.write("````\n")
+            buf.write(f'"{formatted_sample}"')
+            buf.write("\n````\n\n")
+
+        buf.write("## Possible completions:\n\n")
+        buf.write("````\n")
+        if possible_completions:
+            for item in possible_completions if isinstance(possible_completions, list) else [possible_completions]:
+                buf.write(f'- "{item}"\n')
+        else:
+            buf.write("None\n")
+        buf.write("````\n\n")
+
+        buf.write("## Ground truth:\n\n")
+        buf.write("````\n")
+        if ground_truth:
+            for item in ground_truth if isinstance(ground_truth, list) else [ground_truth]:
+                buf.write(f'- "{item}"\n')
+        else:
+            buf.write("None\n")
+        buf.write("````\n")
+
+    return buf.getvalue()

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/tasks/task_names.py

@@ -51,6 +51,8 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.multipl_e.MultiPLEMBPPSh")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPPBPB")
     register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_EvalPlus")
+    register_lazy_task("eval_framework.tasks.benchmarks.mbpp.MBPP_BPB_EvalPlus")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_IDK")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_OLMES")
@@ -68,6 +70,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
     register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA")
     register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TriviaQA_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")

{eval_framework-0.8.1 → eval_framework-0.8.3}/src/eval_framework/utils/generate_task_docs.py

@@ -1,7 +1,5 @@
 import argparse
-import inspect
 import os
-import re
 from pathlib import Path
 
 import tqdm
@@ -27,7 +25,7 @@ def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
         action="store_true",
         default=False,
         required=False,
-        help="If set, examples prompts for each of the formatters will be added in the generated docs.",
+        help="Unused. Only there for backwards compatibility",
     )
     parser.add_argument(
         "--exclude-tasks",
@@ -65,108 +63,16 @@ def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
     return parser.parse_args(args=cli_args)
 
 
-def generate_docs_for_task(
-    output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter], add_prompt_examples: bool
-) -> None:
+def generate_docs_for_task(output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter]) -> None:
     """Generate documentation for a specific task."""
-    eval_ = registry()[task_name]
-    task_class = eval_.task_class()
+    task_class = registry()[task_name].task_class()
 
     try:
-        num_fewshot = 1
-        task = task_class(num_fewshot=num_fewshot)
+        task = task_class(num_fewshot=1)
     except (TypeError, ValueError, AssertionError):
-        try:
-            num_fewshot = 0
-            task = task_class(num_fewshot=num_fewshot)
-        except (TypeError, ValueError, AssertionError):
-            try:
-                task = task_class()
-                num_fewshot = task.num_fewshot
-            except Exception as e:
-                print(f"Failed to instantiate task {task_name}: {e}")
-                return
-
-    with open(f"{output_docs_directory}/{task_name}.md", "w") as f:
-        f.write(f"# {task_name}\n\n")
-        dataset_path = eval_.dataset_path()
-        http_path = f"https://huggingface.co/datasets/{dataset_path}" if dataset_path else None
-
-        f.write("````\n")  # fence with 4 thicks because some prompts have code blocks with 3 thicks
-        f.write(f"NAME = {task_name}".strip() + "\n")
-        if dataset_path is not None:
-            f.write(f"DATASET_PATH = {dataset_path}".strip() + "\n")
-        if hasattr(task, "SAMPLE_SPLIT"):
-            f.write(f"SAMPLE_SPLIT = {task.SAMPLE_SPLIT}".strip() + "\n")
-        if hasattr(task, "FEWSHOT_SPLIT"):
-            f.write(f"FEWSHOT_SPLIT = {task.FEWSHOT_SPLIT}".strip() + "\n")
-        f.write(f"RESPONSE_TYPE = {eval_.response_type().name}".strip() + "\n")
-        metrics_list = [f"{m.__name__}" for m in eval_.metrics()]
-        f.write(f"METRICS = [{', '.join(metrics_list)}]".strip() + "\n")
-        if hasattr(task, "SUBJECTS"):
-            f.write(f"SUBJECTS = {repr(task.SUBJECTS)}".strip() + "\n")
-        if hasattr(task, "LANGUAGE"):
-            f.write(f"LANGUAGE = {repr(task.LANGUAGE)}".strip() + "\n")
-        f.write("````\n\n")
-
-        f.write(f"- Module: `{task_class.__module__}`\n\n")
+        task = task_class(num_fewshot=0)
 
-        try:
-            raw_file_path = inspect.getfile(task_class)
-            # Find the package root 'eval_framework' in the path
-            match = re.search(r"eval_framework.*", raw_file_path)
-            if match:
-                # Reconstruct relative path assuming standard 'src' structure
-                task_file = f"src/{match.group(0)}"
-                # Provide a local relative link (for VS Code) and an absolute link (for GitHub/Web)
-                f.write(f"- File: [{task_file}](../../{task_file}) | [View on GitHub]({REPO_URL}/{task_file})\n\n")
-            else:
-                # Fallback for tasks defined outside the main package (e.g., custom local tasks)
-                f.write(f"- File: `{raw_file_path}`\n\n")
-        except Exception:
-            f.write("- File: `Dynamic or Built-in`\n\n")
-
-        if http_path:
-            f.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
-
-        if not add_prompt_examples:
-            f.write(
-                f"More detailed documentation, with prompt examples and ground truth completions, can be generated "
-                f"with `uv run -m eval_framework.utils.generate_task_docs --add-prompt-examples "
-                f'--only-tasks "{task_name}"`.\n'
-            )
-
-        else:
-            s = next(iter(task.iterate_samples(1)))
-            for split in task.dataset:
-                f.write(f"- `{split}` has {len(task.dataset[split])} samples\n\n")
-
-            for formatter in formatters:
-                f.write(f"## Example prompt with {formatter.__class__.__name__} ({num_fewshot}-shot)\n\n")
-                formatted_sample = formatter.format(s.messages, output_mode="string")
-                f.write("````\n")
-                f.write(f'"{formatted_sample}"')
-                f.write("\n````\n\n")
-
-            f.write("## Possible completions:\n\n")
-            f.write("````\n")
-            if s.possible_completions:
-                for item in (
-                    s.possible_completions if isinstance(s.possible_completions, list) else [s.possible_completions]
-                ):
-                    f.write(f'- "{item}"\n')
-            else:
-                f.write("None\n")
-            f.write("````\n\n")
-
-            f.write("## Ground truth:\n\n")
-            f.write("````\n")
-            if s.ground_truth:
-                for item in s.ground_truth if isinstance(s.ground_truth, list) else [s.ground_truth]:
-                    f.write(f'- "{item}"\n')
-            else:
-                f.write("None\n")
-            f.write("````\n")
+    (output_docs_directory / f"{task_name}.md").write_text(task.markdown_doc(formatters), encoding="utf-8")
 
 
 def generate_readme_list(output_docs_directory: Path, total_tasks: int) -> None:
@@ -227,7 +133,6 @@ def generate_all_docs(args: argparse.Namespace, output_docs_directory: Path) ->
                 output_docs_directory=output_docs_directory,
                 task_name=task_name,
                 formatters=formatters,
-                add_prompt_examples=args.add_prompt_examples,
             )
 
         except Exception as e: