eval-framework 0.8.2__tar.gz → 0.8.4__tar.gz

{eval_framework-0.8.2 → eval_framework-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.8.2
+Version: 0.8.4
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -225,14 +225,14 @@ Requires-Dist: spacy>=3.8.14,<4
 Requires-Dist: jsonschema>=4.26.0,<5
 Requires-Dist: mysql-connector-python>=9.7.0,<10
 Requires-Dist: psycopg2-binary>=2.9.12,<3
-Requires-Dist: sympy>=1.13.1,<2
+Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.39
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
-Requires-Dist: python-iso639>=2026.4.20
+Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.28.1,<1
 Requires-Dist: boto3>=1.43.19,<2
-Requires-Dist: numpy>=2.2.6
+Requires-Dist: numpy>=2.5.1
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
@@ -240,7 +240,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.46.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=2.48.0,<3 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.8.2 → eval_framework-0.8.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.8.2"
+version = "0.8.4"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -32,14 +32,14 @@ dependencies = [
   "jsonschema>=4.26.0,<5",
   "mysql-connector-python>=9.7.0,<10", # required for sql-related tasks
   "psycopg2-binary>=2.9.12,<3", # required for sql-related tasks
-  "sympy>=1.13.1,<2",
+  "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.39",
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
-  "python-iso639>=2026.4.20",
+  "python-iso639>=2026.7.23",
   "wandb>=0.28.1,<1",
   "boto3>=1.43.19,<2",
-  "numpy>=2.2.6",
+  "numpy>=2.5.1",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.46.0,<3",
+  "openai>=2.48.0,<3",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -88,11 +88,11 @@ dev = [
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
-  "types-pyyaml>=6.0.12.20260518,<7",
+  "types-pyyaml>=6.0.12.20260724,<7",
   "types-python-dateutil>=2.9.0.20260716,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.15.22",
+  "ruff>=0.16.0",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -101,7 +101,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.29,<0.11.30"]
+requires = ["uv_build>=0.11.32,<0.11.33"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/metrics/completion/language_checker.py

@@ -41,9 +41,6 @@ class LanguageConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
-        if not response.completion:
-            return []  # No completion means no language to detect, so it is excluded from aggregation
-
         completion_language = response.get_completion_language()
         target_language = response.get_instruction_language()
         if completion_language == target_language == "":
@@ -60,9 +57,6 @@ class LanguageRawConsistencyChecker(BaseMetric[Completion]):
         if response.error is not None:
             return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
 
-        if not response.raw_completion:
-            return []  # No completion means no language to detect, so it is excluded from aggregation
-
         raw_completion_language = response.get_raw_completion_language()
         target_language = response.get_instruction_language()
 

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/tasks/base.py

@@ -3,7 +3,7 @@ import os
 import random
 import traceback
 from abc import ABC
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Self, TypeVar
@@ -14,8 +14,9 @@ from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
 from eval_framework.tasks.dataset_revisions import pinned_revision
+from eval_framework.tasks.markdown_doc import markdown_doc as render_markdown_doc
 from eval_framework.tasks.utils import classproperty, raise_errors
-from template_formatting.formatter import Message, Role
+from template_formatting.formatter import BaseFormatter, Message, Role
 
 if TYPE_CHECKING:
     from eval_framework.llm.base import BaseLLM
@@ -280,6 +281,35 @@ class BaseTask[SubjectType](ABC):
                         done = True
                         break
 
+    def markdown_doc(self, formatters: Sequence[BaseFormatter]) -> str:
+        """Render this task's documentation as markdown."""
+        dataset_path = getattr(self, "DATASET_PATH", None)
+        example_messages = split_sizes = possible_completions = ground_truth = None
+        if dataset_path is None:
+            sample = next(iter(self.iterate_samples(1)))
+            example_messages = sample.messages
+            split_sizes = {split: len(self.dataset[split]) for split in self.dataset}
+            possible_completions = sample.possible_completions
+            ground_truth = sample.ground_truth
+
+        return render_markdown_doc(
+            name=self.NAME,
+            module=type(self).__module__,
+            dataset_path=dataset_path,
+            sample_split=getattr(self, "SAMPLE_SPLIT", None),
+            fewshot_split=getattr(self, "FEWSHOT_SPLIT", None),
+            response_type=self.get_response_type().name,
+            metrics=[m.__name__ for m in self.get_metrics()],
+            subjects=getattr(self, "SUBJECTS", None),
+            language=getattr(self, "LANGUAGE", None),
+            num_fewshot=self.num_fewshot,
+            formatters=formatters,
+            example_messages=example_messages,
+            split_sizes=split_sizes,
+            possible_completions=possible_completions,
+            ground_truth=ground_truth,
+        )
+
     def _create_samples(self, item: dict[str, Any], index: int, subject: str) -> list[Sample]:
         """Creates one or more samples from a single dataset item. Default implementation returns single sample."""
         return [

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/tasks/benchmarks/squad.py

@@ -11,7 +11,7 @@ from huggingface_hub.errors import RevisionNotFoundError
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 from eval_framework.metrics.completion.f1 import F1, F1SquadNormalized
 from eval_framework.metrics.loglikelihood.bits_per_byte import BitsPerByteLoglikelihood
-from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, SubjectType
+from eval_framework.tasks.base import NO_SUBJECT, RANDOM_SEED, BaseTask, Language, ResponseType, Sample, SubjectType
 from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE
 
 
@@ -252,7 +252,7 @@ class SQuAD2_MA(SQUAD2):
     def __init__(self, num_fewshot: int = 0) -> None:
         super().__init__(num_fewshot)
         self.stop_sequences = []
-        self.max_tokens = None
+        self.max_tokens = 10_000
 
     def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
         return (
@@ -261,11 +261,46 @@ class SQuAD2_MA(SQUAD2):
             "Use the given context to answer the question faithfully. Answer only if the "
             f"answer is present in the given context, otherwise respond with '{self.UNANSWERABLE_STR}' "
             "if the answer is not present in the context."
+            "Always begin your answer with 'Final answer:'"
         )
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         return f"Context:\n{item['context']}\n\nQuestion:\n{item['question']}\n"
 
+    def post_process_generated_completion(self, completion_text: str, sample: Sample | None = None) -> str:
+        """Clean up the generated answer."""
+        # Remove common prefixes and clean whitespace
+        cleaned = completion_text.strip()
+        common_prefixes = ["Answer", "Final answer"]
+        # A list comprehension (not a generator) is required here: list.extend
+        # consumes lazily, so a generator over the list being extended never
+        # terminates and grows the list unboundedly until OOM.
+        common_prefixes.extend([f"**{prefix}**" for prefix in common_prefixes])
+        common_prefixes.reverse()
+
+        # Search for the last occurrence of any common prefix, and take only what's after it.
+        for prefix in common_prefixes:
+            idx = cleaned.rfind(prefix + ":")
+            if idx != -1:
+                cleaned = cleaned[idx + len(prefix) + 1 :].strip()
+                break
+        return cleaned
+
+    def _get_ground_truth(self, item: dict[str, Any]) -> list[str]:
+        text_ = item["answers"]["text"]
+        ground_truth_for_unanswerable = [
+            self.UNANSWERABLE_STR,
+            self.UNANSWERABLE_STR + " ",
+            self.UNANSWERABLE_STR.capitalize(),
+        ]
+        ground_truths = text_ if text_ else ground_truth_for_unanswerable
+        return ground_truths
+
+
+class SQuAD2_MA_NO_SYSPROMPT(SQuAD2_MA):
+    def _get_system_prompt_text(self, item: dict[str, Any]) -> str | None:
+        return ""
+
 
 class SQuAD_OLMES(SQUAD):
     """SQuAD variant matching OLMES implementation."""

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/tasks/benchmarks/triviaqa.py

@@ -60,7 +60,7 @@ class TriviaQA_MA(TRIVIAQA):
     def __init__(self, num_fewshot: int = 0) -> None:
         super().__init__(num_fewshot)
         self.stop_sequences = []
-        self.max_tokens = None
+        self.max_tokens = 27_000
 
     def _get_context_text(self, item: dict[str, Any]) -> str:
         return "\n\n".join(item["entity_pages"]["wiki_context"])

eval_framework-0.8.4/src/eval_framework/tasks/markdown_doc.py

@@ -0,0 +1,81 @@
+from collections.abc import Sequence
+from io import StringIO
+from typing import Any
+
+from template_formatting.formatter import BaseFormatter, Message
+
+
+def markdown_doc(
+    *,
+    name: str,
+    module: str,
+    dataset_path: str | None,
+    sample_split: str | None,
+    fewshot_split: str | None,
+    response_type: str,
+    metrics: Sequence[str],
+    subjects: Any,
+    language: Any,
+    num_fewshot: int,
+    formatters: Sequence[BaseFormatter],
+    example_messages: list[Message] | None,
+    split_sizes: dict[str, int] | None,
+    possible_completions: str | list[str] | None,
+    ground_truth: str | list[str] | None,
+) -> str:
+    """Render a task's documentation as markdown"""
+    buf = StringIO()
+    buf.write(f"# {name}\n\n")
+    http_path = f"https://huggingface.co/datasets/{dataset_path}" if dataset_path else None
+
+    buf.write("````\n")  # fence with 4 thicks because some prompts have code blocks with 3 thicks
+    buf.write(f"NAME = {name}".strip() + "\n")
+    if dataset_path is not None:
+        buf.write(f"DATASET_PATH = {dataset_path}".strip() + "\n")
+    if sample_split is not None:
+        buf.write(f"SAMPLE_SPLIT = {sample_split}".strip() + "\n")
+    if fewshot_split is not None:
+        buf.write(f"FEWSHOT_SPLIT = {fewshot_split}".strip() + "\n")
+    buf.write(f"RESPONSE_TYPE = {response_type}".strip() + "\n")
+    buf.write(f"METRICS = [{', '.join(metrics)}]".strip() + "\n")
+    if subjects is not None:
+        buf.write(f"SUBJECTS = {subjects!r}".strip() + "\n")
+    if language is not None:
+        buf.write(f"LANGUAGE = {language!r}".strip() + "\n")
+    buf.write("````\n\n")
+
+    buf.write(f"- Module: `{module}`\n\n")
+
+    if http_path:
+        buf.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
+    else:
+        assert example_messages is not None, "a task without a dataset link must supply an example sample"
+        for split, size in (split_sizes or {}).items():
+            buf.write(f"- `{split}` has {size} samples\n\n")
+
+        for formatter in formatters:
+            buf.write(f"## Example prompt with {formatter.__class__.__name__} ({num_fewshot}-shot)\n\n")
+            formatted_sample = formatter.format(example_messages, output_mode="string")
+            buf.write("````\n")
+            buf.write(f'"{formatted_sample}"')
+            buf.write("\n````\n\n")
+
+        buf.write("## Possible completions:\n\n")
+        buf.write("````\n")
+        if possible_completions:
+            for item in possible_completions if isinstance(possible_completions, list) else [possible_completions]:
+                buf.write(f'- "{item}"\n')
+        else:
+            buf.write("None\n")
+        buf.write("````\n\n")
+
+        buf.write("## Ground truth:\n\n")
+        buf.write("````\n")
+        if ground_truth:
+            for item in ground_truth if isinstance(ground_truth, list) else [ground_truth]:
+                buf.write(f'- "{item}"\n')
+        else:
+            buf.write("None\n")
+        buf.write("````\n")
+
+    return buf.getvalue()

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/tasks/task_names.py

@@ -70,6 +70,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.sciq.SCIQ_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA")
+    register_lazy_task("eval_framework.tasks.benchmarks.squad.SQuAD2_MA_NO_SYSPROMPT")
     register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TRIVIAQA")
     register_lazy_task("eval_framework.tasks.benchmarks.triviaqa.TriviaQA_MA")
     register_lazy_task("eval_framework.tasks.benchmarks.winogrande.WINOGRANDECloze")

{eval_framework-0.8.2 → eval_framework-0.8.4}/src/eval_framework/utils/generate_task_docs.py

@@ -1,7 +1,5 @@
 import argparse
-import inspect
 import os
-import re
 from pathlib import Path
 
 import tqdm
@@ -27,7 +25,7 @@ def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
         action="store_true",
         default=False,
         required=False,
-        help="If set, examples prompts for each of the formatters will be added in the generated docs.",
+        help="Unused. Only there for backwards compatibility",
     )
     parser.add_argument(
         "--exclude-tasks",
@@ -65,108 +63,16 @@ def parse_args(cli_args: list[str] | None = None) -> argparse.Namespace:
     return parser.parse_args(args=cli_args)
 
 
-def generate_docs_for_task(
-    output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter], add_prompt_examples: bool
-) -> None:
+def generate_docs_for_task(output_docs_directory: Path, task_name: str, formatters: list[BaseFormatter]) -> None:
     """Generate documentation for a specific task."""
-    eval_ = registry()[task_name]
-    task_class = eval_.task_class()
+    task_class = registry()[task_name].task_class()
 
     try:
-        num_fewshot = 1
-        task = task_class(num_fewshot=num_fewshot)
+        task = task_class(num_fewshot=1)
     except (TypeError, ValueError, AssertionError):
-        try:
-            num_fewshot = 0
-            task = task_class(num_fewshot=num_fewshot)
-        except (TypeError, ValueError, AssertionError):
-            try:
-                task = task_class()
-                num_fewshot = task.num_fewshot
-            except Exception as e:
-                print(f"Failed to instantiate task {task_name}: {e}")
-                return
-
-    with open(f"{output_docs_directory}/{task_name}.md", "w") as f:
-        f.write(f"# {task_name}\n\n")
-        dataset_path = eval_.dataset_path()
-        http_path = f"https://huggingface.co/datasets/{dataset_path}" if dataset_path else None
-
-        f.write("````\n")  # fence with 4 thicks because some prompts have code blocks with 3 thicks
-        f.write(f"NAME = {task_name}".strip() + "\n")
-        if dataset_path is not None:
-            f.write(f"DATASET_PATH = {dataset_path}".strip() + "\n")
-        if hasattr(task, "SAMPLE_SPLIT"):
-            f.write(f"SAMPLE_SPLIT = {task.SAMPLE_SPLIT}".strip() + "\n")
-        if hasattr(task, "FEWSHOT_SPLIT"):
-            f.write(f"FEWSHOT_SPLIT = {task.FEWSHOT_SPLIT}".strip() + "\n")
-        f.write(f"RESPONSE_TYPE = {eval_.response_type().name}".strip() + "\n")
-        metrics_list = [f"{m.__name__}" for m in eval_.metrics()]
-        f.write(f"METRICS = [{', '.join(metrics_list)}]".strip() + "\n")
-        if hasattr(task, "SUBJECTS"):
-            f.write(f"SUBJECTS = {repr(task.SUBJECTS)}".strip() + "\n")
-        if hasattr(task, "LANGUAGE"):
-            f.write(f"LANGUAGE = {repr(task.LANGUAGE)}".strip() + "\n")
-        f.write("````\n\n")
-
-        f.write(f"- Module: `{task_class.__module__}`\n\n")
+        task = task_class(num_fewshot=0)
 
-        try:
-            raw_file_path = inspect.getfile(task_class)
-            # Find the package root 'eval_framework' in the path
-            match = re.search(r"eval_framework.*", raw_file_path)
-            if match:
-                # Reconstruct relative path assuming standard 'src' structure
-                task_file = f"src/{match.group(0)}"
-                # Provide a local relative link (for VS Code) and an absolute link (for GitHub/Web)
-                f.write(f"- File: [{task_file}](../../{task_file}) | [View on GitHub]({REPO_URL}/{task_file})\n\n")
-            else:
-                # Fallback for tasks defined outside the main package (e.g., custom local tasks)
-                f.write(f"- File: `{raw_file_path}`\n\n")
-        except Exception:
-            f.write("- File: `Dynamic or Built-in`\n\n")
-
-        if http_path:
-            f.write(f"- Link to dataset: [{http_path}]({http_path})\n\n")
-
-        if not add_prompt_examples:
-            f.write(
-                f"More detailed documentation, with prompt examples and ground truth completions, can be generated "
-                f"with `uv run -m eval_framework.utils.generate_task_docs --add-prompt-examples "
-                f'--only-tasks "{task_name}"`.\n'
-            )
-
-        else:
-            s = next(iter(task.iterate_samples(1)))
-            for split in task.dataset:
-                f.write(f"- `{split}` has {len(task.dataset[split])} samples\n\n")
-
-            for formatter in formatters:
-                f.write(f"## Example prompt with {formatter.__class__.__name__} ({num_fewshot}-shot)\n\n")
-                formatted_sample = formatter.format(s.messages, output_mode="string")
-                f.write("````\n")
-                f.write(f'"{formatted_sample}"')
-                f.write("\n````\n\n")
-
-            f.write("## Possible completions:\n\n")
-            f.write("````\n")
-            if s.possible_completions:
-                for item in (
-                    s.possible_completions if isinstance(s.possible_completions, list) else [s.possible_completions]
-                ):
-                    f.write(f'- "{item}"\n')
-            else:
-                f.write("None\n")
-            f.write("````\n\n")
-
-            f.write("## Ground truth:\n\n")
-            f.write("````\n")
-            if s.ground_truth:
-                for item in s.ground_truth if isinstance(s.ground_truth, list) else [s.ground_truth]:
-                    f.write(f'- "{item}"\n')
-            else:
-                f.write("None\n")
-            f.write("````\n")
+    (output_docs_directory / f"{task_name}.md").write_text(task.markdown_doc(formatters), encoding="utf-8")
 
 
 def generate_readme_list(output_docs_directory: Path, total_tasks: int) -> None:
@@ -227,7 +133,6 @@ def generate_all_docs(args: argparse.Namespace, output_docs_directory: Path) ->
                 output_docs_directory=output_docs_directory,
                 task_name=task_name,
                 formatters=formatters,
-                add_prompt_examples=args.add_prompt_examples,
             )
 
         except Exception as e: