eval-framework 0.10.7__tar.gz → 0.11.3__tar.gz

{eval_framework-0.10.7 → eval_framework-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.10.7
+Version: 0.11.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -229,20 +229,20 @@ Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
-Requires-Dist: lxml>=6.1.1,<7
+Requires-Dist: lxml>=6.1.2,<7
 Requires-Dist: python-iso639>=2026.7.23
-Requires-Dist: wandb>=0.28.2,<1
-Requires-Dist: boto3>=1.43.72,<2
+Requires-Dist: wandb>=0.29.0,<1
+Requires-Dist: boto3>=1.43.82,<2
 Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
-Requires-Dist: scipy>=1.18.0,<2
+Requires-Dist: scipy>=1.18.1,<2
 Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,optional] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=3.1.0,<4 ; extra == 'openai'
-Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
+Requires-Dist: openai>=3.5.0,<4 ; extra == 'openai'
+Requires-Dist: tiktoken>=0.14.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'

{eval_framework-0.10.7 → eval_framework-0.11.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.7"
+version = "0.11.3"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -32,13 +32,13 @@ dependencies = [
     "sympy>=1.14.0,<2",
     "llm-sandbox[docker]==0.3.44",
     "jsonlines>=4,<5",
-    "lxml>=6.1.1,<7",
+    "lxml>=6.1.2,<7",
     "python-iso639>=2026.7.23",
-    "wandb>=0.28.2,<1",
-    "boto3>=1.43.72,<2",
+    "wandb>=0.29.0,<1",
+    "boto3>=1.43.82,<2",
     "numpy>=2.5.2",
     "antlr4-python3-runtime==4.11.0",
-    "scipy>=1.18.0,<2",
+    "scipy>=1.18.1,<2",
 ]
 
 [project.license]
@@ -54,8 +54,8 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-    "openai>=3.1.0,<4",
-    "tiktoken>=0.13.0,<1",
+    "openai>=3.5.0,<4",
+    "tiktoken>=0.14.0,<1",
     "transformers>=4.45.2,<5",
 ]
 transformers = [
@@ -86,8 +86,8 @@ dev = [
     "types-pyyaml>=6.0.12.20260815,<7",
     "types-python-dateutil>=2.9.0.20260807,<3",
     "types-requests>=2.33.0.20260712,<3",
-    "plotly>=6.9.0,<7",
-    "ruff>=0.16.3",
+    "plotly>=7.0.0,<8",
+    "ruff>=0.16.5",
     "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -96,7 +96,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.5,<0.12.6"]
+requires = ["uv_build>=0.12.7,<0.12.8"]
 build-backend = "uv_build"
 
 [tool.uv]

{eval_framework-0.10.7 → eval_framework-0.11.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.7"
+version = "0.11.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -36,15 +36,15 @@ dependencies = [
   "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
-  "lxml>=6.1.1,<7",
+  "lxml>=6.1.2,<7",
   "python-iso639>=2026.7.23",
-  "wandb>=0.28.2,<1",
-  "boto3>=1.43.72,<2",
+  "wandb>=0.29.0,<1",
+  "boto3>=1.43.82,<2",
   "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
   "antlr4-python3-runtime==4.11.0",
-  "scipy>=1.18.0,<2",  # required for the aggregation of pass@k metrics
+  "scipy>=1.18.1,<2",  # required for the aggregation of pass@k metrics
 ]
 
 [project.optional-dependencies]
@@ -55,8 +55,8 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=3.1.0,<4",
-  "tiktoken>=0.13.0,<1",
+  "openai>=3.5.0,<4",
+  "tiktoken>=0.14.0,<1",
   "transformers>=4.45.2,<5",
 ]
 transformers = [
@@ -92,8 +92,8 @@ dev = [
   "types-pyyaml>=6.0.12.20260815,<7",
   "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
-  "plotly>=6.9.0,<7",
-  "ruff>=0.16.3",
+  "plotly>=7.0.0,<8",
+  "ruff>=0.16.5",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.5,<0.12.6"]
+requires = ["uv_build>=0.12.7,<0.12.8"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

eval_framework-0.11.3/src/eval_framework/benchmarks/arc_de.py

@@ -0,0 +1,43 @@
+"""ARC German (ARC-DE)."""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import NoSubject
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import ClozeStyle, answer_key_to_index
+
+
+@final
+class ArcDeReader(ChoiceReader):
+    """Reads an ARC-DE row into choice fields: the German question, its answer texts, and the correct index.
+
+    ``answerKey`` arrives as either a 1-based number or a letter; ``answer_key_to_index`` normalises both,
+    which also frees the reader from caring how many answers a given row offers.
+    """
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        return ChoiceFields(
+            raw_question=item["question_de"],
+            choices=item["choices_de"]["text"],
+            correct_index=answer_key_to_index(item["answerKey"]),
+        )
+
+
+# ARC-DE as cloze/ranked classification.
+# https://huggingface.co/datasets/LeoLM/ArcChallenge_de
+ARC_DE_BENCHMARK: Benchmark = ComposedBenchmark.compose(
+    id="ARC_DE",
+    display_name="ARC German",
+    styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"),
+    reader=ArcDeReader(),
+    sample_split="test",
+    fewshot_split="validation",
+    subjects=NoSubject(),
+    dataset_policy=pinned_by_framework("LeoLM/ArcChallenge_de"),
+    language=Language.DEU,
+)

eval_framework-0.11.3/src/eval_framework/benchmarks/csqa_ellamind.py

@@ -0,0 +1,63 @@
+"""German CommonsenseQA (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/csqa-multilingual
+
+CSQA supplies separate easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class CsqaReader(ChoiceReader):
+    """Reads a CSQA item: easy/hard distractor lists per level, shuffled in with the correct answer."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=distractors,
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
+
+
+# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to
+# the reader, so it is orthogonal to the styler choice.
+CSQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU)
+CSQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU)
+CSQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU)
+
+
+def _csqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=CsqaReader(distractor_level),
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=pinned_by_framework("ellamind/csqa-multilingual"),
+        language=Language.DEU,
+    )
+
+
+CSQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", CSQA_ELLAMIND_MC_STYLER, "easy"),
+    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", CSQA_ELLAMIND_MC_STYLER, "hard"),
+    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_EASY_DE", CSQA_ELLAMIND_CLOZE_STYLER, "easy"),
+    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_HARD_DE", CSQA_ELLAMIND_CLOZE_STYLER, "hard"),
+    _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", CSQA_ELLAMIND_BPB_STYLER, "easy"),
+]

eval_framework-0.11.3/src/eval_framework/benchmarks/piqa_ellamind.py

@@ -0,0 +1,66 @@
+"""German PIQA (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/piqa-multilingual
+
+PIQA supplies separate easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class PiqaReader(ChoiceReader):
+    """Reads a PIQA item: a single easy/hard distractor per level, shuffled in with the correct solution."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractor = item["easy_distractor"] if self._distractor_level == "easy" else item["hard_distractor"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_solution"],
+            distractors=[distractor],
+            seed_text=item["goal"] + item["correct_solution"],
+        )
+        return ChoiceFields(raw_question=item["goal"], choices=choices, correct_index=correct_index)
+
+
+_QUESTION_PREFIX = "Ziel: "
+_CUE_TEXT = "Antwort:"
+
+# One styler per format (all sharing the German prefix/cue). The easy/hard distractor axis belongs to
+# the reader, so it is orthogonal to the styler choice.
+PIQA_ELLAMIND_CLOZE_STYLER = ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT)
+PIQA_ELLAMIND_MC_STYLER = MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT)
+PIQA_ELLAMIND_BPB_STYLER = BPBStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT)
+
+
+def _piqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=PiqaReader(distractor_level),
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=pinned_by_framework("ellamind/piqa-multilingual"),
+        language=Language.DEU,
+    )
+
+
+PIQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    _piqa_ellamind_benchmark("PIQA_ELLAMIND_CLOZE_EASY_DE", PIQA_ELLAMIND_CLOZE_STYLER, "easy"),
+    _piqa_ellamind_benchmark("PIQA_ELLAMIND_CLOZE_HARD_DE", PIQA_ELLAMIND_CLOZE_STYLER, "hard"),
+    _piqa_ellamind_benchmark("PIQA_ELLAMIND_MC_EASY_DE", PIQA_ELLAMIND_MC_STYLER, "easy"),
+    _piqa_ellamind_benchmark("PIQA_ELLAMIND_MC_HARD_DE", PIQA_ELLAMIND_MC_STYLER, "hard"),
+    _piqa_ellamind_benchmark("PIQA_ELLAMIND_BPB_DE", PIQA_ELLAMIND_BPB_STYLER, "easy"),
+]

eval_framework-0.11.3/src/eval_framework/benchmarks/siqa_ellamind.py

@@ -0,0 +1,66 @@
+"""German Social IQa (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/siqa-multilingual
+
+SIQA supplies separate easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class SiqaReader(ChoiceReader):
+    """Reads a Social IQa item: the shown question is the context followed by the question, and the
+    easy/hard distractor list for the level is shuffled in with the correct answer."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=distractors,
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(
+            raw_question=f"{item['context']} {item['question']}", choices=choices, correct_index=correct_index
+        )
+
+
+# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to
+# the reader, so it is orthogonal to the styler choice.
+SIQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU)
+SIQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU)
+SIQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU)
+
+
+def _siqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=SiqaReader(distractor_level),
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=pinned_by_framework("ellamind/siqa-multilingual"),
+        language=Language.DEU,
+    )
+
+
+SIQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_EASY_DE", SIQA_ELLAMIND_MC_STYLER, "easy"),
+    _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_HARD_DE", SIQA_ELLAMIND_MC_STYLER, "hard"),
+    _siqa_ellamind_benchmark("SIQA_ELLAMIND_CLOZE_EASY_DE", SIQA_ELLAMIND_CLOZE_STYLER, "easy"),
+    _siqa_ellamind_benchmark("SIQA_ELLAMIND_CLOZE_HARD_DE", SIQA_ELLAMIND_CLOZE_STYLER, "hard"),
+    _siqa_ellamind_benchmark("SIQA_ELLAMIND_BPB_DE", SIQA_ELLAMIND_BPB_STYLER, "easy"),
+]

eval_framework-0.11.3/src/eval_framework/choices.py

@@ -0,0 +1,31 @@
+"""Choice readers: extracting the fields a choice-based styler needs out of a raw dataset item,
+so neither the eval nor the styler has to know a benchmark's item schema."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ChoiceFields:
+    """The fields a choice-based styler needs out of a single dataset item.
+
+    ``choices`` and ``correct_index`` are produced together (a benchmark may shuffle the correct
+    answer in among distractors), so a reader yields them in one ``read`` rather than via separate
+    calls that would each have to re-derive the same shuffle.
+    """
+
+    raw_question: str
+    choices: list[str]
+    correct_index: int
+
+
+class ChoiceReader(ABC):
+    """Reads the fields a styler needs out of a raw dataset item.
+
+    Isolates dataset-schema knowledge here, so neither the eval nor the styler has to know the shape
+    of a benchmark's items.
+    """
+
+    @abstractmethod
+    def read(self, item: dict[str, Any]) -> ChoiceFields: ...