eval-framework 0.13.2__tar.gz → 0.13.3__tar.gz

{eval_framework-0.13.2 → eval_framework-0.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.13.2
+Version: 0.13.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -232,7 +232,7 @@ Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.3,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.30.0,<1
-Requires-Dist: boto3>=1.43.94,<2
+Requires-Dist: boto3>=1.43.95,<2
 Requires-Dist: numpy>=2.5.3
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.1,<2
@@ -241,7 +241,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=3.14.0,<4 ; extra == 'openai'
+Requires-Dist: openai>=3.14.1,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.14.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'

{eval_framework-0.13.2 → eval_framework-0.13.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.13.2"
+version = "0.13.3"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -35,7 +35,7 @@ dependencies = [
     "lxml>=6.1.3,<7",
     "python-iso639>=2026.7.23",
     "wandb>=0.30.0,<1",
-    "boto3>=1.43.94,<2",
+    "boto3>=1.43.95,<2",
     "numpy>=2.5.3",
     "antlr4-python3-runtime==4.11.0",
     "scipy>=1.18.1,<2",
@@ -54,7 +54,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-    "openai>=3.14.0,<4",
+    "openai>=3.14.1,<4",
     "tiktoken>=0.14.0,<1",
     "transformers>=4.45.2,<5",
 ]
@@ -86,7 +86,7 @@ dev = [
     "types-pyyaml>=6.0.12.20260906,<7",
     "types-python-dateutil>=2.9.0.20260807,<3",
     "types-requests>=2.33.0.20260906,<3",
-    "plotly>=7.0.0,<8",
+    "plotly>=7.1.0,<8",
     "ruff>=0.16.7",
     "pip-licenses>=5.5.5",
 ]
@@ -96,7 +96,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.13,<0.12.14"]
+requires = ["uv_build>=0.12.15,<0.12.16"]
 build-backend = "uv_build"
 
 [tool.uv]

{eval_framework-0.13.2 → eval_framework-0.13.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.13.2"
+version = "0.13.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -39,7 +39,7 @@ dependencies = [
   "lxml>=6.1.3,<7",
   "python-iso639>=2026.7.23",
   "wandb>=0.30.0,<1",
-  "boto3>=1.43.94,<2",
+  "boto3>=1.43.95,<2",
   "numpy>=2.5.3",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,7 +55,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=3.14.0,<4",
+  "openai>=3.14.1,<4",
   "tiktoken>=0.14.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -92,7 +92,7 @@ dev = [
   "types-pyyaml>=6.0.12.20260906,<7",
   "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260906,<3",
-  "plotly>=7.0.0,<8",
+  "plotly>=7.1.0,<8",
   "ruff>=0.16.7",
   "pip-licenses>=5.5.5",
 ]
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.13,<0.12.14"]
+requires = ["uv_build>=0.12.15,<0.12.16"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

eval_framework-0.13.3/src/eval_framework/answer.py

@@ -0,0 +1,143 @@
+"""How a benchmark obtains the model's scored answer.
+
+An ``AnswerPolicy`` owns the "answer side" of a composed benchmark — the mode the model answers in
+(loglikelihood over candidates vs. free-form completion), the bounds on any generation (stop sequences,
+token limit), and how the raw generation is distilled into the answer that metrics score. It is injected
+into ``compose`` next to the eval kind, so the kind stays purely about the prompt and candidates.
+"""
+
+import re
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, final, override
+
+from eval_framework.contract import ResponseType
+from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
+    BytesCompletion,
+    BytesLoglikelihood,
+    SequencePositionsCompletion,
+    SequencePositionsLoglikelihood,
+)
+from eval_framework.metrics.efficiency.finish_reason import FinishReason
+from eval_framework.metrics.efficiency.token_counters import TokenCounts
+from eval_framework.shared.types import BaseMetricContext
+from template_formatting.formatter import Message
+
+if TYPE_CHECKING:
+    from eval_framework.metrics.base import BaseMetric
+
+
+class AnswerPolicy(ABC):
+    """The answer side of a kind: the response type, the generation bounds, and answer extraction."""
+
+    @abstractmethod
+    def response_type(self) -> ResponseType:
+        """Whether the model is scored by loglikelihood over candidates or by free-form completion."""
+
+    @abstractmethod
+    def metrics(self) -> list[type["BaseMetric"]]:
+        """The bookkeeping metrics this answer mode always reports (efficiency / token counts), added to the
+        kind's scoring metrics."""
+
+    @abstractmethod
+    def stop_sequences(self) -> list[str]:
+        """Stop sequences for completion generation (empty when nothing is generated)."""
+
+    @abstractmethod
+    def max_tokens(self) -> int | None:
+        """Token limit for completion generation, or None for no limit."""
+
+    @abstractmethod
+    def extract_answer(
+        self,
+        completion_text: str,
+        *,
+        context: BaseMetricContext | list[BaseMetricContext] | None,
+        ground_truth: str | list[str] | None,
+        messages: list[Message],
+    ) -> str:
+        """The answer to score, distilled from the raw generation."""
+
+
+@final
+class PickFromCandidates(AnswerPolicy):
+    """Loglikelihood scoring: the model is scored over fixed candidate completions and the answer is the
+    best-scoring candidate, taken verbatim — nothing is generated, so nothing is bounded or extracted."""
+
+    @override
+    def response_type(self) -> ResponseType:
+        return ResponseType.LOGLIKELIHOODS
+
+    @override
+    def metrics(self) -> list[type["BaseMetric"]]:
+        return [BytesLoglikelihood, SequencePositionsLoglikelihood]
+
+    @override
+    def stop_sequences(self) -> list[str]:
+        return []
+
+    @override
+    def max_tokens(self) -> int | None:
+        return None
+
+    @override
+    def extract_answer(
+        self,
+        completion_text: str,
+        *,
+        context: BaseMetricContext | list[BaseMetricContext] | None,
+        ground_truth: str | list[str] | None,
+        messages: list[Message],
+    ) -> str:
+        return completion_text
+
+
+@final
+class ExtractFromCompletion(AnswerPolicy):
+    """Free-form completion: the model generates (bounded by ``stop_sequences`` / ``max_tokens``) and the
+    scored answer is pulled out with ``answer_re``. ``last_match`` takes the final match, upper-cased (for
+    lenient case-insensitive patterns); otherwise the first match is returned as-is. ``"[invalid]"`` when
+    nothing matches."""
+
+    def __init__(
+        self,
+        answer_re: re.Pattern[str],
+        stop_sequences: list[str] | None = None,
+        *,
+        last_match: bool = False,
+        max_tokens: int | None = None,
+    ) -> None:
+        self._answer_re = answer_re
+        self._stop_sequences = stop_sequences or []
+        self._last_match = last_match
+        self._max_tokens = max_tokens
+
+    @override
+    def response_type(self) -> ResponseType:
+        return ResponseType.COMPLETION
+
+    @override
+    def metrics(self) -> list[type["BaseMetric"]]:
+        return [BytesCompletion, SequencePositionsCompletion, TokenCounts, FinishReason]
+
+    @override
+    def stop_sequences(self) -> list[str]:
+        return self._stop_sequences
+
+    @override
+    def max_tokens(self) -> int | None:
+        return self._max_tokens
+
+    @override
+    def extract_answer(
+        self,
+        completion_text: str,
+        *,
+        context: BaseMetricContext | list[BaseMetricContext] | None,
+        ground_truth: str | list[str] | None,
+        messages: list[Message],
+    ) -> str:
+        if self._last_match:
+            matches = self._answer_re.findall(completion_text)
+            return matches[-1].upper() if matches else "[invalid]"
+        match = self._answer_re.search(completion_text)
+        return match.group(1) if match else "[invalid]"

eval_framework-0.13.3/src/eval_framework/benchmarks/cot.py

@@ -0,0 +1,101 @@
+"""Shared chain-of-thought scaffolding for multiple-choice benchmarks.
+
+A CoT variant asks the model to reason freely and conclude with a stated answer letter, which is pulled back
+out at scoring time (the injected ``ExtractFromCompletion``). The prompt surface — an optional preamble, the body,
+and any inert scored candidates — is injected per benchmark; the CoT contract is fixed here: one free-form
+sample, no assistant cue, the bare answer letter as ground truth, scored by accuracy.
+"""
+
+import re
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, final, override
+
+from eval_framework.answer import ExtractFromCompletion
+from eval_framework.choices import ChoiceReader
+from eval_framework.eval_kind import EvalKind, SampleBody
+from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
+from eval_framework.tasks.task_style import InitialPrompt
+from eval_framework.tasks.utils import get_n_letters
+
+if TYPE_CHECKING:
+    from eval_framework.metrics.base import BaseMetric
+
+# (raw question, in-order choices) -> the prompt body shown to the model.
+CotPrompt = Callable[[str, list[str]], str]
+# (this item's answer-letter keys) -> inert scored candidates carried on the sample (usually none).
+CotCandidates = Callable[[list[str]], list[str]]
+
+
+def tulu3_cot_prompt(raw_question: str, choices: list[str]) -> str:
+    """The parenthesised-option CoT body shared by MMLU-Pro and GPQA.
+
+    Reasoning prompt from Figure 44 of the Tülu 3 paper: https://arxiv.org/pdf/2411.15124
+    """
+    keys = get_n_letters(len(choices))
+    options = "\n".join(f"({key}) {choice}" for key, choice in zip(keys, choices))
+    return (
+        "Answer the following multiple-choice question by giving the correct answer letter in parentheses. "
+        "Provide CONCISE reasoning for the answer, and make sure to finish the response with "
+        '"Therefore, the answer is (ANSWER_LETTER)" where (ANSWER_LETTER) is one of (A), (B), (C), (D), (E), etc.'
+        f"\n\nQuestion: {raw_question}\n{options}"
+        "\n\nAnswer the above question and REMEMBER to finish your response with the exact phrase "
+        '"Therefore, the answer is (ANSWER_LETTER)" where (ANSWER_LETTER) is one of (A), (B), (C), (D), (E), etc.'
+    )
+
+
+def tulu_answer() -> ExtractFromCompletion:
+    """Extracts the parenthesised letter that ``tulu3_cot_prompt`` asks the model to conclude with —
+    ``"Therefore, the answer is (X)"``. Kept here beside the prompt because both encode the same ``(X)``
+    format. The accepted letters are the fixed A–J the prompt's ``"(A), (B), ..., (E), etc."`` implies."""
+    return ExtractFromCompletion(re.compile(r"Therefore, the answer is \(([ABCDEFGHIJ])\)"), ["Question:"])
+
+
+def tulu_answer_v2(n_options: int) -> ExtractFromCompletion:
+    """Lenient variant of ``tulu_answer``: no required ``"Therefore,"``, optional colon and parentheses,
+    case-insensitive, taking the last match. Only the accepted letter range is benchmark-specific, so it is
+    built from ``n_options``."""
+    letters = "".join(get_n_letters(n_options))
+    return ExtractFromCompletion(
+        re.compile(rf"\banswer\s+is:?\s*\(?([{letters}])\b\)?", re.IGNORECASE), last_match=True
+    )
+
+
+@final
+class Cot(EvalKind):
+    """Multiple-choice chain-of-thought (see the module docstring). ``build_prompt`` renders the body,
+    ``preamble`` an optional subject-templated top line, and ``candidates`` any inert scored letters kept for
+    faithful parity with a loglikelihood baseline (free-form scoring ignores them)."""
+
+    def __init__(
+        self,
+        reader: ChoiceReader,
+        *,
+        build_prompt: CotPrompt,
+        preamble: InitialPrompt | None = None,
+        candidates: CotCandidates | None = None,
+    ) -> None:
+        self._reader = reader
+        self._build_prompt = build_prompt
+        self._preamble = preamble
+        self._candidates = candidates
+
+    @override
+    def metrics(self) -> list[type["BaseMetric"]]:
+        return [AccuracyCompletion]
+
+    @override
+    def initial_prompt(self, subject_label: str) -> str | None:
+        return self._preamble(subject_label) if self._preamble is not None else None
+
+    @override
+    def samples(self, item: dict[str, Any]) -> list[SampleBody]:
+        fields = self._reader.read(item)
+        keys = get_n_letters(len(fields.choices))
+        return [
+            SampleBody(
+                prompt=self._build_prompt(fields.raw_question, fields.choices),
+                cue="",  # no assistant cue — the model continues into its own reasoning
+                possible_completions=self._candidates(keys) if self._candidates is not None else [],
+                ground_truth=keys[fields.correct_index],  # the bare answer letter
+            )
+        ]

eval_framework-0.13.3/src/eval_framework/benchmarks/gpqa.py

@@ -0,0 +1,137 @@
+"""GPQA (Graduate-level Google-Proof Q&A): https://huggingface.co/datasets/Idavidrein/gpqa
+
+Gated, expert-written multiple-choice science questions. Each item has one ``Correct Answer`` and three
+``Incorrect Answer N`` distractors; the reader shuffles them together, seeded from the option texts so an
+item's option order is stable across runs. The registered variants:
+
+- ``GPQA_OLMES``: OLMES-style loglikelihood over the full ``gpqa_extended`` config.
+- ``GPQA_DIAMOND_COT`` / ``_V2``: chain-of-thought completion over the harder ``gpqa_diamond`` config; they
+  share one prompt and differ only in how the concluding answer letter is extracted.
+
+One question carries a raw sequence far too long for the prompt budget and is dropped from every config.
+"""
+
+import hashlib
+import random
+import re
+from typing import Any, final, override
+
+from eval_framework.answer import AnswerPolicy
+from eval_framework.benchmarks.cot import Cot, tulu3_cot_prompt, tulu_answer, tulu_answer_v2
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.fewshot import NoFewShot
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import MCStyle
+
+GPQA_DATASET_PATH = "Idavidrein/gpqa"
+
+# One question embeds a raw DNA/protein sequence long enough to blow the prompt budget; it is excluded.
+_OVERLONG_QUESTION = (
+    "Hello, you are embarking on a new project. You need to produce the HP1alpha protein in E. coli. "
+    "Which of these plasmids will you choose?"
+)
+
+
+def _gpqa_preamble(subject_label: str) -> str:
+    # GPQA shows a fixed expert-framing preamble; the subject (a config name) is not part of it.
+    return (
+        "Here are some example questions from experts. An explanation is given before the final answer. "
+        "Answer the final question yourself, giving your reasoning beforehand."
+    )
+
+
+def _preprocess(text: str | None) -> str:
+    """Clean an answer option: GPQA options carry stray ``[title]`` / ``[...]`` markup and doubled spaces."""
+    if text is None:
+        return " "
+    text = text.strip()
+    text = text.replace(" [title]", ". ")
+    text = re.sub(r"\[.*?\]", "", text)
+    text = text.replace("  ", " ")
+    return text
+
+
+@final
+class GpqaReader(ChoiceReader):
+    """Reads a GPQA item: three ``Incorrect Answer N`` distractors and the ``Correct Answer``, shuffled
+    together. The shuffle is seeded from the (preprocessed) option texts, so the same item always yields the
+    same option order regardless of the order in which items are evaluated."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        choices = [_preprocess(item[f"Incorrect Answer {i}"]) for i in range(1, 4)]
+        correct = _preprocess(item["Correct Answer"])
+        seed = int(hashlib.sha256(f"{choices} {correct}".encode()).hexdigest(), 16)
+        rng = random.Random(seed)
+        rng.shuffle(choices)
+        correct_index = rng.randint(0, 3)
+        choices.insert(correct_index, correct)
+        return ChoiceFields(raw_question=item["Question"].strip(), choices=choices, correct_index=correct_index)
+
+
+def _gpqa_cot_candidates(keys: list[str]) -> list[str]:
+    # COT is free-form, so these letter labels are not scored; they are carried alongside the sample to keep
+    # it identical to the loglikelihood-shaped baseline (and thus its formatter hash).
+    return [f" ({key})" for key in keys]
+
+
+def _gpqa_dataset(dataset: DatasetPolicy | None) -> DatasetPolicy:
+    return (
+        dataset
+        if dataset is not None
+        else pinned_by_framework(GPQA_DATASET_PATH).subset(
+            lambda row: row["Question"] != _OVERLONG_QUESTION, description="excluding one over-long question"
+        )
+    )
+
+
+def gpqa_olmes(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = MCStyle(
+        question_prefix="Question: ",
+        cue_text="Answer:",
+        space_prefixed_labels=True,
+        initial_prompt=_gpqa_preamble,
+    )
+    return ComposedBenchmark.choice(
+        id="GPQA_OLMES",
+        reader=GpqaReader(),
+        styler=styler,
+        sample_split="train",
+        fewshot_split="train",
+        subjects=ListOfSubjects(["gpqa_extended"]),
+        dataset_policy=_gpqa_dataset(dataset),
+        language=Language.ENG,
+    )
+
+
+def _gpqa_diamond_cot(id: str, answer: AnswerPolicy, dataset: DatasetPolicy | None = None) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        kind=Cot(GpqaReader(), build_prompt=tulu3_cot_prompt, candidates=_gpqa_cot_candidates),
+        answer=answer,
+        sample_split="train",
+        fewshot=NoFewShot(),
+        subjects=ListOfSubjects(["gpqa_diamond"]),
+        dataset_policy=_gpqa_dataset(dataset),
+        language=Language.ENG,
+    )
+
+
+def gpqa_diamond_cot(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_diamond_cot("GPQA_DIAMOND_COT", tulu_answer(), dataset)
+
+
+def gpqa_diamond_cot_v2(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_diamond_cot("GPQA_DIAMOND_COT_V2", tulu_answer_v2(4), dataset)
+
+
+GPQA_BENCHMARKS: list[Benchmark] = [
+    gpqa_olmes(),
+    gpqa_diamond_cot(),
+    gpqa_diamond_cot_v2(),
+]

{eval_framework-0.13.2 → eval_framework-0.13.3}/src/eval_framework/benchmarks/gpqa_ellamind.py

@@ -4,18 +4,27 @@ https://huggingface.co/datasets/ellamind/gpqa-multilingual
 
 GPQA uses a single distractor set (``incorrect_answers``). The diamond variants restrict evaluation to
 the diamond subset — the 198 hardest questions (``is_diamond``) from the original GPQA-Diamond benchmark.
+The COT variant has the model reason in German and conclude with the answer letter, which is
+leniently regex-extracted from the generation (free-form completion, 0-shot).
 """
 
+import re
 from typing import Any, final, override
 
+from eval_framework.answer import ExtractFromCompletion
+from eval_framework.benchmarks.cot import Cot
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
+from eval_framework.fewshot import NoFewShot
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
 from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+from eval_framework.tasks.utils import get_n_letters
+
+GPQA_ELLAMIND_DATASET_PATH = "ellamind/gpqa-multilingual"
 
 
 @final
@@ -33,7 +42,7 @@ class GpqaReader(ChoiceReader):
 
 
 def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
-    dataset_policy = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
+    dataset_policy = dataset if dataset is not None else pinned_by_framework(GPQA_ELLAMIND_DATASET_PATH)
     return ComposedBenchmark.choice(
         id=id,
         reader=GpqaReader(),
@@ -46,16 +55,49 @@ def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy
     )
 
 
-def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+def _diamond_dataset(dataset: DatasetPolicy | None) -> DatasetPolicy:
     # The diamond variants keep only the ``is_diamond`` rows of the full dataset.
-    source = (
-        dataset
-        if dataset is not None
-        else pinned_by_framework("ellamind/gpqa-multilingual").subset(
-            lambda row: row["is_diamond"], description="the diamond subset"
-        )
+    if dataset is not None:
+        return dataset
+    return pinned_by_framework(GPQA_ELLAMIND_DATASET_PATH).subset(
+        lambda row: row["is_diamond"], description="the diamond subset"
+    )
+
+
+def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark(id, styler, _diamond_dataset(dataset))
+
+
+_ANSWER_PHRASE = '"Daher ist die Antwort (ANTWORTBUCHSTABE)"'
+_LETTER_CLAUSE = "wobei (ANTWORTBUCHSTABE) einer von (A), (B), (C), (D), (E) usw. ist."
+
+
+def tulu3_cot_prompt_de(raw_question: str, choices: list[str]) -> str:
+    """German translation of ``tulu3_cot_prompt`` (Figure 44 of the Tülu 3 paper,
+    https://arxiv.org/pdf/2411.15124): the model reasons briefly and concludes with "Daher ist die Antwort
+    (X)". The answer format is stated twice — once before the question and once as a reminder after it."""
+    keys = get_n_letters(len(choices))
+    options = "\n".join(f"({key}) {choice}" for key, choice in zip(keys, choices))
+    return (
+        "Beantworte die folgende Multiple-Choice-Frage, indem du den Buchstaben der richtigen "
+        "Antwort in Klammern angibst. Begründe deine Antwort KURZ und beende deine Antwort "
+        f"unbedingt mit {_ANSWER_PHRASE}, {_LETTER_CLAUSE}"
+        f"\n\nFrage: {raw_question}\n{options}"
+        "\n\nBeantworte die obige Frage und DENKE DARAN, deine Antwort mit genau dem Satz "
+        f"{_ANSWER_PHRASE} abzuschließen, {_LETTER_CLAUSE}"
+    )
+
+
+def tulu_answer_de() -> ExtractFromCompletion:
+    """Extracts the letter that ``tulu3_cot_prompt_de`` asks the model to conclude with, as leniently as
+    ``tulu_answer_v2``: the last match wins, the parentheses are optional, and there is no stop sequence to
+    cut the generation short. It also accepts the English "answer is X", because a model prompted in German
+    often still concludes in English. The match is anchored on the answer phrase, so a bare "Antwort D" in the
+    reasoning does not count. GPQA always has four options, so only A–D are accepted."""
+    return ExtractFromCompletion(
+        re.compile(r"\b(?:ist\s+die\s+Antwort|Antwort\s+ist|Antwort:|answer\s+is)\s*\(?([A-D])\b\)?", re.IGNORECASE),
+        last_match=True,
     )
-    return _gpqa_ellamind_benchmark(id, styler, source)
 
 
 def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
@@ -86,6 +128,19 @@ def gpqa_ellamind_diamond_bpb_de(dataset: DatasetPolicy | None = None) -> Benchm
     )
 
 
+def gpqa_ellamind_diamond_cot_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id="GPQA_ELLAMIND_DIAMOND_COT_DE",
+        kind=Cot(GpqaReader(), build_prompt=tulu3_cot_prompt_de),
+        answer=tulu_answer_de(),
+        sample_split="train",
+        fewshot=NoFewShot(),
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=_diamond_dataset(dataset),
+        language=Language.DEU,
+    )
+
+
 GPQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
     gpqa_ellamind_mc_de(),
     gpqa_ellamind_cloze_de(),
@@ -93,4 +148,5 @@ GPQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
     gpqa_ellamind_diamond_cloze_de(),
     gpqa_ellamind_bpb_de(),
     gpqa_ellamind_diamond_bpb_de(),
+    gpqa_ellamind_diamond_cot_de(),
 ]