eval-framework 0.3.5__tar.gz → 0.3.7__tar.gz

{eval_framework-0.3.5 → eval_framework-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.5
+Version: 0.3.7
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.5 → eval_framework-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.5"
+version = "0.3.7"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.5 → eval_framework-0.3.7}/src/eval_framework/llm/openai.py

@@ -8,18 +8,38 @@ from functools import partial
 
 import tiktoken
 from openai import OpenAI
-from openai.types.chat import ChatCompletionAssistantMessageParam, ChatCompletionUserMessageParam
+from openai.types.chat import (
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionMessageParam,
+    ChatCompletionSystemMessageParam,
+    ChatCompletionUserMessageParam,
+)
 from tokenizers import Tokenizer
 from transformers import AutoTokenizer
 
 from eval_framework.llm.base import BaseLLM
 from eval_framework.shared.types import ConcatCompression, Error, RawCompletion, RawLoglikelihood
 from eval_framework.tasks.base import Sample
-from template_formatting.formatter import BaseFormatter, ConcatFormatter, HFFormatter, Message
+from template_formatting.formatter import BaseFormatter, ConcatFormatter, HFFormatter, Message, Role
 
 logger = logging.getLogger(__name__)
 
 
+def _to_chat_completion_message(message: Message) -> ChatCompletionMessageParam:
+    match message.role:
+        case Role.SYSTEM:
+            return ChatCompletionSystemMessageParam(role="system", content=message.content)
+        case Role.USER:
+            return ChatCompletionUserMessageParam(role="user", content=message.content)
+        case Role.ASSISTANT:
+            return ChatCompletionAssistantMessageParam(role="assistant", content=message.content)
+        case None:
+            raise ValueError(
+                "Cannot send a Message without a role through the chat completion API; "
+                "the legacy roleless format is only supported for fine-tuning."
+            )
+
+
 class OpenAIModel(BaseLLM):
     """
     LLM wrapper for OpenAI API providing text/chat completions and log-probability evaluation output.
@@ -158,14 +178,7 @@ class OpenAIModel(BaseLLM):
 
             else:
                 # Use chat completion API
-                chat_messages = [
-                    (
-                        ChatCompletionUserMessageParam(role="user", content=m.content)
-                        if m.role is not None and m.role.value.lower() == "user"
-                        else ChatCompletionAssistantMessageParam(role="assistant", content=m.content)
-                    )
-                    for m in single_messages
-                ]
+                chat_messages = [_to_chat_completion_message(m) for m in single_messages]
                 assert self._model_name is not None
                 chat_response = self._client.chat.completions.create(
                     model=self._model_name,

{eval_framework-0.3.5 → eval_framework-0.3.7}/src/eval_framework/metrics/completion/math_minerva_completion.py

@@ -17,21 +17,28 @@ class MathMinervaCompletion(BaseMetric[Completion]):
     Minerva MATH: reports Exact Match and Exact Match (Flex).
     Uses raw_completion to extract multiple candidates; primary for exact_match,
     all candidates with both Minerva and Hendrycks equivalence for exact_match_flex.
+
+    English Minerva extraction is the default. Subclasses select other
+    final-answer styles by overriding ``COT_STYLE`` / ``RELAXED``.
     """
 
     NAME = "Math Minerva Completion"
     KEYS = ["Exact", "Exact Flex"]
     AGGREGATORS = [PassAtK()]
 
+    # Defaults; subclasses override these class attributes to define variants.
+    COT_STYLE: str = "minerva"
+    RELAXED: bool = False
+
     def __init__(
         self,
         use_cot: bool = True,
-        cot_style: str = "minerva",
-        relaxed: bool = False,
+        cot_style: str | None = None,
+        relaxed: bool | None = None,
     ) -> None:
         self.use_cot = use_cot
-        self.cot_style = cot_style
-        self.relaxed = relaxed
+        self.cot_style = cot_style if cot_style is not None else self.COT_STYLE
+        self.relaxed = relaxed if relaxed is not None else self.RELAXED
 
     def calculate(self, response: Completion) -> list[MetricResult]:
         if response.error:
@@ -60,7 +67,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
             ]
 
         raw = response.raw_completion or response.completion
-        all_candidates = extract_answers(raw, use_cot=self.use_cot, cot_style=self.cot_style, relaxed=self.relaxed)
+        all_candidates = extract_answers(
+            raw,
+            use_cot=self.use_cot,
+            cot_style=self.cot_style,
+            relaxed=self.relaxed,
+        )
 
         exact_match = 0.0
         if all_candidates:
@@ -83,10 +95,19 @@ class MathMinervaCompletion(BaseMetric[Completion]):
 class MathMinervaCompletionRelaxed(MathMinervaCompletion):
     """MathMinervaCompletion with relaxed=True by default (flexible final-answer matching)."""
 
-    def __init__(
-        self,
-        use_cot: bool = True,
-        cot_style: str = "minerva",
-        relaxed: bool = True,
-    ) -> None:
-        super().__init__(use_cot=use_cot, cot_style=cot_style, relaxed=relaxed)
+    NAME = "Math Minerva Completion Relaxed"
+    RELAXED = True
+
+
+class MathMinervaCompletionDE(MathMinervaCompletion):
+    """MathMinervaCompletion with German final-answer extraction (``Finale Antwort: …``)."""
+
+    NAME = "Math Minerva Completion DE"
+    COT_STYLE = "minerva_de"
+
+
+class MathMinervaCompletionRelaxedDE(MathMinervaCompletionDE):
+    """MathMinervaCompletionDE with relaxed=True by default."""
+
+    NAME = "Math Minerva Completion Relaxed DE"
+    RELAXED = True

{eval_framework-0.3.5 → eval_framework-0.3.7}/src/eval_framework/metrics/completion/minerva_math_utils.py

@@ -12,6 +12,7 @@ from sympy.parsing.latex.errors import LaTeXParsingError
 
 INVALID_ANSWER = "[invalidanswer]"
 END_SEQ = "I hope it is correct."
+END_SEQ_DE = "Ich hoffe, die Antwort ist korrekt."  # German pendant to END_SEQ
 
 # Minerva normalize_final_answer: appendix D of Lewkowycz et al. (2022)
 SUBSTITUTIONS = [
@@ -143,6 +144,44 @@ def get_unnormalized_answer(text: str, relaxed: bool = False) -> str:
     return INVALID_ANSWER
 
 
+def get_unnormalized_answer_de(text: str, relaxed: bool = False) -> str:
+    """German analogue of ``get_unnormalized_answer``."""
+    if relaxed:
+        match = re.search(
+            r"(?i)(?:finale|endgültige)\s+antwort\s*:\s*"
+            r"(?:die\s+(?:finale\s+|endgültige\s+)?antwort\s+(?:ist|lautet)\s*)?(.*)",
+            text,
+            re.DOTALL,
+        )
+        if match:
+            raw = match.group(1).strip()
+            raw = re.sub(
+                r"\.?\s*ich\s+hoffe,?\s+(?:die\s+antwort|sie|es)\s+(?:ist|sei)\s+korrekt\.?\s*$",
+                "",
+                raw,
+                flags=re.IGNORECASE,
+            ).strip()
+            return raw
+        return INVALID_ANSWER
+    text = text + END_SEQ_DE
+    match = re.search(
+        r"Finale Antwort: Die finale Antwort lautet(.*?)\. Ich hoffe, die Antwort ist korrekt\.",
+        text,
+    )
+    if match:
+        return match.group(1).strip()
+    return INVALID_ANSWER
+
+
+# Registry of supported `cot_style` values
+# Keys are the strings passed by metric configurations; values are language-specific final-answer extractors
+# `(text: str, relaxed: bool) -> str`. Extend this dict to add a new language.
+COT_EXTRACTORS = {
+    "minerva": get_unnormalized_answer,
+    "minerva_de": get_unnormalized_answer_de,
+}
+
+
 def normalized_gold_from_solution(solution: str) -> str | None:
     """Extract and normalize the gold answer from a solution string (last \\boxed{...})."""
     boxed = last_boxed_only_string(solution)
@@ -368,10 +407,12 @@ def extract_answers(
     all_answers: list[str] = []
 
     if use_cot:
-        if cot_style == "minerva":
-            minerva_answer = normalize_final_answer(get_unnormalized_answer(raw, relaxed=relaxed))
-            if minerva_answer and minerva_answer != INVALID_ANSWER:
-                all_answers.append(minerva_answer)
+        if cot_style not in COT_EXTRACTORS:
+            raise ValueError(f"Unknown cot_style {cot_style!r}; valid: {sorted(COT_EXTRACTORS)}")
+        extractor = COT_EXTRACTORS[cot_style]
+        minerva_answer = normalize_final_answer(extractor(raw, relaxed=relaxed))
+        if minerva_answer and minerva_answer != INVALID_ANSWER:
+            all_answers.append(minerva_answer)
         boxed = last_boxed_only_string(raw)
         if boxed is not None:
             try:

{eval_framework-0.3.5 → eval_framework-0.3.7}/src/eval_framework/tasks/benchmarks/ifeval.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
+from eval_framework.metrics.completion.language_checker import LanguageRawConsistencyChecker
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
 
 
@@ -76,3 +77,4 @@ class IFEvalDe(IFEval):
     DATASET_PATH = "jzhang86/de_ifeval"
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = {NO_SUBJECT: Language.DEU}
+    METRICS = [IFEvalMetric, LanguageRawConsistencyChecker]