eval-framework 0.3.4__tar.gz → 0.3.6__tar.gz

{eval_framework-0.3.4 → eval_framework-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.4
+Version: 0.3.6
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.4 → eval_framework-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.4"
+version = "0.3.6"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/evaluation_generator.py

@@ -127,7 +127,18 @@ class EvaluationGenerator:
         return results
 
     def _aggregate_results(self, results: list[Result]) -> dict[str, float | None]:
-        data = pd.DataFrame([r.model_dump() for r in results])
+        data = pd.DataFrame(
+            [
+                {
+                    "metric_name": r.metric_name,
+                    "subject": r.subject,
+                    "key": r.key,
+                    "value": r.value,
+                    "error": r.error,
+                }
+                for r in results
+            ]
+        )
         if len(data) == 0:
             return {}
         data.fillna({"key": ""}, inplace=True)
@@ -251,7 +262,20 @@ class EvaluationGenerator:
         return aggregated_results
 
     def _aggregate_results_with_aggregators(self, results: list[Result]) -> dict[str, float | None]:
-        data = pd.DataFrame([r.model_dump() for r in results])
+        data = pd.DataFrame(
+            [
+                {
+                    "metric_name": r.metric_name,
+                    "metric_class_name": r.metric_class_name,
+                    "subject": r.subject,
+                    "key": r.key,
+                    "value": r.value,
+                    "error": r.error,
+                    "prompt": r.prompt,
+                }
+                for r in results
+            ]
+        )
         if len(data) == 0:
             return {}
         data = data.fillna({"key": ""})
@@ -313,6 +337,7 @@ class EvaluationGenerator:
             raise ValueError("No saved completions found. Run 'run_completions' first.")
 
         metrics_results = self._run_metric_calculators(responses)
+        del responses
         aggregated_results = self._aggregate_results(metrics_results)
         results_with_aggregators = self._aggregate_results_with_aggregators(metrics_results)
         aggregated_results.update(results_with_aggregators)

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/metrics/completion/math_minerva_completion.py

@@ -17,21 +17,28 @@ class MathMinervaCompletion(BaseMetric[Completion]):
     Minerva MATH: reports Exact Match and Exact Match (Flex).
     Uses raw_completion to extract multiple candidates; primary for exact_match,
     all candidates with both Minerva and Hendrycks equivalence for exact_match_flex.
+
+    English Minerva extraction is the default. Subclasses select other
+    final-answer styles by overriding ``COT_STYLE`` / ``RELAXED``.
     """
 
     NAME = "Math Minerva Completion"
     KEYS = ["Exact", "Exact Flex"]
     AGGREGATORS = [PassAtK()]
 
+    # Defaults; subclasses override these class attributes to define variants.
+    COT_STYLE: str = "minerva"
+    RELAXED: bool = False
+
     def __init__(
         self,
         use_cot: bool = True,
-        cot_style: str = "minerva",
-        relaxed: bool = False,
+        cot_style: str | None = None,
+        relaxed: bool | None = None,
     ) -> None:
         self.use_cot = use_cot
-        self.cot_style = cot_style
-        self.relaxed = relaxed
+        self.cot_style = cot_style if cot_style is not None else self.COT_STYLE
+        self.relaxed = relaxed if relaxed is not None else self.RELAXED
 
     def calculate(self, response: Completion) -> list[MetricResult]:
         if response.error:
@@ -60,7 +67,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
             ]
 
         raw = response.raw_completion or response.completion
-        all_candidates = extract_answers(raw, use_cot=self.use_cot, cot_style=self.cot_style, relaxed=self.relaxed)
+        all_candidates = extract_answers(
+            raw,
+            use_cot=self.use_cot,
+            cot_style=self.cot_style,
+            relaxed=self.relaxed,
+        )
 
         exact_match = 0.0
         if all_candidates:
@@ -83,10 +95,19 @@ class MathMinervaCompletion(BaseMetric[Completion]):
 class MathMinervaCompletionRelaxed(MathMinervaCompletion):
     """MathMinervaCompletion with relaxed=True by default (flexible final-answer matching)."""
 
-    def __init__(
-        self,
-        use_cot: bool = True,
-        cot_style: str = "minerva",
-        relaxed: bool = True,
-    ) -> None:
-        super().__init__(use_cot=use_cot, cot_style=cot_style, relaxed=relaxed)
+    NAME = "Math Minerva Completion Relaxed"
+    RELAXED = True
+
+
+class MathMinervaCompletionDE(MathMinervaCompletion):
+    """MathMinervaCompletion with German final-answer extraction (``Finale Antwort: …``)."""
+
+    NAME = "Math Minerva Completion DE"
+    COT_STYLE = "minerva_de"
+
+
+class MathMinervaCompletionRelaxedDE(MathMinervaCompletionDE):
+    """MathMinervaCompletionDE with relaxed=True by default."""
+
+    NAME = "Math Minerva Completion Relaxed DE"
+    RELAXED = True

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/metrics/completion/minerva_math_utils.py

@@ -12,6 +12,7 @@ from sympy.parsing.latex.errors import LaTeXParsingError
 
 INVALID_ANSWER = "[invalidanswer]"
 END_SEQ = "I hope it is correct."
+END_SEQ_DE = "Ich hoffe, die Antwort ist korrekt."  # German pendant to END_SEQ
 
 # Minerva normalize_final_answer: appendix D of Lewkowycz et al. (2022)
 SUBSTITUTIONS = [
@@ -143,6 +144,44 @@ def get_unnormalized_answer(text: str, relaxed: bool = False) -> str:
     return INVALID_ANSWER
 
 
+def get_unnormalized_answer_de(text: str, relaxed: bool = False) -> str:
+    """German analogue of ``get_unnormalized_answer``."""
+    if relaxed:
+        match = re.search(
+            r"(?i)(?:finale|endgültige)\s+antwort\s*:\s*"
+            r"(?:die\s+(?:finale\s+|endgültige\s+)?antwort\s+(?:ist|lautet)\s*)?(.*)",
+            text,
+            re.DOTALL,
+        )
+        if match:
+            raw = match.group(1).strip()
+            raw = re.sub(
+                r"\.?\s*ich\s+hoffe,?\s+(?:die\s+antwort|sie|es)\s+(?:ist|sei)\s+korrekt\.?\s*$",
+                "",
+                raw,
+                flags=re.IGNORECASE,
+            ).strip()
+            return raw
+        return INVALID_ANSWER
+    text = text + END_SEQ_DE
+    match = re.search(
+        r"Finale Antwort: Die finale Antwort lautet(.*?)\. Ich hoffe, die Antwort ist korrekt\.",
+        text,
+    )
+    if match:
+        return match.group(1).strip()
+    return INVALID_ANSWER
+
+
+# Registry of supported `cot_style` values
+# Keys are the strings passed by metric configurations; values are language-specific final-answer extractors
+# `(text: str, relaxed: bool) -> str`. Extend this dict to add a new language.
+COT_EXTRACTORS = {
+    "minerva": get_unnormalized_answer,
+    "minerva_de": get_unnormalized_answer_de,
+}
+
+
 def normalized_gold_from_solution(solution: str) -> str | None:
     """Extract and normalize the gold answer from a solution string (last \\boxed{...})."""
     boxed = last_boxed_only_string(solution)
@@ -368,10 +407,12 @@ def extract_answers(
     all_answers: list[str] = []
 
     if use_cot:
-        if cot_style == "minerva":
-            minerva_answer = normalize_final_answer(get_unnormalized_answer(raw, relaxed=relaxed))
-            if minerva_answer and minerva_answer != INVALID_ANSWER:
-                all_answers.append(minerva_answer)
+        if cot_style not in COT_EXTRACTORS:
+            raise ValueError(f"Unknown cot_style {cot_style!r}; valid: {sorted(COT_EXTRACTORS)}")
+        extractor = COT_EXTRACTORS[cot_style]
+        minerva_answer = normalize_final_answer(extractor(raw, relaxed=relaxed))
+        if minerva_answer and minerva_answer != INVALID_ANSWER:
+            all_answers.append(minerva_answer)
         boxed = last_boxed_only_string(raw)
         if boxed is not None:
             try:

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/response_generator.py

@@ -114,8 +114,8 @@ class ResponseGenerator:
         try:
             raw_loglikelihoods = self.llm.logprobs(samples)
         except Exception as e:
-            if raise_errors():
-                raise e
+            if raise_errors() or self.config.fail_on_error:
+                raise
             logger.info(f"Error: {e.__class__.__name__} {e}")
             raw_loglikelihoods = [
                 RawLoglikelihood(
@@ -166,7 +166,8 @@ class ResponseGenerator:
                     self.llm,
                     stop_sequences=stop_sequences,
                     max_tokens=max_tokens,
-                )  # type: ignore[call-arg]
+                    fail_on_error=self.config.fail_on_error,
+                )
             case ResponseType.LOGLIKELIHOODS:
                 return self._generate_loglikelihoods
             case _:

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/result_processors/result_processor.py

@@ -36,9 +36,9 @@ class ResultsFileProcessor(ResultProcessor):
             return {}
 
     def save_responses(self, responses: list[Completion | Loglikelihood]) -> None:
-        responses_data = [response.model_dump(mode="json", serialize_as_any=True) for response in responses]
         with jsonlines.open(self.output_dir / "output.jsonl", "w") as f:
-            f.write_all(responses_data)
+            for response in responses:
+                f.write(response.model_dump(mode="json", serialize_as_any=True))
 
     def save_response(self, response: Completion | Loglikelihood) -> None:
         with jsonlines.open(self.output_dir / "output.jsonl", "a") as f:
@@ -72,9 +72,9 @@ class ResultsFileProcessor(ResultProcessor):
         return responses
 
     def save_metrics_results(self, results: list[Result]) -> None:
-        result_data = [x.model_dump(mode="json") for x in results]
         with jsonlines.open(self.output_dir / "results.jsonl", "w") as f:
-            f.write_all(result_data)
+            for result in results:
+                f.write(result.model_dump(mode="json"))
 
     def save_metrics_result(self, result: Result) -> None:
         with jsonlines.open(self.output_dir / "results.jsonl", "a") as f:

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/tasks/base.py

@@ -352,12 +352,15 @@ class BaseTask[SubjectType](ABC):
         samples: list[Sample],
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
+        fail_on_error: bool = False,
     ) -> list[Completion]:
         """
         Generates completions for the sample.
         :param sample: sample to generate completions for
         :param stop_sequences: stop sequences to use in completion generation
         :param max_tokens: maximum tokens to use in completion generation
+        :param fail_on_error: if True, re-raise the original exception instead of capturing it
+                              into a per-sample Error completion
         :return: completion
         """
         if stop_sequences is None:
@@ -367,8 +370,8 @@ class BaseTask[SubjectType](ABC):
         try:
             raw_completions = llm.generate(samples=samples, stop_sequences=stop_sequences, max_tokens=max_tokens)
         except Exception as e:
-            if raise_errors():
-                raise e
+            if raise_errors() or fail_on_error:
+                raise
             logger.info(f"Error: {e.__class__.__name__} {e}")
             raw_completions = [
                 RawCompletion(

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/tasks/benchmarks/aidanbench.py

@@ -104,7 +104,12 @@ class AidanBenchOriginal(BaseTask[str]):
         return [Message(role=Role.USER, content=instruction_message)]
 
     def _generation_loop(
-        self, llm: "BaseLLM", stop_sequences: list[str] | None, max_tokens: int | None, initial_samples: list[Sample]
+        self,
+        llm: "BaseLLM",
+        stop_sequences: list[str] | None,
+        max_tokens: int | None,
+        initial_samples: list[Sample],
+        fail_on_error: bool = False,
     ) -> tuple[list[list[Message]], list[Union["Error", None]]]:
         initial_messages = [s.messages for s in initial_samples]
         samples = [(s, False) for s in initial_samples]  # (sample, is_done)
@@ -118,6 +123,7 @@ class AidanBenchOriginal(BaseTask[str]):
                 [samples[i][0] for i in not_done_idx],
                 stop_sequences=stop_sequences,
                 max_tokens=max_tokens,
+                fail_on_error=fail_on_error,
             )
             new_completion_messages: list[list[Message] | None] = [c.messages for c in new_completions]
             new_errors = [c.error for c in new_completions]
@@ -164,11 +170,14 @@ class AidanBenchOriginal(BaseTask[str]):
         samples: list[Sample],
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
+        fail_on_error: bool = False,
     ) -> list[Completion]:
         assert all(len(s.messages) == 1 and s.messages[0].role == Role.USER for s in samples), (
             "Each sample must have exactly one USER message."
         )
-        all_message_histories, errors = self._generation_loop(llm, stop_sequences, max_tokens, samples)
+        all_message_histories, errors = self._generation_loop(
+            llm, stop_sequences, max_tokens, samples, fail_on_error=fail_on_error
+        )
 
         completion_list = []
         for idx, sample in enumerate(samples):

{eval_framework-0.3.4 → eval_framework-0.3.6}/src/eval_framework/tasks/eval_config.py

@@ -27,6 +27,7 @@ KEYS_UNRELATED_TO_RESULTS = {
     "save_intermediate_results",
     "save_logs",
     "delete_output_dir_after_upload",
+    "fail_on_error",
 }
 
 
@@ -59,6 +60,9 @@ class EvalConfig(BaseConfig):
     # how many times to repeat a single sample
     # can be used to reduce variance of tasks with low number of samples, e.g. AIME24
     repeats: Annotated[int, BeforeValidator(lambda v: 1 if v is None else v), Field(ge=1)] = 1
+    # When True, request/sample errors (e.g. unreachable inference endpoint, exhausted retries)
+    # propagate instead of being captured into a blank Error result.
+    fail_on_error: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
     @property