eval-framework 0.3.3__tar.gz → 0.3.5__tar.gz

{eval_framework-0.3.3 → eval_framework-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.3
+Version: 0.3.5
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.3 → eval_framework-0.3.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.3"
+version = "0.3.5"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/evaluation_generator.py

@@ -127,7 +127,18 @@ class EvaluationGenerator:
         return results
 
     def _aggregate_results(self, results: list[Result]) -> dict[str, float | None]:
-        data = pd.DataFrame([r.model_dump() for r in results])
+        data = pd.DataFrame(
+            [
+                {
+                    "metric_name": r.metric_name,
+                    "subject": r.subject,
+                    "key": r.key,
+                    "value": r.value,
+                    "error": r.error,
+                }
+                for r in results
+            ]
+        )
         if len(data) == 0:
             return {}
         data.fillna({"key": ""}, inplace=True)
@@ -251,7 +262,20 @@ class EvaluationGenerator:
         return aggregated_results
 
     def _aggregate_results_with_aggregators(self, results: list[Result]) -> dict[str, float | None]:
-        data = pd.DataFrame([r.model_dump() for r in results])
+        data = pd.DataFrame(
+            [
+                {
+                    "metric_name": r.metric_name,
+                    "metric_class_name": r.metric_class_name,
+                    "subject": r.subject,
+                    "key": r.key,
+                    "value": r.value,
+                    "error": r.error,
+                    "prompt": r.prompt,
+                }
+                for r in results
+            ]
+        )
         if len(data) == 0:
             return {}
         data = data.fillna({"key": ""})
@@ -313,6 +337,7 @@ class EvaluationGenerator:
             raise ValueError("No saved completions found. Run 'run_completions' first.")
 
         metrics_results = self._run_metric_calculators(responses)
+        del responses
         aggregated_results = self._aggregate_results(metrics_results)
         results_with_aggregators = self._aggregate_results_with_aggregators(metrics_results)
         aggregated_results.update(results_with_aggregators)

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/response_generator.py

@@ -78,7 +78,7 @@ class ResponseGenerator:
                 custom_hf_revision=self.config.hf_revision,
             )
 
-        self.response_type, _ = self.task._get_type_and_metrics()
+        self.response_type = self.task.get_response_type()
 
     def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]:
         """
@@ -114,8 +114,8 @@ class ResponseGenerator:
         try:
             raw_loglikelihoods = self.llm.logprobs(samples)
         except Exception as e:
-            if raise_errors():
-                raise e
+            if raise_errors() or self.config.fail_on_error:
+                raise
             logger.info(f"Error: {e.__class__.__name__} {e}")
             raw_loglikelihoods = [
                 RawLoglikelihood(
@@ -166,7 +166,8 @@ class ResponseGenerator:
                     self.llm,
                     stop_sequences=stop_sequences,
                     max_tokens=max_tokens,
-                )  # type: ignore[call-arg]
+                    fail_on_error=self.config.fail_on_error,
+                )
             case ResponseType.LOGLIKELIHOODS:
                 return self._generate_loglikelihoods
             case _:

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/result_processors/result_processor.py

@@ -36,9 +36,9 @@ class ResultsFileProcessor(ResultProcessor):
             return {}
 
     def save_responses(self, responses: list[Completion | Loglikelihood]) -> None:
-        responses_data = [response.model_dump(mode="json", serialize_as_any=True) for response in responses]
         with jsonlines.open(self.output_dir / "output.jsonl", "w") as f:
-            f.write_all(responses_data)
+            for response in responses:
+                f.write(response.model_dump(mode="json", serialize_as_any=True))
 
     def save_response(self, response: Completion | Loglikelihood) -> None:
         with jsonlines.open(self.output_dir / "output.jsonl", "a") as f:
@@ -72,9 +72,9 @@ class ResultsFileProcessor(ResultProcessor):
         return responses
 
     def save_metrics_results(self, results: list[Result]) -> None:
-        result_data = [x.model_dump(mode="json") for x in results]
         with jsonlines.open(self.output_dir / "results.jsonl", "w") as f:
-            f.write_all(result_data)
+            for result in results:
+                f.write(result.model_dump(mode="json"))
 
     def save_metrics_result(self, result: Result) -> None:
         with jsonlines.open(self.output_dir / "results.jsonl", "a") as f:

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/tasks/base.py

@@ -15,7 +15,7 @@ from huggingface_hub.errors import RevisionNotFoundError
 from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
-from eval_framework.tasks.utils import raise_errors
+from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import Message, Role
 
 if TYPE_CHECKING:
@@ -91,8 +91,6 @@ class BaseTask[SubjectType](ABC):
     DATASET_PATH: str
     SAMPLE_SPLIT: str
     FEWSHOT_SPLIT: str
-    RESPONSE_TYPE: ResponseType
-    METRICS: list[type["BaseMetric"]]
     SUBJECTS: list[SubjectType]
     HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
@@ -104,6 +102,10 @@ class BaseTask[SubjectType](ABC):
     # language by subtopic, or `None` (for tasks not specific to a single language).
     LANGUAGE: Language | dict[str, Language] | dict[str, tuple[Language, Language]] | None
 
+    # RESPONSE_TYPE and METRICS use exposed as classproperties, so you can access them via either
+    # `TaskClass.*` or `task.*` (or `task.get_metrics()`). This avoids mypy conflicts from re-declaring class vars.
+    # By default, these values come from TASK_STYLER if set, otherwise from legacy class attributes.
+
     def __init__(self, num_fewshot: int = 0) -> None:
         self.num_fewshot = num_fewshot
         self.stop_sequences: list[str] | None = None
@@ -332,14 +334,12 @@ class BaseTask[SubjectType](ABC):
         return None
 
     def get_metadata(self) -> dict[str, str | list[str]]:
-        response_type, metrics = self._get_type_and_metrics()
-
         meta: dict[str, str | list[str]] = {
             "dataset_path": self.DATASET_PATH,
             "sample_split": self.SAMPLE_SPLIT,
             "fewshot_split": self.FEWSHOT_SPLIT,
-            "response_type": response_type.value,
-            "metrics": [m.NAME for m in metrics],
+            "response_type": self.get_response_type().value,
+            "metrics": [m.NAME for m in self.get_metrics()],
             "subjects": [str(s) for s in self.SUBJECTS],
         }
         if hasattr(self, "TASK_STYLER"):
@@ -352,12 +352,15 @@ class BaseTask[SubjectType](ABC):
         samples: list[Sample],
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
+        fail_on_error: bool = False,
     ) -> list[Completion]:
         """
         Generates completions for the sample.
         :param sample: sample to generate completions for
         :param stop_sequences: stop sequences to use in completion generation
         :param max_tokens: maximum tokens to use in completion generation
+        :param fail_on_error: if True, re-raise the original exception instead of capturing it
+                              into a per-sample Error completion
         :return: completion
         """
         if stop_sequences is None:
@@ -367,8 +370,8 @@ class BaseTask[SubjectType](ABC):
         try:
             raw_completions = llm.generate(samples=samples, stop_sequences=stop_sequences, max_tokens=max_tokens)
         except Exception as e:
-            if raise_errors():
-                raise e
+            if raise_errors() or fail_on_error:
+                raise
             logger.info(f"Error: {e.__class__.__name__} {e}")
             raw_completions = [
                 RawCompletion(
@@ -420,7 +423,26 @@ class BaseTask[SubjectType](ABC):
             )
         return completion_list
 
-    def _get_type_and_metrics(self) -> tuple[ResponseType, list[type["BaseMetric"]]]:
-        if hasattr(self, "TASK_STYLER"):
-            return self.TASK_STYLER.response_type, self.TASK_STYLER.metrics
-        return self.RESPONSE_TYPE, self.METRICS
+    @classmethod
+    def get_response_type(cls) -> ResponseType:
+        """Return the response type of the task (or the styler if it exists)."""
+        if hasattr(cls, "TASK_STYLER"):
+            return cls.TASK_STYLER.response_type
+        return cls.RESPONSE_TYPE
+
+    @classmethod
+    def get_metrics(cls) -> list[type["BaseMetric"]]:
+        """Return the metrics of the task (or the styler if it exists)."""
+        if hasattr(cls, "TASK_STYLER"):
+            return cls.TASK_STYLER.metrics
+        return cls.METRICS
+
+    @classproperty
+    def RESPONSE_TYPE(cls) -> ResponseType:
+        """For backwards compatibility."""
+        return cls.get_response_type()
+
+    @classproperty
+    def METRICS(cls) -> list[type["BaseMetric"]]:
+        """For backwards compatibility."""
+        return cls.get_metrics()

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/tasks/benchmarks/aidanbench.py

@@ -104,7 +104,12 @@ class AidanBenchOriginal(BaseTask[str]):
         return [Message(role=Role.USER, content=instruction_message)]
 
     def _generation_loop(
-        self, llm: "BaseLLM", stop_sequences: list[str] | None, max_tokens: int | None, initial_samples: list[Sample]
+        self,
+        llm: "BaseLLM",
+        stop_sequences: list[str] | None,
+        max_tokens: int | None,
+        initial_samples: list[Sample],
+        fail_on_error: bool = False,
     ) -> tuple[list[list[Message]], list[Union["Error", None]]]:
         initial_messages = [s.messages for s in initial_samples]
         samples = [(s, False) for s in initial_samples]  # (sample, is_done)
@@ -118,6 +123,7 @@ class AidanBenchOriginal(BaseTask[str]):
                 [samples[i][0] for i in not_done_idx],
                 stop_sequences=stop_sequences,
                 max_tokens=max_tokens,
+                fail_on_error=fail_on_error,
             )
             new_completion_messages: list[list[Message] | None] = [c.messages for c in new_completions]
             new_errors = [c.error for c in new_completions]
@@ -164,11 +170,14 @@ class AidanBenchOriginal(BaseTask[str]):
         samples: list[Sample],
         stop_sequences: list[str] | None = None,
         max_tokens: int | None = None,
+        fail_on_error: bool = False,
     ) -> list[Completion]:
         assert all(len(s.messages) == 1 and s.messages[0].role == Role.USER for s in samples), (
             "Each sample must have exactly one USER message."
         )
-        all_message_histories, errors = self._generation_loop(llm, stop_sequences, max_tokens, samples)
+        all_message_histories, errors = self._generation_loop(
+            llm, stop_sequences, max_tokens, samples, fail_on_error=fail_on_error
+        )
 
         completion_list = []
         for idx, sample in enumerate(samples):

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -86,7 +86,7 @@ class NaturalQsOpenCloze(_NaturalQsOpenChoice_Base):
 
 class NaturalQsOpenMC(_NaturalQsOpenChoice_Base):
     NAME = "NaturalQsOpenMC"
-    TASK_STYLER = MCStyle(space_prefixed_labels=True)
+    TASK_STYLER = MCStyle()
 
 
 class NaturalQsOpenMC_OLMES(_NaturalQsOpenChoice_Base):

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/tasks/eval_config.py

@@ -27,6 +27,7 @@ KEYS_UNRELATED_TO_RESULTS = {
     "save_intermediate_results",
     "save_logs",
     "delete_output_dir_after_upload",
+    "fail_on_error",
 }
 
 
@@ -59,6 +60,9 @@ class EvalConfig(BaseConfig):
     # how many times to repeat a single sample
     # can be used to reduce variance of tasks with low number of samples, e.g. AIME24
     repeats: Annotated[int, BeforeValidator(lambda v: 1 if v is None else v), Field(ge=1)] = 1
+    # When True, request/sample errors (e.g. unreachable inference endpoint, exhausted retries)
+    # propagate instead of being captured into a blank Error result.
+    fail_on_error: Annotated[bool, BeforeValidator(lambda v: False if v is None else v)] = False
     # Adding a new member? Remember to update KEYS_UNRELATED_TO_RESULTS if it doesn't impact eval results.
 
     @property
@@ -112,7 +116,7 @@ class EvalConfig(BaseConfig):
     @model_validator(mode="after")
     def validate_llm_judge_defined(self) -> "EvalConfig":
         task = get_task(self.task_name)
-        _, task_metrics = task(num_fewshot=0)._get_type_and_metrics()
+        task_metrics = task(num_fewshot=0).get_metrics()
         for metric_class in task_metrics:
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 assert self.llm_judge_class is not None, "The LLM Judge must be defined for this evaluation task."

{eval_framework-0.3.3 → eval_framework-0.3.5}/src/eval_framework/tasks/utils.py

@@ -8,7 +8,7 @@ import string
 import threading
 from collections.abc import Callable
 from pathlib import Path
-from typing import Any, Literal, NamedTuple
+from typing import Any, Literal, NamedTuple, overload
 
 import dill
 import numpy as np
@@ -22,6 +22,24 @@ logger = logging.getLogger(__name__)
 RANDOM_SEED = 42  # hacky way to get around circular import
 redis_warning_printed = False
 
+
+class classproperty[T]:
+    """Descriptor supporting property-like access on classes and instances."""
+
+    def __init__(self, fget: Callable[[Any], T]) -> None:
+        self.fget = fget
+
+    @overload
+    def __get__(self, obj: None, owner: type[Any]) -> T: ...
+
+    @overload
+    def __get__(self, obj: object, owner: type[Any] | None = None) -> T: ...
+
+    def __get__(self, obj: object | None, owner: type[Any] | None = None) -> T:
+        cls = owner if owner is not None else type(obj)
+        return self.fget(cls)
+
+
 _pools: dict[tuple[str | None, tuple[str, ...] | None], ContainerPoolManager] = {}
 _pools_lock = threading.Lock()