eval-framework 0.3.2__tar.gz → 0.3.4__tar.gz

{eval_framework-0.3.2 → eval_framework-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.2
+Version: 0.3.4
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.2 → eval_framework-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.2"
+version = "0.3.4"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/metrics/completion/drop_completion.py

@@ -18,7 +18,7 @@ class DropMetricContext(BaseMetricContext):
 class DropF1ExactMatch(BaseMetric[Completion]):
     """DROP F1 and exact match. Requires DropMetricContext with answer_tuples."""
 
-    NAME = "DROP F1 / Exact Match"
+    NAME = "Drop F1"
     KEYS = ["f1", "exact_match"]
 
     def calculate(self, response: Completion) -> list[MetricResult]:
@@ -52,5 +52,5 @@ class DropF1ExactMatch(BaseMetric[Completion]):
                 higher_is_better=True,
                 error=response.error,
             )
-            for name, key in zip([n.strip() for n in self.NAME.split("/")], self.KEYS)
+            for name, key in zip(self.NAMES, self.KEYS)
         ]

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/metrics/completion/math_minerva_completion.py

@@ -20,6 +20,7 @@ class MathMinervaCompletion(BaseMetric[Completion]):
     """
 
     NAME = "Math Minerva Completion"
+    KEYS = ["Exact", "Exact Flex"]
     AGGREGATORS = [PassAtK()]
 
     def __init__(
@@ -36,17 +37,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         if response.error:
             return [
                 MetricResult(
-                    metric_name="Exact Match",
+                    metric_name=x,
                     value=None,
                     higher_is_better=True,
                     error=response.error,
-                ),
-                MetricResult(
-                    metric_name="Exact Match (Flex)",
-                    value=None,
-                    higher_is_better=True,
-                    error=response.error,
-                ),
+                )
+                for x in self.NAMES
             ]
 
         gold = response.ground_truth
@@ -55,17 +51,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         if not gold:
             return [
                 MetricResult(
-                    metric_name="Exact Match",
-                    value=None,
-                    higher_is_better=True,
-                    error="No ground truth available",
-                ),
-                MetricResult(
-                    metric_name="Exact Match (Flex)",
+                    metric_name=x,
                     value=None,
                     higher_is_better=True,
                     error="No ground truth available",
-                ),
+                )
+                for x in self.NAMES
             ]
 
         raw = response.raw_completion or response.completion
@@ -84,12 +75,8 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         )
 
         return [
-            MetricResult(metric_name="Exact Match", value=exact_match, higher_is_better=True),
-            MetricResult(
-                metric_name="Exact Match (Flex)",
-                value=exact_match_flex,
-                higher_is_better=True,
-            ),
+            MetricResult(metric_name=name, value=value, higher_is_better=True)
+            for name, value in zip(self.NAMES, [exact_match, exact_match_flex])
         ]
 
 

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/response_generator.py

@@ -78,7 +78,7 @@ class ResponseGenerator:
                 custom_hf_revision=self.config.hf_revision,
             )
 
-        self.response_type, _ = self.task._get_type_and_metrics()
+        self.response_type = self.task.get_response_type()
 
     def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]:
         """

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/tasks/base.py

@@ -15,7 +15,7 @@ from huggingface_hub.errors import RevisionNotFoundError
 from pydantic import BaseModel, ConfigDict
 
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
-from eval_framework.tasks.utils import raise_errors
+from eval_framework.tasks.utils import classproperty, raise_errors
 from template_formatting.formatter import Message, Role
 
 if TYPE_CHECKING:
@@ -91,8 +91,6 @@ class BaseTask[SubjectType](ABC):
     DATASET_PATH: str
     SAMPLE_SPLIT: str
     FEWSHOT_SPLIT: str
-    RESPONSE_TYPE: ResponseType
-    METRICS: list[type["BaseMetric"]]
     SUBJECTS: list[SubjectType]
     HF_REVISION: str | None = None  # tag name, or branch name, or commit hash to ensure reproducibility
 
@@ -104,6 +102,10 @@ class BaseTask[SubjectType](ABC):
     # language by subtopic, or `None` (for tasks not specific to a single language).
     LANGUAGE: Language | dict[str, Language] | dict[str, tuple[Language, Language]] | None
 
+    # RESPONSE_TYPE and METRICS use exposed as classproperties, so you can access them via either
+    # `TaskClass.*` or `task.*` (or `task.get_metrics()`). This avoids mypy conflicts from re-declaring class vars.
+    # By default, these values come from TASK_STYLER if set, otherwise from legacy class attributes.
+
     def __init__(self, num_fewshot: int = 0) -> None:
         self.num_fewshot = num_fewshot
         self.stop_sequences: list[str] | None = None
@@ -332,14 +334,12 @@ class BaseTask[SubjectType](ABC):
         return None
 
     def get_metadata(self) -> dict[str, str | list[str]]:
-        response_type, metrics = self._get_type_and_metrics()
-
         meta: dict[str, str | list[str]] = {
             "dataset_path": self.DATASET_PATH,
             "sample_split": self.SAMPLE_SPLIT,
             "fewshot_split": self.FEWSHOT_SPLIT,
-            "response_type": response_type.value,
-            "metrics": [m.NAME for m in metrics],
+            "response_type": self.get_response_type().value,
+            "metrics": [m.NAME for m in self.get_metrics()],
             "subjects": [str(s) for s in self.SUBJECTS],
         }
         if hasattr(self, "TASK_STYLER"):
@@ -420,7 +420,26 @@ class BaseTask[SubjectType](ABC):
             )
         return completion_list
 
-    def _get_type_and_metrics(self) -> tuple[ResponseType, list[type["BaseMetric"]]]:
-        if hasattr(self, "TASK_STYLER"):
-            return self.TASK_STYLER.response_type, self.TASK_STYLER.metrics
-        return self.RESPONSE_TYPE, self.METRICS
+    @classmethod
+    def get_response_type(cls) -> ResponseType:
+        """Return the response type of the task (or the styler if it exists)."""
+        if hasattr(cls, "TASK_STYLER"):
+            return cls.TASK_STYLER.response_type
+        return cls.RESPONSE_TYPE
+
+    @classmethod
+    def get_metrics(cls) -> list[type["BaseMetric"]]:
+        """Return the metrics of the task (or the styler if it exists)."""
+        if hasattr(cls, "TASK_STYLER"):
+            return cls.TASK_STYLER.metrics
+        return cls.METRICS
+
+    @classproperty
+    def RESPONSE_TYPE(cls) -> ResponseType:
+        """For backwards compatibility."""
+        return cls.get_response_type()
+
+    @classproperty
+    def METRICS(cls) -> list[type["BaseMetric"]]:
+        """For backwards compatibility."""
+        return cls.get_metrics()

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -86,7 +86,7 @@ class NaturalQsOpenCloze(_NaturalQsOpenChoice_Base):
 
 class NaturalQsOpenMC(_NaturalQsOpenChoice_Base):
     NAME = "NaturalQsOpenMC"
-    TASK_STYLER = MCStyle(space_prefixed_labels=True)
+    TASK_STYLER = MCStyle()
 
 
 class NaturalQsOpenMC_OLMES(_NaturalQsOpenChoice_Base):

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/tasks/benchmarks/truthfulqa.py

@@ -106,13 +106,18 @@ class TRUTHFULQA_OLMES(TRUTHFULQA):
     """
 
     NAME = "TruthfulQA_OLMES"
+    FEWSHOT_SPLIT = "validation"  # use dataset few-shot for multiple-choice options
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         question = item["question"]
         choices = item[self.target_identifier]["choices"]
         labels = get_n_letters(len(choices))
         options = "\n".join(f" {label}. {choice}" for label, choice in zip(labels, choices))
-        return f"Q: {question}\n{options}\n"
+        return f"Question: {question}\n{options}\n"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
         labels_arr = item[self.target_identifier]["labels"]
@@ -126,7 +131,17 @@ class TRUTHFULQA_OLMES(TRUTHFULQA):
         return [f" {letter}" for letter in letters]
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
-        return self.FEWSHOT_ITEMS[: self.num_fewshot]
+        # Reuse BaseTask's split-based sampler.
+        return BaseTask._sample_fewshot_examples(self, item)
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        cue_text = self._get_cue_text(item)
+        labels_arr = item[self.target_identifier]["labels"]
+        letters = get_n_letters(len(labels_arr))
+        # Pick one correct option for the demonstration answer.
+        correct_letters = [letters[i] for i, label in enumerate(labels_arr) if label == 1]
+        letter = correct_letters[0] if correct_letters else letters[0]
+        return f"{cue_text} {letter}"
 
 
 class TRUTHFULQA_IDK(TRUTHFULQA):

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/tasks/eval_config.py

@@ -112,7 +112,7 @@ class EvalConfig(BaseConfig):
     @model_validator(mode="after")
     def validate_llm_judge_defined(self) -> "EvalConfig":
         task = get_task(self.task_name)
-        _, task_metrics = task(num_fewshot=0)._get_type_and_metrics()
+        task_metrics = task(num_fewshot=0).get_metrics()
         for metric_class in task_metrics:
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 assert self.llm_judge_class is not None, "The LLM Judge must be defined for this evaluation task."

{eval_framework-0.3.2 → eval_framework-0.3.4}/src/eval_framework/tasks/utils.py

@@ -8,7 +8,7 @@ import string
 import threading
 from collections.abc import Callable
 from pathlib import Path
-from typing import Any, Literal, NamedTuple
+from typing import Any, Literal, NamedTuple, overload
 
 import dill
 import numpy as np
@@ -22,6 +22,24 @@ logger = logging.getLogger(__name__)
 RANDOM_SEED = 42  # hacky way to get around circular import
 redis_warning_printed = False
 
+
+class classproperty[T]:
+    """Descriptor supporting property-like access on classes and instances."""
+
+    def __init__(self, fget: Callable[[Any], T]) -> None:
+        self.fget = fget
+
+    @overload
+    def __get__(self, obj: None, owner: type[Any]) -> T: ...
+
+    @overload
+    def __get__(self, obj: object, owner: type[Any] | None = None) -> T: ...
+
+    def __get__(self, obj: object | None, owner: type[Any] | None = None) -> T:
+        cls = owner if owner is not None else type(obj)
+        return self.fget(cls)
+
+
 _pools: dict[tuple[str | None, tuple[str, ...] | None], ContainerPoolManager] = {}
 _pools_lock = threading.Lock()