eval-framework 0.3.0__tar.gz → 0.3.2__tar.gz

{eval_framework-0.3.0 → eval_framework-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.0
+Version: 0.3.2
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.0 → eval_framework-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.0"
+version = "0.3.2"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/response_generator.py

@@ -67,14 +67,18 @@ class ResponseGenerator:
         if config.perturbation_config is not None:
             perturbation_task_class = create_perturbation_class(task_class, config.perturbation_config)
             self.task = perturbation_task_class.with_overwrite(
-                self.few_shot, custom_subjects=self.config.task_subjects, custom_hf_revision=self.config.hf_revision
+                self.few_shot,
+                custom_subjects=self.config.task_subjects,
+                custom_hf_revision=self.config.hf_revision,
             )
         else:
             self.task = task_class.with_overwrite(
-                self.few_shot, custom_subjects=self.config.task_subjects, custom_hf_revision=self.config.hf_revision
+                self.few_shot,
+                custom_subjects=self.config.task_subjects,
+                custom_hf_revision=self.config.hf_revision,
             )
 
-        self.response_type = task_class.RESPONSE_TYPE
+        self.response_type, _ = self.task._get_type_and_metrics()
 
     def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]:
         """
@@ -89,7 +93,10 @@ class ResponseGenerator:
         task_stop_sequences = getattr(self.task, "stop_sequences", None)
         task_max_tokens = self.config.max_tokens or getattr(self.task, "max_tokens", None)
         # if both task and model define a max_token, the smaller value is used
-        max_tokens = min([x for x in [llm_max_tokens, task_max_tokens] if x is not None], default=None)
+        max_tokens = min(
+            [x for x in [llm_max_tokens, task_max_tokens] if x is not None],
+            default=None,
+        )
         logger.info(f"Set max_tokens to {max_tokens}")
         # if both task and model define stop sequences, those are merged into one list
         stop_sequences_merged = (llm_stop_sequences or []) + (task_stop_sequences or [])
@@ -117,7 +124,9 @@ class ResponseGenerator:
                     loglikelihoods={},
                     loglikelihoods_sequence_positions={},
                     raw_loglikelihood_error=Error(
-                        error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc()
+                        error_class=e.__class__.__name__,
+                        message=str(e),
+                        traceback=traceback.format_exc(),
                     ),
                 )
                 for _ in range(len(samples))
@@ -142,7 +151,9 @@ class ResponseGenerator:
             )
         return loglikelihood_list
 
-    def _generative_output_type_selector(self) -> Callable[[list[Sample]], list[Completion] | list[Loglikelihood]]:
+    def _generative_output_type_selector(
+        self,
+    ) -> Callable[[list[Sample]], list[Completion] | list[Loglikelihood]]:
         """
         Selects the generative output type based on the response type.
         :return: function to generate responses
@@ -151,7 +162,10 @@ class ResponseGenerator:
             case ResponseType.COMPLETION:
                 stop_sequences, max_tokens = self._llm_task_param_precedence()
                 return partial(
-                    self.task.generate_completions, self.llm, stop_sequences=stop_sequences, max_tokens=max_tokens
+                    self.task.generate_completions,
+                    self.llm,
+                    stop_sequences=stop_sequences,
+                    max_tokens=max_tokens,
                 )  # type: ignore[call-arg]
             case ResponseType.LOGLIKELIHOODS:
                 return self._generate_loglikelihoods
@@ -245,7 +259,9 @@ class ResponseGenerator:
 
         samples_batch: list[Sample] = []
         with tqdm(
-            total=total_num_samples, desc=f"Processing {self.response_type.value}", disable=get_disable_bar_flag()
+            total=total_num_samples,
+            desc=f"Processing {self.response_type.value}",
+            disable=get_disable_bar_flag(),
         ) as pbar:
             samples = self.task.iterate_samples(self.num_samples)
             for i, sample in enumerate(repeat_samples(samples, repeats)):

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/suite.py

@@ -46,7 +46,17 @@ def parse_strings_to_task_or_suite(v: str | list) -> str | list:
     return [{"tasks": item, "name": item} if isinstance(item, str) else item for item in v]
 
 
-_VALID_METHODS = {"mean", "median", "passthrough"}
+_VALID_METHODS = {"mean", "median"}
+
+
+class MetricSource(BaseModel):
+    """A single (child, metric) pair used as an input to a SuiteAggregate. See the examples folder
+    for how these are used."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    child: str
+    metric: str
 
 
 class SuiteAggregate(BaseModel):
@@ -55,7 +65,7 @@ class SuiteAggregate(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     name: str
-    metric: Annotated[list[str], BeforeValidator(lambda v: [v] if isinstance(v, str) else v)]
+    sources: list[MetricSource]
     method: str | Callable[[list[float]], float] = "mean"
 
     @field_validator("method")
@@ -197,52 +207,44 @@ def compute_aggregates(
     aggregates: list[SuiteAggregate],
     child_results: dict[str, SuiteResult],
 ) -> dict[str, float | None]:
-    """Compute suite-level stats from children's results.
+    """Compute suite-level stats from explicitly named (child, metric) sources.
 
-    Each SuiteAggregate either reduces a metric across all children
-    (method="mean", "median", or a callable) or surfaces a metric
-    from exactly one child as-is (method="passthrough").
+    For each `SuiteAggregate`, the value from each `MetricSource` is looked up by
+    child name and exact metric key. Sources whose child is missing or whose metric is
+    None or NaN are silently skipped. If no sources yield a valid value the aggregate is None.
     """
     result: dict[str, float | None] = {}
 
     for agg in aggregates:
-        if agg.method == "passthrough":
-            child = child_results.get(agg.name)
+        values: list[float] = []
+        for source in agg.sources:
+            child = child_results.get(source.child)
             if child is None:
-                raise ValueError(
-                    f"SuiteAggregate '{agg.name}' uses method='passthrough' but no child "
-                    f"named '{agg.name}' exists. Available children: {list(child_results.keys())}."
+                logger.warning(
+                    f"SuiteAggregate '{agg.name}' uses source '{source.child}' which is not a child of the suite. "
+                    f"Available children: {list(child_results.keys())}."
                 )
-            # don't count NaN and None values.
-            result[agg.name] = next(
-                (v for m in agg.metric if (v := child.aggregates.get(m)) is not None and not math.isnan(v)),
-                None,
-            )
-        else:
-            values: dict[str, float] = {}
-            for child_name, child in child_results.items():
-                for m in agg.metric:
-                    val = child.aggregates.get(m)
-                    if val is not None and not math.isnan(val):
-                        values[child_name] = val
-                        break
-            result[agg.name] = _apply_method(agg.method, values) if values else None
+                continue
+            val = child.aggregates.get(source.metric)
+            if val is not None and not math.isnan(val):
+                values.append(val)
+            else:
+                logger.warning(f"The value for source '{source.child}' with metric '{source.metric}' is None or NaN.")
+        result[agg.name] = _apply_method(agg.method, values) if values else None
 
     return result
 
 
 def _apply_method(
     method: str | Callable[[list[float]], float],
-    values: dict[str, float],
+    values: list[float],
 ) -> float:
-    vals = list(values.values())
-
     if callable(method):
-        return method(vals)
+        return method(values)
     elif method == "mean":
-        return float(np.mean(vals))
+        return float(np.mean(values))
     elif method == "median":
-        return float(np.median(vals))
+        return float(np.median(values))
     else:
         raise ValueError(f"Unknown aggregation method: '{method}'. Use mean or median.")
 

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/base.py

@@ -34,6 +34,7 @@ class ResponseType(Enum):
 class TaskStyle(Enum):
     MULTIPLE_CHOICE = "multiple_choice"
     CLOZE = "cloze"
+    BPB = "bpb"
 
 
 class Language(Enum):
@@ -311,7 +312,7 @@ class BaseTask[SubjectType](ABC):
 
     def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
         if hasattr(self, "TASK_STYLER"):
-            return self.TASK_STYLER.get_possible_completions(self._get_choices(item))
+            return self.TASK_STYLER.get_possible_completions(self._get_choices(item), self._get_correct_index(item))
         return None
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
@@ -331,12 +332,7 @@ class BaseTask[SubjectType](ABC):
         return None
 
     def get_metadata(self) -> dict[str, str | list[str]]:
-        if hasattr(self, "TASK_STYLER"):
-            response_type = self.TASK_STYLER.response_type
-            metrics = self.TASK_STYLER.metrics
-        else:
-            response_type = self.RESPONSE_TYPE
-            metrics = self.METRICS
+        response_type, metrics = self._get_type_and_metrics()
 
         meta: dict[str, str | list[str]] = {
             "dataset_path": self.DATASET_PATH,
@@ -423,3 +419,8 @@ class BaseTask[SubjectType](ABC):
                 )
             )
         return completion_list
+
+    def _get_type_and_metrics(self) -> tuple[ResponseType, list[type["BaseMetric"]]]:
+        if hasattr(self, "TASK_STYLER"):
+            return self.TASK_STYLER.response_type, self.TASK_STYLER.metrics
+        return self.RESPONSE_TYPE, self.METRICS

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/benchmarks/global_mmlu.py

@@ -477,7 +477,7 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
     SUBJECTS = list(product(GLOBAL_MMLU_LANGUAGES, MMLU_SUBJECTS))
     PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"] + get_n_letters(4)
-    LANGUAGE = {
+    LANGUAGE: Language | dict[str, Language] | None = {
         str((lang_code.split("_")[0], subject)): LANGUAGE_NAME_MAP[lang_code]
         for lang_code, subjects in LANGUAGE_SUBJECTS_MAP.items()
         for subject in subjects
@@ -531,3 +531,9 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
 
     def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None:
         return [f" {key}" for key in self.keys]
+
+
+class GlobalMMLU_German(GlobalMMLU):
+    NAME = "GlobalMMLU_German"
+    SUBJECTS = [("de", subject) for subject in MMLU_SUBJECTS]
+    LANGUAGE = Language.DEU

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/benchmarks/mbpp.py

@@ -276,7 +276,8 @@ class MBPP_OLMES(MBPP):
 
     def __init__(self, num_fewshot: int = 3) -> None:
         super().__init__(num_fewshot)
-        assert num_fewshot == 3, "MBPP_OLMES requires exactly 3 fewshot examples"
+        if num_fewshot != 3:
+            logger.warning(f"MBPP_OLMES supports only 3-shot, got {num_fewshot}")
         self.stop_sequences = ["```", '\n"""', "\nassert", "\n#"]
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/benchmarks/mmmlu.py

@@ -426,7 +426,7 @@ class MMMLU(BaseTask[tuple[str, str]]):
     METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
     SUBJECTS = list(product(MMMLU_LANGS, MMLU_SUBJECTS))
     PERTURBATION_UNMODIFIABLE_WORDS = ["Question"] + get_n_letters(4)
-    LANGUAGE = {
+    LANGUAGE: Language | dict[str, Language] | None = {
         str((lang_code.split("_")[0], subject)): LANGUAGE_NAME_MAP[lang_code]
         for lang_code, subjects in LANGUAGE_SUBJECTS_MAP.items()
         for subject in subjects
@@ -480,6 +480,12 @@ class MMMLU(BaseTask[tuple[str, str]]):
         return [f" {key}" for key in self.keys]
 
 
+class MMMLU_German(MMMLU):
+    NAME = "MMMLU_German"
+    SUBJECTS = [("DE_DE", subject) for subject in MMLU_SUBJECTS]
+    LANGUAGE = Language.DEU
+
+
 class MMMLU_GERMAN_COT(MMMLU):
     NAME = "MMMLU_GERMAN_COT"
     RESPONSE_TYPE = ResponseType.COMPLETION

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/benchmarks/naturalqs_open.py

@@ -3,6 +3,7 @@ from typing import Any
 from eval_framework.metrics.completion.drop_completion import DropF1ExactMatch, DropMetricContext
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
 from eval_framework.tasks.task_style import (
+    BPBStyle,
     ClozeStyle,
     MCStyle,
     answer_key_to_index,
@@ -93,3 +94,10 @@ class NaturalQsOpenMC_OLMES(_NaturalQsOpenChoice_Base):
 
     NAME = "NaturalQsOpenMC_OLMES"
     TASK_STYLER = MCStyle(space_prefixed_labels=True)
+
+
+class NaturalQsOpenBPB(_NaturalQsOpenChoice_Base):
+    """BPB-only variant."""
+
+    NAME = "NaturalQsOpenBPB"
+    TASK_STYLER = BPBStyle()

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/eval_config.py

@@ -112,7 +112,8 @@ class EvalConfig(BaseConfig):
     @model_validator(mode="after")
     def validate_llm_judge_defined(self) -> "EvalConfig":
         task = get_task(self.task_name)
-        for metric_class in task.METRICS:
+        _, task_metrics = task(num_fewshot=0)._get_type_and_metrics()
+        for metric_class in task_metrics:
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 assert self.llm_judge_class is not None, "The LLM Judge must be defined for this evaluation task."
         return self

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/task_names.py

@@ -119,8 +119,10 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu_pro.MMLU_PRO_COT")
     register_lazy_task("eval_framework.tasks.benchmarks.mmlu.MMLU_COT")
     register_lazy_task("eval_framework.tasks.benchmarks.mmmlu.MMMLU")
+    register_lazy_task("eval_framework.tasks.benchmarks.mmmlu.MMMLU_German")
     register_lazy_task("eval_framework.tasks.benchmarks.mmmlu.MMMLU_GERMAN_COT")
     register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU")
+    register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German")
     register_lazy_task("eval_framework.tasks.benchmarks.pawsx.PAWSX")
     register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA")
     register_lazy_task("eval_framework.tasks.benchmarks.piqa.PIQA_IDK")
@@ -190,6 +192,7 @@ def register_all_tasks() -> None:
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenCloze")
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC")
     register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenMC_OLMES")
+    register_lazy_task("eval_framework.tasks.benchmarks.naturalqs_open.NaturalQsOpenBPB")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQACloze")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC_OLMES")
     register_lazy_task("eval_framework.tasks.benchmarks.social_iqa.SocialIQAMC")

{eval_framework-0.3.0 → eval_framework-0.3.2}/src/eval_framework/tasks/task_style.py

@@ -52,6 +52,10 @@ dataset attributes and data-access methods.  Variants only differ in ``TASK_STYL
     class ARC_MC(_ARC_Base):
         NAME = "ARC_MC"
         TASK_STYLER = MCStyle(space_prefixed_labels=True)
+
+    class ARC_BPB(_ARC_Base):
+        NAME = "ARC_BPB"
+        TASK_STYLER = BPBStyle()
 """
 
 import hashlib
@@ -111,8 +115,13 @@ class TaskStyler(ABC):
         """Return the ground-truth string for scoring."""
 
     @abstractmethod
-    def get_possible_completions(self, choices: list[str]) -> list[str]:
-        """Return the list of scored completion strings."""
+    def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]:
+        """Return the list of completion strings to be evaluated.
+
+        ``correct_index`` is only required by ``BPBStyle``, which scores solely the
+        ground-truth completion. ``MCStyle`` and ``ClozeStyle`` score all choices and
+        ignore it; callers may omit it when using those stylers.
+        """
 
     @abstractmethod
     def get_cue_text(self) -> str:
@@ -196,7 +205,8 @@ class MCStyle(TaskStyler):
         labels = get_n_letters(len(choices))
         return f" {labels[correct_index]}"
 
-    def get_possible_completions(self, choices: list[str]) -> list[str]:
+    def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]:
+        """Note: `correct_index` is ignored for `MCStyle` and only used for `BPBStyle`."""
         return [f" {label}" for label in get_n_letters(len(choices))]
 
 
@@ -241,10 +251,12 @@ class ClozeStyle(TaskStyler):
         question_prefix: str = "Question: ",
         cue_text: str = "Answer:",
         trailing_newline: bool = True,
+        leading_space_continuations: bool = True,
     ) -> None:
         self.question_prefix = question_prefix
         self._cue_text = cue_text
         self.trailing_newline = trailing_newline
+        self.leading_space_continuations = leading_space_continuations
 
     def get_cue_text(self) -> str:
         return self._cue_text
@@ -254,10 +266,41 @@ class ClozeStyle(TaskStyler):
         return f"{text}\n" if self.trailing_newline else text
 
     def get_ground_truth(self, choices: list[str], correct_index: int) -> str:
-        return f" {choices[correct_index]}"
+        return f" {choices[correct_index]}" if self.leading_space_continuations else choices[correct_index]
+
+    def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]:
+        return [f" {c}" for c in choices] if self.leading_space_continuations else [f"{c}" for c in choices]
+
+
+class BPBStyle(ClozeStyle):
+    """BPB-only styler: prompt identical to ClozeStyle, but scores only the ground-truth completion.
+
+    One LLM forward pass per sample instead of N (one per choice), making evaluation
+    significantly faster when accuracy metrics are not needed.
+
+    Args:
+        question_prefix:   Prepended to the raw question (default ``"Question: "``).
+        cue_text:          Assistant cue after the prompt (default ``"Answer:"``).
+        trailing_newline:  When ``True`` (default), the instruction ends with ``"\\n"``.
+
+    Assembled prompt example (3 choices)::
+
+        "Question: What is the capital of France?\\n"
+
+        Scored completions: [" Paris"]  ← ground truth only, one forward pass
+        Ground truth:        " Paris"
+    """
+
+    metrics: list[type["BaseMetric"]] = [BitsPerByteLoglikelihood]
+    task_style = TaskStyle.BPB
 
-    def get_possible_completions(self, choices: list[str]) -> list[str]:
-        return [f" {c}" for c in choices]
+    def get_possible_completions(self, choices: list[str], correct_index: int | None = None) -> list[str]:
+        if correct_index is None:
+            raise ValueError(
+                "BPBStyle evaluates the loglikelihood of the ground truth answer only,"
+                "and thus requires the correct index."
+            )
+        return [f" {choices[correct_index]}"] if self.leading_space_continuations else [choices[correct_index]]
 
 
 # ---------------------------------------------------------------------------