eval-framework 0.3.2__tar.gz → 0.3.3__tar.gz

{eval_framework-0.3.2 → eval_framework-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.2
+Version: 0.3.3
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.2 → eval_framework-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.2"
+version = "0.3.3"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.2 → eval_framework-0.3.3}/src/eval_framework/metrics/completion/drop_completion.py

@@ -18,7 +18,7 @@ class DropMetricContext(BaseMetricContext):
 class DropF1ExactMatch(BaseMetric[Completion]):
     """DROP F1 and exact match. Requires DropMetricContext with answer_tuples."""
 
-    NAME = "DROP F1 / Exact Match"
+    NAME = "Drop F1"
     KEYS = ["f1", "exact_match"]
 
     def calculate(self, response: Completion) -> list[MetricResult]:
@@ -52,5 +52,5 @@ class DropF1ExactMatch(BaseMetric[Completion]):
                 higher_is_better=True,
                 error=response.error,
             )
-            for name, key in zip([n.strip() for n in self.NAME.split("/")], self.KEYS)
+            for name, key in zip(self.NAMES, self.KEYS)
         ]

{eval_framework-0.3.2 → eval_framework-0.3.3}/src/eval_framework/metrics/completion/math_minerva_completion.py

@@ -20,6 +20,7 @@ class MathMinervaCompletion(BaseMetric[Completion]):
     """
 
     NAME = "Math Minerva Completion"
+    KEYS = ["Exact", "Exact Flex"]
     AGGREGATORS = [PassAtK()]
 
     def __init__(
@@ -36,17 +37,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         if response.error:
             return [
                 MetricResult(
-                    metric_name="Exact Match",
+                    metric_name=x,
                     value=None,
                     higher_is_better=True,
                     error=response.error,
-                ),
-                MetricResult(
-                    metric_name="Exact Match (Flex)",
-                    value=None,
-                    higher_is_better=True,
-                    error=response.error,
-                ),
+                )
+                for x in self.NAMES
             ]
 
         gold = response.ground_truth
@@ -55,17 +51,12 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         if not gold:
             return [
                 MetricResult(
-                    metric_name="Exact Match",
-                    value=None,
-                    higher_is_better=True,
-                    error="No ground truth available",
-                ),
-                MetricResult(
-                    metric_name="Exact Match (Flex)",
+                    metric_name=x,
                     value=None,
                     higher_is_better=True,
                     error="No ground truth available",
-                ),
+                )
+                for x in self.NAMES
             ]
 
         raw = response.raw_completion or response.completion
@@ -84,12 +75,8 @@ class MathMinervaCompletion(BaseMetric[Completion]):
         )
 
         return [
-            MetricResult(metric_name="Exact Match", value=exact_match, higher_is_better=True),
-            MetricResult(
-                metric_name="Exact Match (Flex)",
-                value=exact_match_flex,
-                higher_is_better=True,
-            ),
+            MetricResult(metric_name=name, value=value, higher_is_better=True)
+            for name, value in zip(self.NAMES, [exact_match, exact_match_flex])
         ]
 
 

{eval_framework-0.3.2 → eval_framework-0.3.3}/src/eval_framework/tasks/benchmarks/truthfulqa.py

@@ -106,13 +106,18 @@ class TRUTHFULQA_OLMES(TRUTHFULQA):
     """
 
     NAME = "TruthfulQA_OLMES"
+    FEWSHOT_SPLIT = "validation"  # use dataset few-shot for multiple-choice options
+    PERTURBATION_UNMODIFIABLE_WORDS = ["Question", "Answer"]
+
+    def _get_cue_text(self, item: dict[str, Any]) -> str:
+        return "Answer:"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         question = item["question"]
         choices = item[self.target_identifier]["choices"]
         labels = get_n_letters(len(choices))
         options = "\n".join(f" {label}. {choice}" for label, choice in zip(labels, choices))
-        return f"Q: {question}\n{options}\n"
+        return f"Question: {question}\n{options}\n"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str | None | list[str]:
         labels_arr = item[self.target_identifier]["labels"]
@@ -126,7 +131,17 @@ class TRUTHFULQA_OLMES(TRUTHFULQA):
         return [f" {letter}" for letter in letters]
 
     def _sample_fewshot_examples(self, item: dict[str, Any]) -> list[dict]:
-        return self.FEWSHOT_ITEMS[: self.num_fewshot]
+        # Reuse BaseTask's split-based sampler.
+        return BaseTask._sample_fewshot_examples(self, item)
+
+    def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
+        cue_text = self._get_cue_text(item)
+        labels_arr = item[self.target_identifier]["labels"]
+        letters = get_n_letters(len(labels_arr))
+        # Pick one correct option for the demonstration answer.
+        correct_letters = [letters[i] for i, label in enumerate(labels_arr) if label == 1]
+        letter = correct_letters[0] if correct_letters else letters[0]
+        return f"{cue_text} {letter}"
 
 
 class TRUTHFULQA_IDK(TRUTHFULQA):