eval-framework 0.5.2__tar.gz → 0.5.3__tar.gz

{eval_framework-0.5.2 → eval_framework-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.5.2
+Version: 0.5.3
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.5.2 → eval_framework-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.5.2"
+version = "0.5.3"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -100,7 +100,7 @@ dev = [
   "types-python-dateutil>=2.9.0.20260518,<3",
   "types-requests>=2.33.0.20260518,<3",
   "plotly>=6.8.0,<7",
-  "ruff>=0.15.18",
+  "ruff>=0.15.20",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -109,7 +109,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.23,<0.11.24"]
+requires = ["uv_build>=0.11.24,<0.11.25"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.5.2 → eval_framework-0.5.3}/src/eval_framework/tasks/dataset_revisions.py

@@ -4,7 +4,7 @@ Overwrites ``task-dataset-revisions.json`` in this package with task class name
 
 Usage::
 
-    uv run python -m eval_framework.tasks.benchmarks.dataset_revisions
+    uv run python -m eval_framework.tasks.dataset_revisions
 """
 
 import json

{eval_framework-0.5.2 → eval_framework-0.5.3}/src/eval_framework/tasks/task-dataset-revisions.json

@@ -14,6 +14,7 @@
     "GOLDENSWAG": "f98259e4185c9ab868cf695ac98e9d863eba7986",
     "GOLDENSWAG_IDK": "f98259e4185c9ab868cf695ac98e9d863eba7986",
     "GPQA_OLMES": "633f5ee89ab8ad4522a9f850766b73f62147ffdd",
+    "GSM8KBPB": "740312add88f781978c0658806c59bc2815b9866",
     "GSM8KReasoning": "740312add88f781978c0658806c59bc2815b9866",
     "GSM8K_OLMES": "740312add88f781978c0658806c59bc2815b9866",
     "GlobalMMLU": "0e619dbeb34206cd48705a1a0ea7fb21cae09993",
@@ -25,6 +26,7 @@
     "IFEval": "966cd89545d6b6acfd7638bc708b98261ca58e84",
     "IFEvalDe": "4f52d847003b3c83cc282e9d296853a24b34b19a",
     "MATH500": "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be",
+    "MATHMinervaBPB": "21a5633873b6a120296cce3e2df9d5550074f4a3",
     "MATHMinerva_OLMES": "21a5633873b6a120296cce3e2df9d5550074f4a3",
     "MBPPBPB": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
     "MBPP_OLMES": "4bb6404fdc6cacfda99d4ac4205087b89d32030c",
@@ -55,8 +57,10 @@
     "PIQA_IDK": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0",
     "PIQA_OLMES": "2e8ac2dffd59bac8c3c6714948f4c551a0848bb0",
     "SCIQ_OLMES": "2c94ad3e1aafab77146f384e23536f97a4849815",
+    "SQuAD2_MA": "3ffb306f725f7d2ce8394bc1873b24868140c412",
     "SQuAD_OLMES": "7b6d24c440a36b6815f21b70d25016731768db1f",
     "SocialIQAMC_OLMES": "8835ceb9141d7896d9d968634a9b21ae440e3ec5",
     "TRIVIAQA": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
+    "TriviaQA_MA": "0f7faf33a3908546c6fd5b73a660e0f8ff173c2f",
     "WINOGRANDECloze": "01e74176c63542e6b0bcb004dcdea22d94fb67b5"
 }

eval_framework-0.5.2/src/eval_framework/metrics/completion/cwe_accuracy.py

@@ -1,49 +0,0 @@
-import re
-
-from eval_framework.metrics.base import BaseMetric, MetricResult
-from eval_framework.shared.types import Completion, Error
-
-
-class CWEAccuracy(BaseMetric[Completion]):
-    """Metric for Common Word Extraction tasks"""
-
-    NAME = "CWEAccuracy"
-
-    def calculate(self, response: Completion) -> list[MetricResult]:
-        if response.error is not None:
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
-
-        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
-        if not ground_truths:
-            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
-
-        try:
-            # Get model's answer
-            model_answer = response.completion
-
-            # Check if all words in the correct answer are present in the model's answer
-            is_correct = self._is_answer_correct(ground_truths, model_answer)
-
-            return [
-                MetricResult(
-                    metric_name=self.NAME, value=1.0 if is_correct else 0.0, higher_is_better=True, error=response.error
-                )
-            ]
-        except Exception as e:
-            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
-
-    def _is_answer_correct(self, correct_answer: list[str], model_answer: str) -> bool:
-        """Check if all words in correct_answer are present in model_answer as whole words"""
-        model_answer = model_answer.strip().lower()
-        correct_answer = [correct.strip().lower() for correct in correct_answer]
-
-        # For each word in the correct answer, check if it exists as a whole word in the model answer
-        for word in correct_answer:
-            # Create a regex pattern that matches the word as a whole word
-            # \b represents a word boundary
-            pattern = r"\b" + re.escape(word) + r"\b"
-            if not re.search(pattern, model_answer):
-                return False
-
-        return True

eval_framework-0.5.2/src/eval_framework/metrics/completion/niah_accuracy.py

@@ -1,163 +0,0 @@
-import re
-import unicodedata
-
-from eval_framework.metrics.base import (
-    BaseMetric,
-    MetricResult,
-)
-from eval_framework.shared.types import Completion, Error, LanguageMetricContext, extract_context_metric
-
-# Dictionary of "none" words in different languages
-NONE_DICT = {
-    "en": ["none"],
-    "ko": ["없음"],
-    "pl": ["brak"],
-    "zh": ["无"],
-    "vi": ["Không có"],
-    "ja": ["なし", "数字はありません"],
-    "ta": ["ஏதுமில்லை"],
-    "hu": ["nincs"],
-    "fr": ["aucun"],
-    "no": ["ingen"],
-    "uk": ["немає", "Нема"],
-    "ru": ["нет"],
-    "de": ["Keine vorhanden"],
-    "es": ["ninguno"],
-    "sv": ["inga"],
-    "fi": ["ei mikään"],
-    "cs": ["žádné", "žádná"],
-    "sr": ["nema"],
-    "pt": ["nenhum"],
-    "it": ["nessuno"],
-    "fa": ["هیچ کدام"],
-    "sw": ["hakuna"],
-    "nl": ["geen"],
-    "st": ["ha ho letho"],
-    "hi": ["कोई नहीं"],
-    "da": ["ingen"],
-}
-
-
-def clean_text(text: str) -> str:
-    """Clean text by removing spaces and normalizing"""
-    return text.strip().lower().replace("\u200c", "").replace(" ", "")
-
-
-class NIAHAccuracy(BaseMetric[Completion]):
-    """Metric for Needle in a Haystack tasks"""
-
-    NAME = "NIAHAccuracy"
-
-    def calculate(self, response: Completion) -> list[MetricResult]:
-        if response.error is not None:
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
-
-        context = extract_context_metric(response, LanguageMetricContext)
-
-        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
-
-        try:
-            # Extract task and language from metadata
-            assert response.context is not None
-            language = context.language
-
-            # Get model's answer
-            model_answer = response.completion
-
-            # Determine which comparison function to use based on the task
-            none_values = set(v for values in NONE_DICT.values() for v in values)
-            if ground_truths[0] in none_values:
-                is_correct = self._compare_none(language, model_answer)
-            else:
-                is_correct = self._compare_numbers(language, ground_truths, model_answer)
-
-            return [
-                MetricResult(
-                    metric_name=self.NAME, value=float(is_correct), higher_is_better=True, error=response.error
-                )
-            ]
-        except Exception as e:
-            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
-            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
-
-    def _compare_numbers(self, lang: str, correct_answer: list[str], model_answer: str) -> bool:
-        """Compare numbers for regular NIAH tasks"""
-        if "-" in lang:
-            inst_lang = lang.split("-")[1]
-        else:
-            inst_lang = lang
-
-        if not model_answer:
-            return False
-
-        processed_model_answer = unicodedata.normalize("NFKC", model_answer)
-
-        none_words = NONE_DICT.get(inst_lang, ["none"])
-        # Check if any word in none_words is present in the processed answer; if yes, auto-fail
-        for word in none_words:
-            if word in processed_model_answer or clean_text(word) in processed_model_answer:
-                return False
-
-        # Extract all numeric substrings from the processed answer
-        numeric_strings = re.findall(r"\d+", processed_model_answer)
-
-        # Remove numbers that consist of a single digit
-        numeric_strings = [num for num in numeric_strings if len(num) > 1]
-
-        # Remove duplicates while preserving the original order
-        numeric_strings = list(dict.fromkeys(numeric_strings))
-
-        # If no numerics are found after processing, return False
-        if not numeric_strings:
-            return False
-
-        # Convert the extracted number strings to integers
-        try:
-            extracted_numbers = [int(num) for num in numeric_strings]
-        except Exception:
-            return False
-
-        # Convert correct_answers elements to integers to ensure numeric comparison
-        try:
-            correct_converted = [int(item) for item in correct_answer]
-        except Exception:
-            return False
-
-        # Check that the number of extracted numbers matches the length of correct_answers
-        if len(extracted_numbers) != len(correct_converted):
-            return False
-
-        # Compare the extracted numbers with the correct answers
-        if set(extracted_numbers) == set(correct_converted):
-            return True
-        else:
-            return False
-
-    def _compare_none(self, lang: str, model_answer: str) -> bool:
-        """Compare for NIAH none tasks"""
-        # Lower-case all inputs for consistent, case-insensitive processing
-        if "-" in lang:
-            inst_lang = lang.split("-")[1]
-        else:
-            inst_lang = lang
-
-        processed_model_answer = clean_text(unicodedata.normalize("NFKC", model_answer))
-        none_words = [clean_text(word) for word in NONE_DICT[inst_lang]]
-
-        # Remove single digit numbers from the processed answer
-        processed_model_answer = re.sub(r"\b\d\b", "", processed_model_answer)
-
-        # Extract all multi-digit numeric substrings from the processed answer
-        numeric_strings = re.findall(r"\d\d+", processed_model_answer)
-
-        # If any multi-digit numbers are found, return False
-        if numeric_strings:
-            return False
-
-        # Check if any of the words in none_words are present
-        for word in none_words:
-            if word in processed_model_answer:
-                return True
-
-        # If none of the none_words are found, return False
-        return False