eval-framework 0.2.7__py3-none-any.whl

eval_framework/metrics/completion/csv_format.py

@@ -0,0 +1,102 @@
+import json
+
+from pydantic import BaseModel
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+SEPARATOR_MAP = {"comma": ",", "semicolon": ";", "space": " ", "tab": "\t"}
+
+
+class CSVFormatEvaluation(BaseModel):
+    implicit: bool = False
+    has_csv: bool = False
+    is_separator_respected: bool = False
+    is_column_count_respected: bool = False
+
+
+class CSVFormat(BaseMetric[Completion]):
+    NAME = "CSV Format"
+    KEYS = ["has_csv", "is_separator_respected", "is_column_count_respected"]
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=None, higher_is_better=True, error=response.error)
+                for k in self.KEYS
+            ]
+
+        if response.completion == "":
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=0.0, higher_is_better=True, error=response.error)
+                for k in self.KEYS
+            ]
+
+        grading = evaluate_csv_format(response)
+
+        results = []
+        for key in self.KEYS:
+            result = MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=float(getattr(grading, key)),
+                higher_is_better=True,
+                error=response.error,
+            )
+            results.append(result)
+        return results
+
+
+def extract_csv_from_text(text: str, min_rows: int = 2, min_columns: int = 2) -> tuple[list[str] | None, str | None]:
+    lines = text.split("\n")
+    delimiters = set(SEPARATOR_MAP.values())
+    best_delimiter = None
+    csv_lines: list[str] = []
+
+    # Iterate over lines to find potential delimiters and consistent substring counts
+    for i, line in enumerate(lines):
+        for delimiter in delimiters:
+            substrings = line.split(delimiter)
+            if len(substrings) < min_columns:
+                continue
+
+            current_csv_lines = [line]
+            for j in range(i + 1, len(lines)):
+                next_line = lines[j]
+                next_substrings = next_line.split(delimiter)
+                if len(next_substrings) != len(substrings):
+                    break
+                current_csv_lines.append(next_line)
+            if len(current_csv_lines) >= min_rows and len(current_csv_lines) > len(csv_lines):
+                best_delimiter = delimiter
+                csv_lines = current_csv_lines
+
+    if not csv_lines:
+        return None, None
+
+    return csv_lines, best_delimiter
+
+
+def evaluate_csv_format(response: Completion) -> CSVFormatEvaluation:
+    expected_output = json.loads(str(response.ground_truth))
+
+    expected_separator_code = expected_output["separator"]
+    csv_lines, separator = extract_csv_from_text(response.completion)
+
+    if not csv_lines:
+        return CSVFormatEvaluation(has_csv=False, implicit=not expected_separator_code)
+
+    csv_format_evaluation = CSVFormatEvaluation(has_csv=True, implicit=not expected_separator_code)
+
+    if not expected_separator_code:
+        csv_format_evaluation.is_separator_respected = separator in SEPARATOR_MAP.values()
+    else:
+        csv_format_evaluation.is_separator_respected = separator == SEPARATOR_MAP.get(expected_separator_code)
+
+    expected_column_count = len(expected_output["columns"])
+    column_counts = [len(csv_lines.split(separator)) for csv_lines in csv_lines]
+
+    csv_format_evaluation.is_column_count_respected = all(
+        column_count == expected_column_count for column_count in column_counts
+    )
+
+    return csv_format_evaluation

eval_framework/metrics/completion/cwe_accuracy.py

@@ -0,0 +1,49 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+
+
+class CWEAccuracy(BaseMetric[Completion]):
+    """Metric for Common Word Extraction tasks"""
+
+    NAME = "CWEAccuracy"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        try:
+            # Get model's answer
+            model_answer = response.completion
+
+            # Check if all words in the correct answer are present in the model's answer
+            is_correct = self._is_answer_correct(ground_truths, model_answer)
+
+            return [
+                MetricResult(
+                    metric_name=self.NAME, value=1.0 if is_correct else 0.0, higher_is_better=True, error=response.error
+                )
+            ]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+    def _is_answer_correct(self, correct_answer: list[str], model_answer: str) -> bool:
+        """Check if all words in correct_answer are present in model_answer as whole words"""
+        model_answer = model_answer.strip().lower()
+        correct_answer = [correct.strip().lower() for correct in correct_answer]
+
+        # For each word in the correct answer, check if it exists as a whole word in the model answer
+        for word in correct_answer:
+            # Create a regex pattern that matches the word as a whole word
+            # \b represents a word boundary
+            pattern = r"\b" + re.escape(word) + r"\b"
+            if not re.search(pattern, model_answer):
+                return False
+
+        return True

eval_framework/metrics/completion/exponential_similarity.py

@@ -0,0 +1,65 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Error
+
+
+class ExponentialSimilarity(BaseMetric[Completion]):
+    NAME = "ExponentialSimilarity"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        try:
+            # Try to calculate exponential similarity for each ground truth
+            similarities = []
+            for gt in ground_truths:
+                try:
+                    gt_float = float(gt)
+                    completion_float = float(response.completion)
+                    similarities.append(calculate_exponential_similarity(gt_float, completion_float))
+                except (ValueError, TypeError):
+                    # Skip this ground truth if conversion fails
+                    continue
+
+            # If we have any valid similarities, return the max
+            if similarities:
+                return [
+                    MetricResult(
+                        metric_name=self.NAME, value=max(similarities), higher_is_better=True, error=response.error
+                    )
+                ]
+            else:
+                # If all conversions failed, return an error
+                error = Error(
+                    error_class="ValueError",
+                    message="Could not convert ground truth or completion to float",
+                    traceback="",
+                )
+                return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+        except Exception as e:
+            error = Error(error_class=e.__class__.__name__, message=str(e), traceback="")
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=error)]
+
+
+def calculate_exponential_similarity(p_true: float, p_pred: float) -> float:
+    """
+    Compute the exponential similarity (SpaceDigest version) between
+    the gold percentage and predicted value.
+
+    Parameters:
+    - p_true (float): The gold/reference percentage.
+    - p_pred (float): The predicted scalar.
+    - d (float): Base of the exponent. Default is 2.
+    - c (float): Coefficient in exponent. Default is 10.
+
+    Returns:
+    - float: Similarity score between 0 and 1.
+    """
+    d = 2
+    c = 10
+
+    return d ** (-c * abs(p_true / 100 - p_pred / 100))

eval_framework/metrics/completion/f1.py

@@ -0,0 +1,42 @@
+from collections import Counter
+from typing import Any
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class F1(BaseMetric[Completion]):
+    NAME = "F1"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truths = [gt for gt in response.ground_truth_list if gt is not None]
+        if not ground_truths:
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+
+        hyp_tokens = response.completion.lower().split()
+        f1_scores = [calculate_f1(gt.lower().split(), hyp_tokens) for gt in ground_truths]
+        max_f1 = max(f1_scores)
+
+        return [MetricResult(metric_name=self.NAME, value=max_f1, higher_is_better=True, error=response.error)]
+
+
+def calculate_f1(ref_tokens: list[Any], hyp_tokens: list[Any]) -> float:
+    """Calculate F1 score between two texts based on token overlap."""
+    if not ref_tokens and not hyp_tokens:
+        return 1.0
+    if not ref_tokens or not hyp_tokens:
+        return 0.0
+
+    common = Counter(ref_tokens) & Counter(hyp_tokens)
+    num_same = sum(common.values())
+
+    if num_same == 0:
+        return 0.0
+
+    precision = num_same / len(hyp_tokens)
+    recall = num_same / len(ref_tokens)
+
+    return 2 * precision * recall / (precision + recall)

eval_framework/metrics/completion/format_checker.py

@@ -0,0 +1,56 @@
+import json
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class CheckJsonFormat(BaseMetric[Completion]):
+    NAME = "JSON Format"
+
+    def _preprocess(self, completion: str) -> str:
+        completion = completion.strip()
+        for prefix in ["```json", "```Json", "```JSON", "```"]:
+            completion = completion.removeprefix(prefix)
+        completion = completion.removesuffix("```")
+        completion = completion.strip()
+        return completion
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        json_text = self._preprocess(response.completion)
+
+        try:
+            json.loads(json_text)
+            is_valid_json = True
+        except ValueError as _:
+            is_valid_json = False
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(is_valid_json), higher_is_better=True, error=response.error)
+        ]
+
+
+class CheckPostScriptFormat(BaseMetric[Completion]):
+    """
+    This metric is honestly not that great
+    In the original IFEval implementation it just checks whether the
+    text contains the string (P.)P.S. or variants thereof such as p. s.
+    It doesn't check for parsing
+    """
+
+    NAME = "Postscript Format"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        postscript_pattern = r"\s*(P\.S\.|P\.P\.S\.)"
+        postscript = re.findall(postscript_pattern, response.completion, flags=re.MULTILINE)
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=1.0 if postscript else 0.0, higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/grid_difference.py

@@ -0,0 +1,77 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class GridDifference(BaseMetric[Completion]):
+    NAME = "grid_difference"
+
+    def count_differences(self, character_list_1: list[str], character_list_2: list[str]) -> int:
+        count = 0
+        for character_1, character_2 in zip(character_list_1, character_list_2):
+            if character_1 != character_2:
+                count += 1
+        return count
+
+    def calculate_score(
+        self, output_ground_truth_difference_count: int, input_ground_truth_difference_count: int
+    ) -> float:
+        if output_ground_truth_difference_count == 0 and input_ground_truth_difference_count == 0:
+            return 1.0
+        score = 1.0 - (float(output_ground_truth_difference_count) / float(input_ground_truth_difference_count))
+        return score
+
+    def extract_grid_from_prompt(self, prompt: str) -> str:
+        # Extract grid between known markers
+        start_marker = "Below is the input grid with masked regions:"
+        end_marker = "Please output the completed grid"
+
+        # Use regex with DOTALL flag to match across newlines
+        match = re.search(re.escape(start_marker) + r"(.*?)" + re.escape(end_marker), prompt, re.DOTALL)
+
+        if match:
+            grid = match.group(1).strip()
+            return grid
+
+        return ""
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        input_grid = self.extract_grid_from_prompt(prompt=response.last_user_instruction).split()
+        output_grid = response.completion.split()
+
+        assert response.ground_truth_list[0], "Ground truth list is empty or not provided in the response."
+        ground_truth_grid = response.ground_truth_list[0].split()
+
+        input_ground_truth_differences_count = self.count_differences(input_grid, ground_truth_grid)
+        output_ground_truth_differences_count = self.count_differences(output_grid, ground_truth_grid)
+
+        exact_match = True
+        score = 1.0
+        normalized_score = 1.0
+        if output_ground_truth_differences_count != 0:
+            exact_match = False
+            score = self.calculate_score(
+                output_ground_truth_differences_count,
+                input_ground_truth_differences_count,
+            )
+            normalized_score = max(score, 0.0)
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}_exact_match",
+                value=float(exact_match),
+                higher_is_better=True,
+                error=response.error,
+            ),
+            MetricResult(metric_name=f"{self.NAME}_score", value=score, higher_is_better=True, error=response.error),
+            MetricResult(
+                metric_name=f"{self.NAME}_normalized_score",
+                value=normalized_score,
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]

eval_framework/metrics/completion/ifeval.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+from eval_framework.external.ifeval_impl.utils import process_results
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class IFEvalMetricContext(BaseMetricContext):
+    key: int
+    instruction_id_list: list[str]
+    prompt: str
+    additional_kwargs: list[dict[str, Any]]
+
+
+class IFEvalMetric(BaseMetric[Completion]):
+    NAME = "IFEval"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        context = extract_context_metric(response, IFEvalMetricContext)
+
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=f"{self.NAME}/prompt_level_strict_acc",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error,
+                ),
+                MetricResult(
+                    metric_name=f"{self.NAME}/prompt_level_loose_acc",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error,
+                ),
+            ]
+
+        grading = process_results(context, [response.completion])
+
+        results = [
+            MetricResult(
+                metric_name=f"{self.NAME}/prompt_level_strict_acc",
+                value=float(grading["prompt_level_strict_acc"]),
+                higher_is_better=True,
+                error=response.error,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/prompt_level_loose_acc",
+                value=float(grading["prompt_level_loose_acc"]),
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]
+        # this framework does not support a custom aggregation step (see agg_inst_level_acc()) so work around
+        # by returning the result for each instruction as a separate MetricResult
+        results += [
+            MetricResult(
+                metric_name=f"{self.NAME}/inst_level_strict_acc",
+                value=float(v),
+                higher_is_better=True,
+                error=response.error,
+            )
+            for v in grading["inst_level_strict_acc"]
+        ]
+        results += [
+            MetricResult(
+                metric_name=f"{self.NAME}/inst_level_loose_acc",
+                value=float(v),
+                higher_is_better=True,
+                error=response.error,
+            )
+            for v in grading["inst_level_loose_acc"]
+        ]
+        return results

eval_framework/metrics/completion/json_format.py

@@ -0,0 +1,179 @@
+import json
+from collections.abc import Mapping
+from typing import Any
+
+import jsonschema  # type: ignore
+from pydantic import BaseModel
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class JsonFormatEvaluation(BaseModel):
+    is_just_json: bool = False
+    is_valid_json: bool = False
+    fulfills_schema: bool | None = None
+    exact_match: bool | None = None
+    json_parsing_error: str | None = None
+    schema_validation_error: str | None = None
+
+
+class JsonFormat(BaseMetric[Completion]):
+    NAME = "JSON Format"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        keys = [
+            "is_just_json",
+            "is_valid_json",
+            "fulfills_schema",
+            "exact_match",
+        ]
+
+        if response.error is not None:
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=None, higher_is_better=True, error=response.error)
+                for k in keys
+            ]
+
+        if response.completion == "":
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=0.0, higher_is_better=True, error=response.error)
+                for k in keys
+            ]
+
+        json_dict, grading = self._extract_and_parse_json(response.completion)
+
+        ground_truth_dict = json.loads(str(response.ground_truth))
+        schema = ground_truth_dict["json_schema"]
+        expected_object = ground_truth_dict.get("expected_output", None)
+
+        if schema and json_dict is None:
+            grading.fulfills_schema = False
+        if schema and json_dict is not None:
+            grading = self._validate_json_against_schema(json_dict, schema, grading)
+        if expected_object is not None and json_dict is not None:
+            grading.exact_match = json_dict == expected_object
+
+        results = []
+        for key in keys:
+            result = MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=float(getattr(grading, key)) if getattr(grading, key) is not None else None,
+                higher_is_better=True,
+                error=response.error,
+                code_execution_trace=(grading.json_parsing_error or "") + (grading.schema_validation_error or ""),
+            )
+            results.append(result)
+        return results
+
+    @staticmethod
+    def _validate_json_against_schema(
+        json_obj: object, schema: Mapping[str, Any], evaluation_result: JsonFormatEvaluation
+    ) -> JsonFormatEvaluation:
+        evaluation_result = evaluation_result.model_copy(deep=True)
+        try:
+            jsonschema.validate(json_obj, schema)
+            evaluation_result.fulfills_schema = True
+        except jsonschema.exceptions.ValidationError as e:
+            evaluation_result.fulfills_schema = False
+            evaluation_result.schema_validation_error = type(e).__name__
+        except jsonschema.exceptions.SchemaError as e:
+            evaluation_result.schema_validation_error = type(e).__name__
+        return evaluation_result
+
+    @staticmethod
+    def _extract_and_parse_json(completion: str) -> tuple[object, JsonFormatEvaluation]:
+        evaluation_result = JsonFormatEvaluation()
+        json_dict = None
+        try:
+            json_dict = json.loads(remove_comments(completion.strip("`")))
+            evaluation_result.is_just_json = True
+            evaluation_result.is_valid_json = True
+        except Exception as _:
+            try:
+                json_string = remove_comments(get_json_object(completion))
+                json_dict = json.loads(json_string)
+                evaluation_result.is_valid_json = True
+            except Exception as e:
+                evaluation_result.json_parsing_error = type(e).__name__
+        return json_dict, evaluation_result
+
+
+def get_json_object(text: str) -> str:
+    """
+    Extract the first valid JSON object or array from text.
+
+    This function handles nested brackets properly by using a bracket counting
+    approach to find complete JSON structures, rather than using regex which
+    can incorrectly match outer brackets containing non-JSON content.
+    """
+
+    def find_json_at_position(text: str, start_pos: int, open_char: str, close_char: str) -> str | None:
+        """Find a complete JSON object/array starting at the given position."""
+        if start_pos >= len(text) or text[start_pos] != open_char:
+            return None
+
+        bracket_count = 0
+        in_string = False
+        escaped = False
+
+        for i in range(start_pos, len(text)):
+            char = text[i]
+
+            if escaped:
+                escaped = False
+                continue
+
+            if char == "\\" and in_string:
+                escaped = True
+                continue
+
+            if char == '"' and not escaped:
+                in_string = not in_string
+                continue
+
+            if not in_string:
+                if char == open_char:
+                    bracket_count += 1
+                elif char == close_char:
+                    bracket_count -= 1
+                    if bracket_count == 0:
+                        # Found complete JSON structure
+                        candidate = text[start_pos : i + 1]
+                        # Test if it's valid JSON
+                        try:
+                            json.loads(candidate)
+                            return candidate
+                        except json.JSONDecodeError:
+                            return None
+
+        return None
+
+    # Look for JSON objects {} and arrays []
+    json_candidates = []
+
+    # Search for objects starting with {
+    for i in range(len(text)):
+        if text[i] == "{":
+            candidate = find_json_at_position(text, i, "{", "}")
+            if candidate:
+                json_candidates.append(candidate)
+
+    # Search for arrays starting with [
+    for i in range(len(text)):
+        if text[i] == "[":
+            candidate = find_json_at_position(text, i, "[", "]")
+            if candidate:
+                json_candidates.append(candidate)
+
+    if not json_candidates:
+        raise RuntimeError(f"No valid JSON object found in {text}.")
+
+    # Return the longest valid JSON (most likely to be the main content)
+    return max(json_candidates, key=len)
+
+
+def remove_comments(text: str, comment_indicator: str = "//") -> str:
+    lines = text.splitlines()
+    lines = [line.split(comment_indicator)[0] for line in lines]
+    return "\n".join([line for line in lines if line.strip()])