eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/completion/format_checker.py

@@ -0,0 +1,56 @@
+import json
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class CheckJsonFormat(BaseMetric[Completion]):
+    NAME = "JSON Format"
+
+    def _preprocess(self, completion: str) -> str:
+        completion = completion.strip()
+        for prefix in ["```json", "```Json", "```JSON", "```"]:
+            completion = completion.removeprefix(prefix)
+        completion = completion.removesuffix("```")
+        completion = completion.strip()
+        return completion
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        json_text = self._preprocess(response.completion)
+
+        try:
+            json.loads(json_text)
+            is_valid_json = True
+        except ValueError as _:
+            is_valid_json = False
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(is_valid_json), higher_is_better=True, error=response.error)
+        ]
+
+
+class CheckPostScriptFormat(BaseMetric[Completion]):
+    """
+    This metric is honestly not that great
+    In the original IFEval implementation it just checks whether the
+    text contains the string (P.)P.S. or variants thereof such as p. s.
+    It doesn't check for parsing
+    """
+
+    NAME = "Postscript Format"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        postscript_pattern = r"\s*(P\.S\.|P\.P\.S\.)"
+        postscript = re.findall(postscript_pattern, response.completion, flags=re.MULTILINE)
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=1.0 if postscript else 0.0, higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/grid_difference.py

@@ -0,0 +1,77 @@
+import re
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class GridDifference(BaseMetric[Completion]):
+    NAME = "grid_difference"
+
+    def count_differences(self, character_list_1: list[str], character_list_2: list[str]) -> int:
+        count = 0
+        for character_1, character_2 in zip(character_list_1, character_list_2):
+            if character_1 != character_2:
+                count += 1
+        return count
+
+    def calculate_score(
+        self, output_ground_truth_difference_count: int, input_ground_truth_difference_count: int
+    ) -> float:
+        if output_ground_truth_difference_count == 0 and input_ground_truth_difference_count == 0:
+            return 1.0
+        score = 1.0 - (float(output_ground_truth_difference_count) / float(input_ground_truth_difference_count))
+        return score
+
+    def extract_grid_from_prompt(self, prompt: str) -> str:
+        # Extract grid between known markers
+        start_marker = "Below is the input grid with masked regions:"
+        end_marker = "Please output the completed grid"
+
+        # Use regex with DOTALL flag to match across newlines
+        match = re.search(re.escape(start_marker) + r"(.*?)" + re.escape(end_marker), prompt, re.DOTALL)
+
+        if match:
+            grid = match.group(1).strip()
+            return grid
+
+        return ""
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        input_grid = self.extract_grid_from_prompt(prompt=response.last_user_instruction).split()
+        output_grid = response.completion.split()
+
+        assert response.ground_truth_list[0], "Ground truth list is empty or not provided in the response."
+        ground_truth_grid = response.ground_truth_list[0].split()
+
+        input_ground_truth_differences_count = self.count_differences(input_grid, ground_truth_grid)
+        output_ground_truth_differences_count = self.count_differences(output_grid, ground_truth_grid)
+
+        exact_match = True
+        score = 1.0
+        normalized_score = 1.0
+        if output_ground_truth_differences_count != 0:
+            exact_match = False
+            score = self.calculate_score(
+                output_ground_truth_differences_count,
+                input_ground_truth_differences_count,
+            )
+            normalized_score = max(score, 0.0)
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}_exact_match",
+                value=float(exact_match),
+                higher_is_better=True,
+                error=response.error,
+            ),
+            MetricResult(metric_name=f"{self.NAME}_score", value=score, higher_is_better=True, error=response.error),
+            MetricResult(
+                metric_name=f"{self.NAME}_normalized_score",
+                value=normalized_score,
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]

eval_framework/metrics/completion/ifeval.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+from eval_framework.external.ifeval_impl.utils import process_results
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class IFEvalMetricContext(BaseMetricContext):
+    key: int
+    instruction_id_list: list[str]
+    prompt: str
+    additional_kwargs: list[dict[str, Any]]
+
+
+class IFEvalMetric(BaseMetric[Completion]):
+    NAME = "IFEval"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        context = extract_context_metric(response, IFEvalMetricContext)
+
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=f"{self.NAME}/prompt_level_strict_acc",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error,
+                ),
+                MetricResult(
+                    metric_name=f"{self.NAME}/prompt_level_loose_acc",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error,
+                ),
+            ]
+
+        grading = process_results(context, [response.completion])
+
+        results = [
+            MetricResult(
+                metric_name=f"{self.NAME}/prompt_level_strict_acc",
+                value=float(grading["prompt_level_strict_acc"]),
+                higher_is_better=True,
+                error=response.error,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/prompt_level_loose_acc",
+                value=float(grading["prompt_level_loose_acc"]),
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]
+        # this framework does not support a custom aggregation step (see agg_inst_level_acc()) so work around
+        # by returning the result for each instruction as a separate MetricResult
+        results += [
+            MetricResult(
+                metric_name=f"{self.NAME}/inst_level_strict_acc",
+                value=float(v),
+                higher_is_better=True,
+                error=response.error,
+            )
+            for v in grading["inst_level_strict_acc"]
+        ]
+        results += [
+            MetricResult(
+                metric_name=f"{self.NAME}/inst_level_loose_acc",
+                value=float(v),
+                higher_is_better=True,
+                error=response.error,
+            )
+            for v in grading["inst_level_loose_acc"]
+        ]
+        return results

eval_framework/metrics/completion/json_format.py

@@ -0,0 +1,171 @@
+import json
+from collections.abc import Mapping
+from typing import Any
+
+import jsonschema  # type: ignore
+from pydantic import BaseModel
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class JsonFormatEvaluation(BaseModel):
+    is_just_json: bool = False
+    is_valid_json: bool = False
+    fulfills_schema: bool | None = None
+    json_parsing_error: str | None = None
+    schema_validation_error: str | None = None
+
+
+class JsonFormat(BaseMetric[Completion]):
+    NAME = "JSON Format"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        keys = [
+            "is_just_json",
+            "is_valid_json",
+            "fulfills_schema",
+        ]
+
+        if response.error is not None:
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=None, higher_is_better=True, error=response.error)
+                for k in keys
+            ]
+
+        if response.completion == "":
+            return [
+                MetricResult(metric_name=f"{self.NAME}/{k}", value=0.0, higher_is_better=True, error=response.error)
+                for k in keys
+            ]
+
+        json_dict, grading = self._extract_and_parse_json(response.completion)
+        schema = json.loads(str(response.ground_truth))["json_schema"]
+        if schema and json_dict is None:
+            grading.fulfills_schema = False
+        if schema and json_dict is not None:
+            grading = self._validate_json_against_schema(json_dict, schema, grading)
+
+        results = []
+        for key in keys:
+            result = MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=float(getattr(grading, key)) if getattr(grading, key) is not None else None,
+                higher_is_better=True,
+                error=response.error,
+                code_execution_trace=(grading.json_parsing_error or "") + (grading.schema_validation_error or ""),
+            )
+            results.append(result)
+        return results
+
+    @staticmethod
+    def _validate_json_against_schema(
+        json_obj: object, schema: Mapping[str, Any], evaluation_result: JsonFormatEvaluation
+    ) -> JsonFormatEvaluation:
+        evaluation_result = evaluation_result.model_copy(deep=True)
+        try:
+            jsonschema.validate(json_obj, schema)
+            evaluation_result.fulfills_schema = True
+        except jsonschema.exceptions.ValidationError as e:
+            evaluation_result.fulfills_schema = False
+            evaluation_result.schema_validation_error = type(e).__name__
+        except jsonschema.exceptions.SchemaError as e:
+            evaluation_result.schema_validation_error = type(e).__name__
+        return evaluation_result
+
+    @staticmethod
+    def _extract_and_parse_json(completion: str) -> tuple[object, JsonFormatEvaluation]:
+        evaluation_result = JsonFormatEvaluation()
+        json_dict = None
+        try:
+            json_dict = json.loads(remove_comments(completion.strip("`")))
+            evaluation_result.is_just_json = True
+            evaluation_result.is_valid_json = True
+        except Exception as _:
+            try:
+                json_string = remove_comments(get_json_object(completion))
+                json_dict = json.loads(json_string)
+                evaluation_result.is_valid_json = True
+            except Exception as e:
+                evaluation_result.json_parsing_error = type(e).__name__
+        return json_dict, evaluation_result
+
+
+def get_json_object(text: str) -> str:
+    """
+    Extract the first valid JSON object or array from text.
+
+    This function handles nested brackets properly by using a bracket counting
+    approach to find complete JSON structures, rather than using regex which
+    can incorrectly match outer brackets containing non-JSON content.
+    """
+
+    def find_json_at_position(text: str, start_pos: int, open_char: str, close_char: str) -> str | None:
+        """Find a complete JSON object/array starting at the given position."""
+        if start_pos >= len(text) or text[start_pos] != open_char:
+            return None
+
+        bracket_count = 0
+        in_string = False
+        escaped = False
+
+        for i in range(start_pos, len(text)):
+            char = text[i]
+
+            if escaped:
+                escaped = False
+                continue
+
+            if char == "\\" and in_string:
+                escaped = True
+                continue
+
+            if char == '"' and not escaped:
+                in_string = not in_string
+                continue
+
+            if not in_string:
+                if char == open_char:
+                    bracket_count += 1
+                elif char == close_char:
+                    bracket_count -= 1
+                    if bracket_count == 0:
+                        # Found complete JSON structure
+                        candidate = text[start_pos : i + 1]
+                        # Test if it's valid JSON
+                        try:
+                            json.loads(candidate)
+                            return candidate
+                        except json.JSONDecodeError:
+                            return None
+
+        return None
+
+    # Look for JSON objects {} and arrays []
+    json_candidates = []
+
+    # Search for objects starting with {
+    for i in range(len(text)):
+        if text[i] == "{":
+            candidate = find_json_at_position(text, i, "{", "}")
+            if candidate:
+                json_candidates.append(candidate)
+
+    # Search for arrays starting with [
+    for i in range(len(text)):
+        if text[i] == "[":
+            candidate = find_json_at_position(text, i, "[", "]")
+            if candidate:
+                json_candidates.append(candidate)
+
+    if not json_candidates:
+        raise RuntimeError(f"No valid JSON object found in {text}.")
+
+    # Return the longest valid JSON (most likely to be the main content)
+    return max(json_candidates, key=len)
+
+
+def remove_comments(text: str, comment_indicator: str = "//") -> str:
+    lines = text.splitlines()
+    lines = [line.split(comment_indicator)[0] for line in lines]
+    return "\n".join([line for line in lines if line.strip()])

eval_framework/metrics/completion/language_checker.py

@@ -0,0 +1,74 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.llm.graders.language import AVAILABLE_LANGUAGES
+from eval_framework.shared.types import Completion
+
+
+class LanguageChecker(BaseMetric[Completion]):
+    NAME = "Language Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.ground_truth is None:
+            raise LogicError("Language detection needs ground_truth.")
+        if response.ground_truth not in AVAILABLE_LANGUAGES:
+            raise LogicError("Checking for unknown or unavailable language.")
+
+        completion_language = response.get_completion_language()
+        target_language = response.ground_truth
+        value = float(completion_language == target_language)
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class GermanCompletionChecker(BaseMetric[Completion]):
+    NAME = "German Completion Check"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        raw_completion_language = response.get_raw_completion_language()
+        value = float(raw_completion_language == "de")
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class LanguageConsistencyChecker(BaseMetric[Completion]):
+    NAME = "Language Consistency"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        completion_language = response.get_completion_language()
+        target_language = response.get_instruction_language()
+        if completion_language == target_language == "":
+            return []  # No language information could be determined
+        else:
+            value = float(completion_language == target_language)
+            return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class LanguageRawConsistencyChecker(BaseMetric[Completion]):
+    NAME = "Language Consistency Raw"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        raw_completion_language = response.get_raw_completion_language()
+        target_language = response.get_instruction_language()
+
+        if raw_completion_language == target_language == "":
+            return []  # No language information could be determined
+        else:
+            value = float(raw_completion_language == target_language)
+            return [
+                MetricResult(
+                    metric_name=self.NAME,
+                    value=value,
+                    higher_is_better=True,
+                    error=response.error,
+                )
+            ]

eval_framework/metrics/completion/length_control.py

@@ -0,0 +1,83 @@
+import json
+from enum import Enum
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.text_counter import ParagraphCounter, SentenceCounter, WordCounter
+from eval_framework.shared.types import Completion
+
+
+class LengthRequirementUnit(Enum):
+    WORDS = "words"
+    SENTENCES = "sentences"
+    PARAGRAPHS = "paragraphs"
+
+
+class LengthRequirementType(Enum):
+    MIN = "minimum"
+    MAX = "maximum"
+    TARGET = "target"
+
+
+class LengthControl(BaseMetric[Completion]):
+    NAME = "length_control"
+
+    def __init__(self, tolerance: float = 1 / 6) -> None:
+        super().__init__()
+        self.tolerance = tolerance
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=f"{self.NAME}/fulfills_length_requirement",
+                    value=None,
+                    higher_is_better=True,
+                    error=response.error if response.error is not None else None,
+                )
+            ]
+
+        expectations = json.loads(str(response.ground_truth))
+        stripped_completion = response.completion.strip()
+
+        match LengthRequirementUnit(expectations["unit"]):
+            case LengthRequirementUnit.WORDS:
+                count = WordCounter._count_words(stripped_completion)
+            case LengthRequirementUnit.SENTENCES:
+                count = SentenceCounter._count_sentences(stripped_completion)
+            case LengthRequirementUnit.PARAGRAPHS:
+                count = ParagraphCounter._count_paragraphs(stripped_completion)
+            case _:
+                raise NotImplementedError(f"LengthRequirementUnit {expectations['unit']} is not supported.")
+
+        expected_count = int(expectations["count"])
+        normalized_distance_to_target = (count - expected_count) / float(expected_count)
+        absolute_normalized_distance_to_target = abs(normalized_distance_to_target)
+
+        match LengthRequirementType(expectations["type"]):
+            case LengthRequirementType.TARGET:
+                fulfills_length_requirement = absolute_normalized_distance_to_target <= self.tolerance
+            case LengthRequirementType.MIN:
+                fulfills_length_requirement = count >= expected_count
+            case LengthRequirementType.MAX:
+                fulfills_length_requirement = count <= expected_count
+            case _:
+                raise NotImplementedError(f"LengthRequirementType {expectations['type']} is not supported.")
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/normalized_distance_to_target",
+                value=float(normalized_distance_to_target),
+                higher_is_better=False,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/absolute_normalized_distance_to_target",
+                value=float(absolute_normalized_distance_to_target),
+                higher_is_better=False,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/fulfills_length_requirement",
+                value=float(fulfills_length_requirement),
+                higher_is_better=True,
+                error=response.error,
+            ),
+        ]