eval-framework 0.2.7__py3-none-any.whl

eval_framework/metrics/completion/repetition.py

@@ -0,0 +1,88 @@
+import re
+from collections import Counter
+from collections.abc import Sequence
+from typing import Final
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class WordRepetition(BaseMetric[Completion]):
+    """Word Repetition Metric
+
+    This metric checks for repetitions of words in the completion text for a
+    given window size and repetition threshold. The window size defines the
+    consecutive word count to consider a repetition, and min_repetitions
+    specifies the minimum repetition count that triggers the metric. This metric
+    returns 0.0 if no repetitions are found, and 1.0 if a sufficient number of
+    repetitions are found. For example, if the completion contains a two-word
+    sequence that repeats once (such as "hello world hello world"), this metric
+    would trigger with a window size of 2 and min_repetitions set to 1.
+    """
+
+    NAME = "WordRepetition"
+    HIGHER_IS_BETTER: Final[bool] = False
+
+    def __init__(self, window_size: int = 128, min_repetitions: int = 1) -> None:
+        """
+        Initialize the WordRepetition metric.
+
+        Args:
+            window_size (int): The number of consecutive words to consider as a
+                sequence.
+            min_repetitions (int): The minimum number of times a sequence must
+                repeat to be considered a repetition. Set to 1 to catch any
+                repetition.
+        """
+        super().__init__()
+        self.window_size = window_size
+        self.min_repetitions = min_repetitions
+
+        if self.min_repetitions < 1:
+            raise ValueError("min_repetitions must be at least 1")
+
+        if self.window_size < 1:
+            raise ValueError("window_size must be at least 1")
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [
+                MetricResult(
+                    metric_name=self.NAME,
+                    value=None,
+                    higher_is_better=self.HIGHER_IS_BETTER,
+                    error=response.error,
+                )
+            ]
+
+        has_repetition = _has_repetition(
+            text=response.completion,
+            window_size=self.window_size,
+            min_repetitions=self.min_repetitions,
+        )
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(has_repetition),
+                higher_is_better=self.HIGHER_IS_BETTER,
+                error=response.error,
+            )
+        ]
+
+
+def _has_repetition(text: str, window_size: int, min_repetitions: int) -> bool:
+    """Check if the text contains any word sequences of a given size that repeat"""
+    sequences = _word_sequences(_to_words(text), window_size)
+    counts = Counter(sequences)
+    return any([count > min_repetitions for count in counts.values()])
+
+
+def _to_words(text: str) -> Sequence[str]:
+    """A somewhat crude function to tokenize a string into words."""
+    return re.findall(r"\b\w+\b", text, re.UNICODE)
+
+
+def _word_sequences(text_words: Sequence[str], window_size: int) -> Sequence[Sequence[str]]:
+    """Get all contiguous sub-sequences of a given size from a word sequence."""
+    return [tuple(text_words[i : i + window_size]) for i in range(len(text_words) - window_size + 1)]

eval_framework/metrics/completion/rouge_1.py

@@ -0,0 +1,35 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.f1 import calculate_f1
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_1(BaseMetric[Completion]):
+    """ROUGE-1"""
+
+    NAME = "ROUGE-1"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-1 ground_truth cannot be None.")
+
+        # ROUGE-1 captures word sequence similarity by focusing on unigrams
+        rouge = max([_calculate_rouge_1(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _calculate_rouge_1(candidate: str, reference: str) -> float:
+    """
+    Calculate ROUGE-1 precision, recall, and F1 score between candidate and reference texts.
+    """
+
+    # Tokenize the candidate and reference summaries
+    candidate_tokens = candidate.split()
+    reference_tokens = reference.split()
+
+    return calculate_f1(reference_tokens, candidate_tokens)

eval_framework/metrics/completion/rouge_2.py

@@ -0,0 +1,45 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.f1 import calculate_f1
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_2(BaseMetric[Completion]):
+    """ROUGE-2"""
+
+    NAME = "ROUGE-2"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-2 ground_truth cannot be None.")
+
+        # ROUGE-2 captures word sequence similarity by focusing on bigrams,
+        # which makes it sensitive to the order and co-occurrence of words to some extent.
+        rouge = max([_calculate_rouge_2(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _generate_bigrams(tokens: list[str]) -> list[tuple[str, str]]:
+    """Generate bigrams from a list of tokens."""
+    return [(tokens[i], tokens[i + 1]) for i in range(len(tokens) - 1)]
+
+
+def _calculate_rouge_2(completion: str, ground_truth: str) -> float:
+    """
+    Calculate ROUGE-2 precision, recall, and F1 score between candidate and reference texts.
+    """
+
+    # Tokenize the candidate and reference summaries
+    candidate_tokens = completion.split()
+    reference_tokens = ground_truth.split()
+
+    # Generate bigrams for candidate and reference
+    candidate_bigrams = _generate_bigrams(candidate_tokens)
+    reference_bigrams = _generate_bigrams(reference_tokens)
+
+    return calculate_f1(reference_bigrams, candidate_bigrams)

eval_framework/metrics/completion/rouge_geometric_mean.py

@@ -0,0 +1,36 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.rouge_1 import ROUGE_1
+from eval_framework.metrics.completion.rouge_2 import ROUGE_2
+from eval_framework.metrics.completion.rouge_l import ROUGE_L
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_GEOMETRIC_MEAN(BaseMetric[Completion]):
+    """ROUGE Geometric Mean"""
+
+    NAME = "ROUGE-Geometric-Mean"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if any(gt is None for gt in response.ground_truth_list):
+            raise LogicError("When calculating ROUGE Geometric Mean ground_truth cannot be None.")
+
+        # Calculate ROUGE-1, ROUGE-2, and ROUGE-L
+        rouge_1 = ROUGE_1().calculate(response)[0].value
+        rouge_2 = ROUGE_2().calculate(response)[0].value
+        rouge_l = ROUGE_L().calculate(response)[0].value
+
+        # Calculate the geometric mean of ROUGE-1, ROUGE-2, and ROUGE-L
+        if rouge_1 is None or rouge_2 is None or rouge_l is None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        geometric_mean = (rouge_1 * rouge_2 * rouge_l) ** (1 / 3)
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(geometric_mean), higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/rouge_l.py

@@ -0,0 +1,52 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_L(BaseMetric[Completion]):
+    """ROUGE-L"""
+
+    NAME = "ROUGE-L"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-L ground_truth cannot be None.")
+
+        # ROUGE-L is essentially an F1 score, but it’s a specific F1 score based on
+        # the Longest Common Subsequence (LCS) between a candidate summary and a reference summary.
+        rouge = max([_calculate_rouge_l(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _longest_common_subsequence_length(candidate_tokens: list[str], reference_tokens: list[str]) -> int:
+    candidate_len, reference_len = len(candidate_tokens), len(reference_tokens)
+    lcs_matrix = [[0] * (reference_len + 1) for _ in range(candidate_len + 1)]
+
+    for i in range(candidate_len + 1):
+        for j in range(reference_len + 1):
+            if i == 0 or j == 0:
+                lcs_matrix[i][j] = 0
+            elif candidate_tokens[i - 1] == reference_tokens[j - 1]:
+                lcs_matrix[i][j] = lcs_matrix[i - 1][j - 1] + 1
+            else:
+                lcs_matrix[i][j] = max(lcs_matrix[i - 1][j], lcs_matrix[i][j - 1])
+
+    return lcs_matrix[candidate_len][reference_len]
+
+
+def _calculate_rouge_l(completion: str, ground_truth: str) -> float:
+    lcs_length = _longest_common_subsequence_length(completion.split(), ground_truth.split())
+    if lcs_length == 0:
+        return 0.0
+    precision = lcs_length / len(completion.split())
+    recall = lcs_length / len(ground_truth.split())
+    if precision + recall == 0:
+        f1_score = 0.0
+    else:
+        f1_score = (2 * precision * recall) / (precision + recall)
+    return f1_score

eval_framework/metrics/completion/struct_eval_metrics.py

@@ -0,0 +1,248 @@
+import csv
+import io
+import json
+import tomllib
+from typing import Any
+
+import xmltodict
+import yaml
+from lxml import etree
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class StructMetricContext(BaseMetricContext):
+    output_type: str
+    paths: list[str]
+
+
+class StructMetric(BaseMetric[Completion]):
+    NAME = "StructMetric"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, StructMetricContext)
+
+        output_type = context.output_type
+
+        try:
+            match output_type.lower():
+                case "json":
+                    result = json.loads(response.completion)
+                case "yaml":
+                    result = list(yaml.safe_load_all(response.completion))
+                    if isinstance(result, list) and len(result) == 1:
+                        result = result[0]
+                    else:
+                        raise yaml.YAMLError("Multiple documents found in YAML")
+                case "toml":
+                    result = tomllib.loads(response.completion)
+                case "xml":
+                    result = xmltodict.parse(response.completion)
+                case "csv":
+                    csv_output = csv.DictReader(io.StringIO(response.completion))
+                    # Check for unclosed quotes
+                    if response.completion.count('"') % 2 != 0:
+                        raise csv.Error("Unclosed quote in CSV")
+                    if not csv_output.fieldnames:
+                        raise csv.Error("CSV has no headers")
+                    result = {"csv_headers": csv_output.fieldnames, "csv_rows": list(csv_output)}
+                case _:
+                    raise ValueError(f"Unsupported format: {output_type}")
+            valid_format = 1.0
+        except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError, csv.Error, Exception):
+            valid_format = 0.0
+
+        has_required_fields = 0.0
+        if valid_format == 1:
+            # assert "paths" in response.eval_kwargs, "Paths must be provided in eval_kwargs"
+            assert context.paths is not None, "Paths must be provided in context"
+            paths = context.paths
+            assert isinstance(paths, list), "Paths must be a list of strings"
+            valid_paths = 0
+            for path in paths:
+                if path_exists(result, path):
+                    valid_paths += 1
+            has_required_fields = valid_paths / len(paths) if paths else 1.0
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/valid_format",
+                value=valid_format,
+                higher_is_better=True,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/has_keywords",
+                value=has_required_fields,
+                higher_is_better=True,
+            ),
+        ]
+
+
+def is_valid_html(html: str) -> bool:
+    parser = etree.HTMLParser(recover=False)
+    try:
+        etree.fromstring(html.encode("utf-8"), parser)
+    except etree.XMLSyntaxError:
+        return False
+    return len(parser.error_log) == 0
+
+
+class RenderableStructMetricContext(BaseMetricContext):
+    output_type: str
+    keywords: list[str]
+
+
+class RenderableStructMetric(StructMetric):
+    NAME = "RenderableStructMetric"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, RenderableStructMetricContext)
+
+        output_type = context.output_type
+
+        valid_format = 0.0
+        match output_type.lower():
+            case "html":
+                valid_format = float(is_valid_html(response.completion))
+            case _:
+                raise ValueError(f"Unsupported format for RenderableStructMetric: {output_type}")
+
+        assert context.keywords is not None, "Keywords must be provided in context"
+        keywords = context.keywords
+        assert isinstance(keywords, list), "Keywords must be a list of strings"
+        has_keywords = 1.0
+        if keywords:
+            has_keywords = sum(1 for keyword in keywords if keyword.lower() in response.completion.lower()) / len(
+                keywords
+            )
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/valid_format",
+                value=valid_format,
+                higher_is_better=True,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/has_keywords",
+                value=has_keywords,
+                higher_is_better=True,
+            ),
+        ]
+
+
+# adapted from: https://github.com/TIGER-AI-Lab/StructEval/blob/main/structeval/eval_engine/eval_utils.py
+def tokenize_path(path: str) -> list[str]:
+    """
+    Tokenize a dot-notation path, handling back-ticks and array indices.
+
+    Args:
+        path: The path string (e.g. "users.0.name" or "users[0].name")
+
+    Returns:
+        List of path tokens
+    """
+    # Special‑case: treat CSV header paths as a single token
+    if path.startswith("csv::"):
+        return [path]
+
+    tokens, buf, in_bt = [], "", False
+    i, n = 0, len(path)
+
+    while i < n:
+        ch = path[i]
+
+        # Toggle back-tick state
+        if ch == "`":
+            in_bt = not in_bt
+            i += 1
+            continue
+
+        # Dot delimiter (when not inside back-ticks)
+        if ch == "." and not in_bt:
+            if buf:
+                tokens.append(buf)
+                buf = ""
+            i += 1
+            continue
+
+        # Bracket "[index]" treated as separate token
+        if ch == "[" and not in_bt:
+            if buf:
+                tokens.append(buf)
+                buf = ""
+            j = path.find("]", i)
+            if j == -1:
+                raise ValueError(f"Unclosed '[' in path: {path}")
+            tokens.append(path[i : j + 1])  # e.g. "[0]"
+            i = j + 1
+            continue
+
+        # Regular character
+        buf += ch
+        i += 1
+
+    if buf:
+        tokens.append(buf)
+    return tokens
+
+
+# adapted from: https://github.com/TIGER-AI-Lab/StructEval/blob/main/structeval/eval_engine/eval_utils.py
+def path_exists(data: Any, path: str) -> bool:
+    """
+    Check if a path exists in a structured data object.
+
+    Args:
+        data: The structured data to check
+        path: The path to check (dot notation)
+
+    Returns:
+        True if path exists, False otherwise
+    """
+    tokens = tokenize_path(path)
+
+    def walk(node: Any, toks: list[str]) -> bool:
+        if not toks:
+            return True
+        tok, *rest = toks
+
+        # CSV header rule (root level only)
+        if isinstance(node, dict) and "csv_headers" in node and tok.startswith("csv::"):
+            header = tok[5:]
+            return header in node["csv_headers"] and not rest  # must be terminal
+
+        # Wildcard
+        if tok == "*":
+            if isinstance(node, list):
+                return any(walk(item, rest) for item in node)
+            return False
+
+        # Fixed index [n]
+        if tok.startswith("[") and tok.endswith("]"):
+            try:
+                idx = int(tok[1:-1])
+            except ValueError:
+                return False
+            return isinstance(node, list) and 0 <= idx < len(node) and walk(node[idx], rest)
+
+        # Dict key handling (JSON/YAML/TOML/XML)
+        if isinstance(node, dict):
+            # 1️⃣ Literal key match (works for "@id" too)
+            if tok in node:
+                return walk(node[tok], rest)
+
+            # 2️⃣ XML attribute fallback: "@id" → "id"
+            if tok.startswith("@"):
+                attr = tok[1:]
+                if attr in node:
+                    return walk(node[attr], rest)
+
+        return False
+
+    return walk(data, tokens)

eval_framework/metrics/completion/ter.py

@@ -0,0 +1,67 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class TER(BaseMetric[Completion]):
+    """Translation Error Rate is an error metric for machine translation that
+    measures the number of edits required to change a system output into one
+    of the references
+    Source: http://www.cs.umd.edu/~snover/tercom/
+    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf
+    """
+
+    NAME = "TER"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=False, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating TER we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            ter_score = sacrebleu.corpus_ter(sacre_formatted_completion, sacre_formatted_ground_truth).score
+            scores.append(ter_score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(min(scores)), higher_is_better=False, error=response.error)
+        ]
+
+
+class LINEWISE_TER(BaseMetric[Completion]):
+    """Minimum Line-level TER (Translation Edit Rate) score."""
+
+    NAME = "Linewise TER"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=False, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating TER we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                ter_score = sacrebleu.corpus_ter(sacre_formatted_completion, sacre_formatted_ground_truth).score
+                scores.append(ter_score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(min(scores, default=100)),
+                higher_is_better=False,
+                error=response.error,
+            )
+        ]