eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/completion/rouge_geometric_mean.py

@@ -0,0 +1,36 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.metrics.completion.rouge_1 import ROUGE_1
+from eval_framework.metrics.completion.rouge_2 import ROUGE_2
+from eval_framework.metrics.completion.rouge_l import ROUGE_L
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_GEOMETRIC_MEAN(BaseMetric[Completion]):
+    """ROUGE Geometric Mean"""
+
+    NAME = "ROUGE-Geometric-Mean"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if any(gt is None for gt in response.ground_truth_list):
+            raise LogicError("When calculating ROUGE Geometric Mean ground_truth cannot be None.")
+
+        # Calculate ROUGE-1, ROUGE-2, and ROUGE-L
+        rouge_1 = ROUGE_1().calculate(response)[0].value
+        rouge_2 = ROUGE_2().calculate(response)[0].value
+        rouge_l = ROUGE_L().calculate(response)[0].value
+
+        # Calculate the geometric mean of ROUGE-1, ROUGE-2, and ROUGE-L
+        if rouge_1 is None or rouge_2 is None or rouge_l is None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        geometric_mean = (rouge_1 * rouge_2 * rouge_l) ** (1 / 3)
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(geometric_mean), higher_is_better=True, error=response.error
+            )
+        ]

eval_framework/metrics/completion/rouge_l.py

@@ -0,0 +1,52 @@
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class ROUGE_L(BaseMetric[Completion]):
+    """ROUGE-L"""
+
+    NAME = "ROUGE-L"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        if response.completion == "":
+            return [MetricResult(metric_name=self.NAME, value=0.0, higher_is_better=True, error=response.error)]
+        if None in response.ground_truth_list:
+            raise LogicError("When calculating ROUGE-L ground_truth cannot be None.")
+
+        # ROUGE-L is essentially an F1 score, but it’s a specific F1 score based on
+        # the Longest Common Subsequence (LCS) between a candidate summary and a reference summary.
+        rouge = max([_calculate_rouge_l(response.completion, gt) for gt in response.ground_truth_list])  # type: ignore[arg-type]
+        return [MetricResult(metric_name=self.NAME, value=float(rouge), higher_is_better=True, error=response.error)]
+
+
+def _longest_common_subsequence_length(candidate_tokens: list[str], reference_tokens: list[str]) -> int:
+    candidate_len, reference_len = len(candidate_tokens), len(reference_tokens)
+    lcs_matrix = [[0] * (reference_len + 1) for _ in range(candidate_len + 1)]
+
+    for i in range(candidate_len + 1):
+        for j in range(reference_len + 1):
+            if i == 0 or j == 0:
+                lcs_matrix[i][j] = 0
+            elif candidate_tokens[i - 1] == reference_tokens[j - 1]:
+                lcs_matrix[i][j] = lcs_matrix[i - 1][j - 1] + 1
+            else:
+                lcs_matrix[i][j] = max(lcs_matrix[i - 1][j], lcs_matrix[i][j - 1])
+
+    return lcs_matrix[candidate_len][reference_len]
+
+
+def _calculate_rouge_l(completion: str, ground_truth: str) -> float:
+    lcs_length = _longest_common_subsequence_length(completion.split(), ground_truth.split())
+    if lcs_length == 0:
+        return 0.0
+    precision = lcs_length / len(completion.split())
+    recall = lcs_length / len(ground_truth.split())
+    if precision + recall == 0:
+        f1_score = 0.0
+    else:
+        f1_score = (2 * precision * recall) / (precision + recall)
+    return f1_score

eval_framework/metrics/completion/struct_eval_metrics.py

@@ -0,0 +1,248 @@
+import csv
+import io
+import json
+import tomllib
+from typing import Any
+
+import xmltodict
+import yaml
+from lxml import etree
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+
+class StructMetricContext(BaseMetricContext):
+    output_type: str
+    paths: list[str]
+
+
+class StructMetric(BaseMetric[Completion]):
+    NAME = "StructMetric"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, StructMetricContext)
+
+        output_type = context.output_type
+
+        try:
+            match output_type.lower():
+                case "json":
+                    result = json.loads(response.completion)
+                case "yaml":
+                    result = list(yaml.safe_load_all(response.completion))
+                    if isinstance(result, list) and len(result) == 1:
+                        result = result[0]
+                    else:
+                        raise yaml.YAMLError("Multiple documents found in YAML")
+                case "toml":
+                    result = tomllib.loads(response.completion)
+                case "xml":
+                    result = xmltodict.parse(response.completion)
+                case "csv":
+                    csv_output = csv.DictReader(io.StringIO(response.completion))
+                    # Check for unclosed quotes
+                    if response.completion.count('"') % 2 != 0:
+                        raise csv.Error("Unclosed quote in CSV")
+                    if not csv_output.fieldnames:
+                        raise csv.Error("CSV has no headers")
+                    result = {"csv_headers": csv_output.fieldnames, "csv_rows": list(csv_output)}
+                case _:
+                    raise ValueError(f"Unsupported format: {output_type}")
+            valid_format = 1.0
+        except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError, csv.Error, Exception):
+            valid_format = 0.0
+
+        has_required_fields = 0.0
+        if valid_format == 1:
+            # assert "paths" in response.eval_kwargs, "Paths must be provided in eval_kwargs"
+            assert context.paths is not None, "Paths must be provided in context"
+            paths = context.paths
+            assert isinstance(paths, list), "Paths must be a list of strings"
+            valid_paths = 0
+            for path in paths:
+                if path_exists(result, path):
+                    valid_paths += 1
+            has_required_fields = valid_paths / len(paths) if paths else 1.0
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/valid_format",
+                value=valid_format,
+                higher_is_better=True,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/has_keywords",
+                value=has_required_fields,
+                higher_is_better=True,
+            ),
+        ]
+
+
+def is_valid_html(html: str) -> bool:
+    parser = etree.HTMLParser(recover=False)
+    try:
+        etree.fromstring(html.encode("utf-8"), parser)
+    except etree.XMLSyntaxError:
+        return False
+    return len(parser.error_log) == 0
+
+
+class RenderableStructMetricContext(BaseMetricContext):
+    output_type: str
+    keywords: list[str]
+
+
+class RenderableStructMetric(StructMetric):
+    NAME = "RenderableStructMetric"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, RenderableStructMetricContext)
+
+        output_type = context.output_type
+
+        valid_format = 0.0
+        match output_type.lower():
+            case "html":
+                valid_format = float(is_valid_html(response.completion))
+            case _:
+                raise ValueError(f"Unsupported format for RenderableStructMetric: {output_type}")
+
+        assert context.keywords is not None, "Keywords must be provided in context"
+        keywords = context.keywords
+        assert isinstance(keywords, list), "Keywords must be a list of strings"
+        has_keywords = 1.0
+        if keywords:
+            has_keywords = sum(1 for keyword in keywords if keyword.lower() in response.completion.lower()) / len(
+                keywords
+            )
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/valid_format",
+                value=valid_format,
+                higher_is_better=True,
+            ),
+            MetricResult(
+                metric_name=f"{self.NAME}/has_keywords",
+                value=has_keywords,
+                higher_is_better=True,
+            ),
+        ]
+
+
+# adapted from: https://github.com/TIGER-AI-Lab/StructEval/blob/main/structeval/eval_engine/eval_utils.py
+def tokenize_path(path: str) -> list[str]:
+    """
+    Tokenize a dot-notation path, handling back-ticks and array indices.
+
+    Args:
+        path: The path string (e.g. "users.0.name" or "users[0].name")
+
+    Returns:
+        List of path tokens
+    """
+    # Special‑case: treat CSV header paths as a single token
+    if path.startswith("csv::"):
+        return [path]
+
+    tokens, buf, in_bt = [], "", False
+    i, n = 0, len(path)
+
+    while i < n:
+        ch = path[i]
+
+        # Toggle back-tick state
+        if ch == "`":
+            in_bt = not in_bt
+            i += 1
+            continue
+
+        # Dot delimiter (when not inside back-ticks)
+        if ch == "." and not in_bt:
+            if buf:
+                tokens.append(buf)
+                buf = ""
+            i += 1
+            continue
+
+        # Bracket "[index]" treated as separate token
+        if ch == "[" and not in_bt:
+            if buf:
+                tokens.append(buf)
+                buf = ""
+            j = path.find("]", i)
+            if j == -1:
+                raise ValueError(f"Unclosed '[' in path: {path}")
+            tokens.append(path[i : j + 1])  # e.g. "[0]"
+            i = j + 1
+            continue
+
+        # Regular character
+        buf += ch
+        i += 1
+
+    if buf:
+        tokens.append(buf)
+    return tokens
+
+
+# adapted from: https://github.com/TIGER-AI-Lab/StructEval/blob/main/structeval/eval_engine/eval_utils.py
+def path_exists(data: Any, path: str) -> bool:
+    """
+    Check if a path exists in a structured data object.
+
+    Args:
+        data: The structured data to check
+        path: The path to check (dot notation)
+
+    Returns:
+        True if path exists, False otherwise
+    """
+    tokens = tokenize_path(path)
+
+    def walk(node: Any, toks: list[str]) -> bool:
+        if not toks:
+            return True
+        tok, *rest = toks
+
+        # CSV header rule (root level only)
+        if isinstance(node, dict) and "csv_headers" in node and tok.startswith("csv::"):
+            header = tok[5:]
+            return header in node["csv_headers"] and not rest  # must be terminal
+
+        # Wildcard
+        if tok == "*":
+            if isinstance(node, list):
+                return any(walk(item, rest) for item in node)
+            return False
+
+        # Fixed index [n]
+        if tok.startswith("[") and tok.endswith("]"):
+            try:
+                idx = int(tok[1:-1])
+            except ValueError:
+                return False
+            return isinstance(node, list) and 0 <= idx < len(node) and walk(node[idx], rest)
+
+        # Dict key handling (JSON/YAML/TOML/XML)
+        if isinstance(node, dict):
+            # 1️⃣ Literal key match (works for "@id" too)
+            if tok in node:
+                return walk(node[tok], rest)
+
+            # 2️⃣ XML attribute fallback: "@id" → "id"
+            if tok.startswith("@"):
+                attr = tok[1:]
+                if attr in node:
+                    return walk(node[attr], rest)
+
+        return False
+
+    return walk(data, tokens)

eval_framework/metrics/completion/ter.py

@@ -0,0 +1,67 @@
+import sacrebleu
+
+from eval_framework.exceptions import LogicError
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class TER(BaseMetric[Completion]):
+    """Translation Error Rate is an error metric for machine translation that
+    measures the number of edits required to change a system output into one
+    of the references
+    Source: http://www.cs.umd.edu/~snover/tercom/
+    Paper: http://mt-archive.info/AMTA-2006-Snover.pdf
+    """
+
+    NAME = "TER"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=False, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            if ground_truth == "" or ground_truth is None:
+                raise LogicError("When calculating TER we need a ground truth.")
+
+            sacre_formatted_completion = [response.completion]
+            sacre_formatted_ground_truth = [[ground_truth]]
+            ter_score = sacrebleu.corpus_ter(sacre_formatted_completion, sacre_formatted_ground_truth).score
+            scores.append(ter_score)
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(min(scores)), higher_is_better=False, error=response.error)
+        ]
+
+
+class LINEWISE_TER(BaseMetric[Completion]):
+    """Minimum Line-level TER (Translation Edit Rate) score."""
+
+    NAME = "Linewise TER"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=False, error=response.error)]
+
+        scores = []
+        for ground_truth in response.ground_truth_list:
+            for sentence in response.completion.split("\n"):
+                if sentence == "":
+                    continue
+
+                if ground_truth == "" or ground_truth is None:
+                    raise LogicError("When calculating TER we need a ground truth.")
+
+                sacre_formatted_completion = [sentence]
+                sacre_formatted_ground_truth = [[ground_truth]]
+                ter_score = sacrebleu.corpus_ter(sacre_formatted_completion, sacre_formatted_ground_truth).score
+                scores.append(ter_score)
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(min(scores, default=100)),
+                higher_is_better=False,
+                error=response.error,
+            )
+        ]

eval_framework/metrics/completion/text_counter.py

@@ -0,0 +1,182 @@
+import re
+
+import nltk
+
+from eval_framework.metrics.base import (
+    BaseMetric,
+    MetricResult,
+)
+from eval_framework.shared.types import BaseMetricContext, Completion, extract_context_metric
+
+ALPHABETS = "([A-Za-z])"
+PREFIXES = "(Mr|St|Mrs|Ms|Dr|www)[.]"
+SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)"
+STARTERS = (
+    r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
+)
+ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
+WEBSITES = "[.](com|net|org|io|gov|edu|me)"
+DIGITS = "([0-9])"
+MULTIPLE_DOTS = r"\.{2,}"
+
+
+class WordCounterMetricContext(BaseMetricContext):
+    comparison: str
+    word_count: int
+
+
+class WordCounter(BaseMetric[Completion]):
+    NAME = "Word Count"
+
+    @staticmethod
+    def _count_words(text: str) -> int:
+        tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+")
+        tokens = tokenizer.tokenize(text)
+        num_words = len(tokens)
+        return num_words
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, WordCounterMetricContext)
+
+        assert context.comparison in ["less than", "at least"], f"'comparison' is not valid: {context.comparison}"
+
+        num_words = self._count_words(response.completion)
+        if context.comparison == "less than":
+            valid_word_count = num_words < context.word_count
+        if context.comparison == "at least":
+            valid_word_count = num_words >= context.word_count
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(valid_word_count), higher_is_better=True, error=response.error
+            )
+        ]
+
+
+class SentenceCounterMetricContext(BaseMetricContext):
+    comparison: str
+    sentence_count: int
+
+
+class SentenceCounter(BaseMetric[Completion]):
+    NAME = "Sentence Count"
+
+    @staticmethod
+    def _count_sentences(text: str) -> int:
+        # Note that nltk.tokenize.sent_tokenize would be a straightforward alternative but is also not ideal. Example:
+        #
+        # "Mr. Jones gave me $10,000.00... And then he left. Numbers 5...10. Numbers 5..10. Review: bad food,
+        #    bad service,..., so I'd miss it."
+        #
+        # this: ['Mr. Jones gave me $10,000.00...', 'And then he left.', 'Numbers 5...', '10.', 'Numbers 5..', '10.',
+        #    'Review: bad food, bad service,...', ", so I'd miss it."].
+        # nltk: ['Mr. Jones gave me $10,000.00... And then he left.', 'Numbers 5...10.',
+        #    "Numbers 5..10. Review: bad food, bad service,..., so I'd miss it."]
+
+        text = f" {text} "
+        text = text.replace("\n", " ")
+        text = re.sub(PREFIXES, "\\1<prd>", text)
+        text = re.sub(WEBSITES, "<prd>\\1", text)
+        text = re.sub(DIGITS + "[.]" + DIGITS, "\\1<prd>\\2", text)
+        text = re.sub(
+            MULTIPLE_DOTS,
+            lambda match: "<prd>" * len(match.group(0)) + "<stop>",
+            text,
+        )
+        text = text.replace("Ph.D.", "Ph<prd>D<prd>")
+        text = re.sub(r"\s" + ALPHABETS + "[.] ", " \\1<prd> ", text)
+        text = re.sub(ACRONYMS + " " + STARTERS, "\\1<stop> \\2", text)
+        text = re.sub(
+            ALPHABETS + "[.]" + ALPHABETS + "[.]" + ALPHABETS + "[.]",
+            "\\1<prd>\\2<prd>\\3<prd>",
+            text,
+        )
+        text = re.sub(ALPHABETS + "[.]" + ALPHABETS + "[.]", "\\1<prd>\\2<prd>", text)
+        text = re.sub(" " + SUFFIXES + "[.] " + STARTERS, " \\1<stop> \\2", text)
+        text = re.sub(" " + SUFFIXES + "[.]", " \\1<prd>", text)
+        text = re.sub(" " + ALPHABETS + "[.]", " \\1<prd>", text)
+        text = text.replace(".”", "”.")
+        text = text.replace('."', '".')
+        text = text.replace('!"', '"!')
+        text = text.replace('?"', '"?')
+        text = text.replace(".", ".<stop>")
+        text = text.replace("?", "?<stop>")
+        text = text.replace("!", "!<stop>")
+        text = text.replace("<prd>", ".")
+        sentences = text.split("<stop>")
+        sentences = [s.strip() for s in sentences]
+        if sentences and not sentences[-1]:
+            sentences = sentences[:-1]
+        return len(sentences)
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, SentenceCounterMetricContext)
+
+        assert context.comparison in ["less than", "at least"], f"'comparison' is not valid: {context.comparison}"
+
+        num_sentences = self._count_sentences(response.completion)
+        if context.comparison == "less than":
+            valid_sentence_count = num_sentences < context.sentence_count
+        elif context.comparison == "at least":
+            valid_sentence_count = num_sentences >= context.sentence_count
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(valid_sentence_count), higher_is_better=True, error=response.error
+            )
+        ]
+
+
+class ParagraphCounterMetricContext(BaseMetricContext):
+    comparison: str
+    paragraph_count: int
+
+
+class ParagraphCounter(BaseMetric[Completion]):
+    NAME = "Paragraph Count"
+
+    @staticmethod
+    def _count_paragraphs(text: str) -> int:
+        paragraphs = re.split(r"\s?\n\n\s?", text)
+        return len(paragraphs)
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        context = extract_context_metric(response, ParagraphCounterMetricContext)
+
+        assert context.comparison in ["less than", "at least"], f"'comparison' is not valid: {context.comparison}"
+
+        num_paragraphs = self._count_paragraphs(response.completion)
+        if context.comparison == "less than":
+            valid_paragraph_count = num_paragraphs < context.paragraph_count
+        elif context.comparison == "at least":
+            valid_paragraph_count = num_paragraphs >= context.paragraph_count
+
+        return [
+            MetricResult(
+                metric_name=self.NAME, value=float(valid_paragraph_count), higher_is_better=True, error=response.error
+            )
+        ]
+
+
+class ResponseToOriginalLengthRatio(BaseMetric[Completion]):
+    NAME = "Response to Original Length Ratio"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        len_original = len(response.last_user_instruction)
+        if len_original > 0:
+            score = len(response.completion) / len_original
+            return [MetricResult(metric_name=self.NAME, value=score, higher_is_better=False, error=response.error)]
+        else:
+            return []

eval_framework/metrics/efficiency/bytes_per_sequence_position.py

@@ -0,0 +1,48 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion, Loglikelihood
+
+
+class BytesLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "Bytes"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error or response.concat_compression is None:
+            value = None
+        else:
+            value = response.concat_compression.num_bytes
+
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class SequencePositionsLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "SequencePositions"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error or response.concat_compression is None:
+            value = None
+        else:
+            value = response.concat_compression.num_tokens
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class BytesCompletion(BaseMetric[Completion]):
+    NAME = "Bytes"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error or response.concat_compression is None:
+            value = None
+        else:
+            value = response.concat_compression.num_bytes
+
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]
+
+
+class SequencePositionsCompletion(BaseMetric[Completion]):
+    NAME = "SequencePositions"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error or response.concat_compression is None:
+            value = None
+        else:
+            value = response.concat_compression.num_tokens
+        return [MetricResult(metric_name=self.NAME, value=value, higher_is_better=True, error=response.error)]

eval_framework/metrics/llm/base.py

@@ -0,0 +1,8 @@
+from eval_framework.llm.base import BaseLLM
+from eval_framework.metrics.base import BaseMetric
+from eval_framework.shared.types import Completion
+
+
+class BaseLLMJudgeMetric(BaseMetric[Completion]):
+    def __init__(self, llm_judge: BaseLLM) -> None:
+        self._llm_judge = llm_judge