opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/heuristics/readability.py

@@ -0,0 +1,129 @@
+"""Readability heuristics backed by the ``textstat`` library."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+
+try:  # pragma: no cover - optional dependency
+    import textstat as _textstat_lib
+except ImportError:  # pragma: no cover - optional dependency
+    _textstat_lib = None
+
+
+class Readability(BaseMetric):
+    """Compute common readability statistics using ``textstat``.
+
+    The metric reports the Flesch Reading Ease (0–100) alongside the Flesch–Kincaid
+    grade level. The score value is the reading-ease score normalised to ``[0, 1]``.
+    You can optionally enforce grade bounds to turn the metric into a guardrail.
+
+    Args:
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+        min_grade: Inclusive lower bound for the acceptable grade.
+        max_grade: Inclusive upper bound for the acceptable grade.
+        language: Locale forwarded to ``textstat`` when counting syllables.
+        textstat_module: Optional ``textstat``-compatible module for dependency
+            injection (mainly used in tests).
+        enforce_bounds: When ``True`` the metric returns ``1.0`` if the grade lies
+            within bounds and ``0.0`` otherwise, effectively acting as a guardrail.
+    """
+
+    def __init__(
+        self,
+        *,
+        name: str = "readability_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        min_grade: Optional[float] = None,
+        max_grade: Optional[float] = None,
+        language: str = "en_US",
+        textstat_module: Optional[Any] = None,
+        enforce_bounds: bool = False,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        if textstat_module is not None:
+            self._textstat = textstat_module
+        else:
+            if _textstat_lib is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "Readability metric requires the optional 'textstat' package. "
+                    "Install via `pip install textstat`."
+                )
+            self._textstat = _textstat_lib
+
+        self._min_grade = min_grade
+        self._max_grade = max_grade
+        self._language = language
+        self._enforce_bounds = enforce_bounds
+
+    def score(
+        self,
+        output: str,
+        **ignored_kwargs: Any,
+    ) -> ScoreResult:
+        if not output or not output.strip():
+            raise MetricComputationError("Text is empty (Readability metric).")
+
+        cleaned = output.strip()
+        sentence_count = self._textstat.sentence_count(cleaned)
+        word_count = self._textstat.lexicon_count(cleaned, removepunct=True)
+        if sentence_count <= 0 or word_count <= 0:
+            raise MetricComputationError(
+                "Unable to parse text for readability metrics."
+            )
+
+        syllable_count = self._textstat.syllable_count(cleaned, lang=self._language)
+        reading_ease = float(self._textstat.flesch_reading_ease(cleaned))
+        fk_grade = float(self._textstat.flesch_kincaid_grade(cleaned))
+
+        words_per_sentence = word_count / sentence_count
+        syllables_per_word = syllable_count / word_count if word_count else 0.0
+        within_bounds = self._is_within_grade_bounds(fk_grade)
+
+        if self._enforce_bounds:
+            value = 1.0 if within_bounds else 0.0
+            reason = (
+                "Text meets readability targets"
+                if within_bounds
+                else "Text falls outside readability targets"
+            )
+        else:
+            normalised = max(0.0, min(100.0, reading_ease)) / 100.0
+            value = normalised
+            reason = (
+                f"Flesch Reading Ease: {reading_ease:.2f} | "
+                f"Flesch-Kincaid Grade: {fk_grade:.2f}"
+            )
+
+        metadata = {
+            "flesch_reading_ease": reading_ease,
+            "flesch_kincaid_grade": fk_grade,
+            "words_per_sentence": words_per_sentence,
+            "syllables_per_word": syllables_per_word,
+            "sentence_count": sentence_count,
+            "word_count": word_count,
+            "syllable_count": syllable_count,
+            "min_grade": self._min_grade,
+            "max_grade": self._max_grade,
+            "within_grade_bounds": within_bounds,
+        }
+
+        return ScoreResult(
+            value=value,
+            name=self.name,
+            reason=reason,
+            metadata=metadata,
+        )
+
+    def _is_within_grade_bounds(self, grade: float) -> bool:
+        if self._min_grade is not None and grade < self._min_grade:
+            return False
+        if self._max_grade is not None and grade > self._max_grade:
+            return False
+        return True

opik/evaluation/metrics/heuristics/regex_match.py

@@ -1,5 +1,5 @@
 import re
-from typing import Any, Union
+from typing import Any, Union, Optional
 
 from .. import base_metric, score_result
 
@@ -15,6 +15,7 @@ class RegexMatch(base_metric.BaseMetric):
         regex: The regular expression pattern to match against. Can be a string or a compiled regex pattern.
         name: The name of the metric. Defaults to "regex_match_metric".
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when there are no parent span/trace to inherit project name from.
 
     Example:
         >>> from opik.evaluation.metrics import RegexMatch
@@ -32,10 +33,12 @@ class RegexMatch(base_metric.BaseMetric):
         regex: Union[str, re.Pattern],
         name: str = "regex_match_metric",
         track: bool = True,
+        project_name: Optional[str] = None,
     ):
         super().__init__(
             name=name,
             track=track,
+            project_name=project_name,
         )
 
         self._regex_pattern: re.Pattern = (

opik/evaluation/metrics/heuristics/rouge.py

@@ -0,0 +1,148 @@
+from typing import Any, List, Optional, Union
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+try:
+    from rouge_score import rouge_scorer
+except ImportError:
+    rouge_scorer = None
+
+
+class ROUGE(base_metric.BaseMetric):
+    """
+    A metric that computes the ROUGE, or Recall-Oriented Understudy for Gisting Evaluation score between an output and reference string mainly used for evaluating text summarization.
+    ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
+    This metrics is a wrapper around the Google Research reimplementation of ROUGE, which is based on the `rouge-score` library.
+
+    References:
+        - https://github.com/google-research/google-research/tree/master/rouge
+        - https://huggingface.co/spaces/evaluate-metric/rouge
+
+    Args:
+        name: The name of the metric. Defaults to "rouge_metric".
+        track: Whether to track the metric. Defaults to True.
+        rouge_type: Type of ROUGE score to compute. Defaults to "rouge1". Must be one of the following:
+                    - "rouge1": unigram (1-gram) based scoring
+                    - "rouge2": bigram (2-gram) based scoring
+                    - "rougeL": Longest common subsequence based scoring
+                    - "rougeLSum": splits text using '\\n'"
+        use_stemmer: Whether to use stemming when computing ROUGE. Defaults to False.
+        split_summaries: Whether to split summaries into sentences. Defaults to False.
+        tokenizer: A tokenizer to use when splitting summaries into sentences. Defaults to None.
+        project_name: Optional project name to track the metric in for the cases when there are no parent span/trace to inherit project name from.
+
+    Example:
+        >>> from opik.evaluation.metrics import ROUGE
+        >>> rouge_metric = ROUGE()
+        >>> result = rouge_metric.score(
+        ...     output="The quick brown fox jumps over the lazy dog.",
+        ...     reference="The quick brown fox jumps over the lazy dog."
+        ... )
+        >>> print(result.value)
+        1.0
+    """
+
+    def __init__(
+        self,
+        name: str = "rouge_metric",
+        track: bool = True,
+        rouge_type: str = "rouge1",
+        use_stemmer: bool = False,
+        split_summaries: bool = False,
+        tokenizer: Optional[Any] = None,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+
+        valid_rouge_types = {"rouge1", "rouge2", "rougeL", "rougeLsum"}
+        if rouge_type not in valid_rouge_types:
+            raise MetricComputationError(
+                f"Invalid rouge_type '{rouge_type}'. Must be one of {valid_rouge_types}."
+            )
+
+        self._rouge_type = rouge_type
+        self._rouge = _build_rouge_backend(
+            rouge_type=rouge_type,
+            use_stemmer=use_stemmer,
+            split_summaries=split_summaries,
+            tokenizer=tokenizer,
+        )
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, List[str]],
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        """
+        Compute the ROUGE score based on the given rouge_type between the output and reference strings.
+
+        Args:
+            output: The output string to score.
+            reference: The reference string or list of reference strings.
+            **ignored_kwargs: Additional keyword arguments that are ignored.
+
+        Returns:
+            score_result.ScoreResult with:
+              - `value`: The ROUGE score (float).
+              - `name`: The metric name.
+              - `reason`: A short explanation (e.g. "rouge1 score: 0.91").
+
+        Raises:
+            MetricComputationError:
+                - If the candidate or any reference is empty.
+                - If the reference is not a string or a list of strings.
+        """
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty.")
+
+        if isinstance(reference, str):
+            if not reference.strip():
+                raise MetricComputationError("Reference is empty.")
+
+            reference = [reference]
+        elif isinstance(reference, list):
+            if len(reference) == 0:
+                raise MetricComputationError("Reference is empty.")
+
+            if not all(isinstance(item, str) for item in reference):
+                raise MetricComputationError(
+                    "Reference must be a string or a list of strings."
+                )
+
+            for ref_str in reference:
+                if not ref_str.strip():
+                    raise MetricComputationError("Encountered empty reference.")
+
+        rouge_score_type = self._rouge_type
+        if self._rouge is None:
+            raise MetricComputationError("ROUGE backend is not initialized.")
+        results = self._rouge.score_multi(reference, output)
+        rouge_f1_value = results[rouge_score_type].fmeasure
+
+        return score_result.ScoreResult(
+            value=rouge_f1_value,
+            name=self.name,
+            reason=f"{rouge_score_type} score: {rouge_f1_value:.4f}",
+        )
+
+
+def _build_rouge_backend(
+    *,
+    rouge_type: str,
+    use_stemmer: bool,
+    split_summaries: bool,
+    tokenizer: Optional[Any],
+) -> Optional[Any]:
+    if rouge_scorer is None:
+        raise ImportError(
+            "`rouge-score` libraries are required for ROUGE score calculation. "
+            "Install via `pip install rouge-score`."
+        )
+
+    return rouge_scorer.RougeScorer(
+        [rouge_type],
+        use_stemmer=use_stemmer,
+        split_summaries=split_summaries,
+        tokenizer=tokenizer,
+    )

opik/evaluation/metrics/heuristics/sentiment.py

@@ -0,0 +1,98 @@
+from typing import Any, Optional
+
+from opik.evaluation.metrics import base_metric, score_result
+from opik.exceptions import MetricComputationError
+
+try:
+    import nltk
+    from nltk.sentiment import vader
+except ImportError:
+    nltk = None
+    vader = None
+
+
+class Sentiment(base_metric.BaseMetric):
+    """
+    A metric that analyzes the sentiment of text using NLTK's VADER sentiment analyzer.
+
+    Returns sentiment scores for positive, neutral, negative, and compound sentiment.
+    The compound score is a normalized score between -1.0 (extremely negative) and
+    1.0 (extremely positive).
+
+    Args:
+        name: The name of the metric. Defaults to "sentiment_metric".
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
+
+    Example:
+        >>> from opik.evaluation.metrics import Sentiment
+        >>> sentiment_metric = Sentiment()
+        >>> result = sentiment_metric.score("I love this product! It's amazing.")
+        >>> print(result.value)  # Compound score (e.g., 0.8802)
+        >>> print(result.metadata)  # All sentiment scores
+    """
+
+    def __init__(
+        self,
+        name: str = "sentiment_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(
+            name=name,
+            track=track,
+            project_name=project_name,
+        )
+
+        if nltk is None or vader is None:
+            raise ImportError(
+                "`nltk` library is required for sentiment analysis. "
+                "Install via `pip install nltk` and then download the vader_lexicon: "
+                "`python -m nltk.downloader vader_lexicon`."
+            )
+
+        try:
+            self._analyzer = vader.SentimentIntensityAnalyzer()
+        except LookupError:
+            # If vader_lexicon is not downloaded, attempt to download it
+            nltk.download("vader_lexicon")
+            self._analyzer = vader.SentimentIntensityAnalyzer()
+
+    def score(self, output: str, **ignored_kwargs: Any) -> score_result.ScoreResult:
+        """
+        Analyze the sentiment of the provided text.
+
+        Args:
+            output: The text to analyze for sentiment.
+            **ignored_kwargs: Additional keyword arguments that are ignored.
+
+        Returns:
+            score_result.ScoreResult: A ScoreResult object with:
+                - value: The compound sentiment score (-1.0 to 1.0)
+                - name: The metric name
+                - reason: A brief explanation of the sentiment analysis
+                - metadata: Dictionary containing all sentiment scores (pos, neu, neg, compound)
+
+        Raises:
+            MetricComputationError: If the input text is empty.
+        """
+        if not output.strip():
+            raise MetricComputationError("Empty text provided for sentiment analysis.")
+
+        sentiment_scores = self._analyzer.polarity_scores(output)
+        compound_score = sentiment_scores["compound"]
+
+        if compound_score >= 0.05:
+            sentiment_category = "positive"
+        elif compound_score <= -0.05:
+            sentiment_category = "negative"
+        else:
+            sentiment_category = "neutral"
+
+        return score_result.ScoreResult(
+            value=compound_score,
+            name=self.name,
+            reason=f"Text sentiment analysis: {sentiment_category} (compound score: {compound_score:.4f})",
+            metadata=sentiment_scores,
+        )

opik/evaluation/metrics/heuristics/spearman.py

@@ -0,0 +1,88 @@
+"""Spearman rank correlation between reference and predicted rankings."""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+
+
+class SpearmanRanking(BaseMetric):
+    """
+    Compute Spearman's rank correlation for two rankings of the same items.
+
+    Scores are normalised to ``[0.0, 1.0]`` where `1.0` indicates perfect rank
+    agreement and `0.0` indicates complete disagreement (``rho = -1``).
+
+    References:
+      - Spearman's rank correlation coefficient (Wikipedia overview)
+        https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient
+      - SciPy documentation: ``scipy.stats.spearmanr``
+        https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html
+
+    Args:
+        name: Display name for the metric result. Defaults to
+            ``"spearman_ranking_metric"``.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+
+    Example:
+        >>> from opik.evaluation.metrics import SpearmanRanking
+        >>> metric = SpearmanRanking()
+        >>> result = metric.score(
+        ...     output=["b", "a", "c"],
+        ...     reference=["a", "b", "c"],
+        ... )
+        >>> round(result.metadata["rho"], 2)  # doctest: +SKIP
+        -0.5
+    """
+
+    def __init__(
+        self,
+        name: str = "spearman_ranking_metric",
+        track: bool = True,
+        project_name: str | None = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+
+    def score(
+        self,
+        output: Sequence[Any],
+        reference: Sequence[Any],
+        **ignored_kwargs: Any,
+    ) -> ScoreResult:
+        if len(output) != len(reference):
+            raise MetricComputationError(
+                "output and reference rankings must have the same length."
+            )
+        if len(output) == 0:
+            raise MetricComputationError(
+                "Rankings cannot be empty for Spearman correlation."
+            )
+
+        ref_ranks = {item: idx for idx, item in enumerate(reference)}
+        if set(output) != set(reference):
+            raise MetricComputationError("Rankings must contain the same items.")
+
+        diffs_sq = 0
+        for idx, item in enumerate(output):
+            ref_idx = ref_ranks[item]
+            diffs_sq += (idx - ref_idx) ** 2
+
+        n = len(output)
+        if n == 1:
+            rho = 1.0
+        else:
+            rho = 1 - (6 * diffs_sq) / (n * (n * n - 1))
+
+        # normalize to [0, 1] for convenience
+        normalized = (rho + 1) / 2
+
+        return ScoreResult(
+            value=normalized,
+            name=self.name,
+            reason=f"Spearman correlation (normalized): {normalized:.4f}",
+            metadata={"rho": rho},
+        )

opik/evaluation/metrics/heuristics/tone.py

@@ -0,0 +1,155 @@
+"""Rule-based tone metric for assistant responses."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Iterable, Optional, Sequence
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+
+# Default tone lexicons/phrases kept inline for easier discoverability.
+_POSITIVE_LEXICON = {
+    "appreciate",
+    "assist",
+    "glad",
+    "helpful",
+    "please",
+    "thank",
+    "welcome",
+    "happy",
+    "support",
+    "great",
+    "excellent",
+    "wonderful",
+}
+
+_NEGATIVE_LEXICON = {
+    "angry",
+    "awful",
+    "bad",
+    "complain",
+    "frustrated",
+    "hate",
+    "incompetent",
+    "terrible",
+    "useless",
+    "stupid",
+    "idiot",
+}
+
+_FORBIDDEN_PHRASES = {
+    "shut up",
+    "this is pointless",
+    "not my problem",
+    "i refuse to assist",
+}
+
+
+class Tone(BaseMetric):
+    """
+    Flag tone issues like excessive negativity, shouting, or forbidden phrases.
+
+    Args:
+        name: Display name for the metric result. Defaults to ``"tone_metric"``.
+        track: Whether to automatically track results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+        min_sentiment: Minimum sentiment score required (``-1.0`` to ``1.0`` scale).
+        max_upper_ratio: Maximum allowed ratio of uppercase characters.
+        max_exclamations: Cap on the number of exclamation marks.
+        positive_lexicon: Optional iterable of positive tokens counted for sentiment.
+        negative_lexicon: Optional iterable of negative tokens counted for sentiment.
+        forbidden_phrases: Optional sequence of phrases that immediately fail the
+            check.
+
+    Example:
+        >>> from opik.evaluation.metrics import Tone
+        >>> metric = Tone(max_exclamations=2)
+        >>> result = metric.score("THANK YOU for your patience!!!")
+        >>> result.value  # doctest: +SKIP
+        0.0
+    """
+
+    def __init__(
+        self,
+        name: str = "tone_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        min_sentiment: float = -0.2,
+        max_upper_ratio: float = 0.3,
+        max_exclamations: int = 3,
+        positive_lexicon: Optional[Iterable[str]] = None,
+        negative_lexicon: Optional[Iterable[str]] = None,
+        forbidden_phrases: Optional[Sequence[str]] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._min_sentiment = min_sentiment
+        self._max_upper_ratio = max_upper_ratio
+        self._max_exclamations = max_exclamations
+        self._positive = set(
+            word.lower() for word in (positive_lexicon or _POSITIVE_LEXICON)
+        )
+        self._negative = set(
+            word.lower() for word in (negative_lexicon or _NEGATIVE_LEXICON)
+        )
+        phrases = forbidden_phrases or _FORBIDDEN_PHRASES
+        self._forbidden = [phrase.lower() for phrase in phrases]
+
+    def score(self, output: str, **ignored_kwargs: Any) -> ScoreResult:
+        if not output or not output.strip():
+            raise MetricComputationError("Text is empty (Tone metric).")
+
+        tokens = re.findall(r"\b\w+\b", output.lower())
+        if not tokens:
+            raise MetricComputationError("Unable to tokenize text for Tone metric.")
+
+        sentiment_score = self._compute_sentiment(tokens)
+        upper_ratio = _uppercase_ratio(output)
+        exclamation_count = output.count("!")
+        forbidden_hit = any(phrase in output.lower() for phrase in self._forbidden)
+
+        passes = (
+            sentiment_score >= self._min_sentiment
+            and upper_ratio <= self._max_upper_ratio
+            and exclamation_count <= self._max_exclamations
+            and not forbidden_hit
+        )
+
+        metadata = {
+            "sentiment_score": sentiment_score,
+            "uppercase_ratio": upper_ratio,
+            "exclamation_count": exclamation_count,
+            "forbidden_hit": forbidden_hit,
+            "thresholds": {
+                "min_sentiment": self._min_sentiment,
+                "max_upper_ratio": self._max_upper_ratio,
+                "max_exclamations": self._max_exclamations,
+            },
+        }
+
+        reason = (
+            "Tone is within configured guardrails"
+            if passes
+            else "Tone violates guardrails"
+        )
+        value = 1.0 if passes else 0.0
+        return ScoreResult(
+            value=value, name=self.name, reason=reason, metadata=metadata
+        )
+
+    def _compute_sentiment(self, tokens: Sequence[str]) -> float:
+        pos_hits = sum(token in self._positive for token in tokens)
+        neg_hits = sum(token in self._negative for token in tokens)
+        total = pos_hits + neg_hits
+        if total == 0:
+            return 0.0
+        return (pos_hits - neg_hits) / total
+
+
+def _uppercase_ratio(text: str) -> float:
+    letters = [char for char in text if char.isalpha()]
+    if not letters:
+        return 0.0
+    upper = sum(1 for char in letters if char.isupper())
+    return upper / len(letters)