opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/heuristics/bleu.py

@@ -1,4 +1,6 @@
-from typing import Any, List, Optional, Tuple, Union
+import warnings
+from contextlib import contextmanager
+from typing import Any, Iterator, List, Optional, Tuple, Union
 
 from opik.exceptions import MetricComputationError
 from opik.evaluation.metrics import base_metric, score_result
@@ -15,8 +17,12 @@ class BaseBLEU(base_metric.BaseMetric):
     and weights initialization. This class is not intended to be used directly.
 
     References:
-      - NLTK BLEU smoothing:
-        https://www.nltk.org/api/nltk.translate.bleu_score.html#nltk.translate.bleu_score.SmoothingFunction
+      - BLEU: Papineni et al., "BLEU: a Method for Automatic Evaluation of Machine Translation" (ACL 2002)
+        https://aclanthology.org/P02-1040/
+      - NLTK BLEU documentation
+        https://www.nltk.org/api/nltk.translate.bleu_score.html
+      - Hugging Face Evaluate: BLEU metric overview
+        https://huggingface.co/spaces/evaluate-metric/bleu
 
     Args:
         name: The name of the metric (e.g. "sentence_bleu_metric" or "corpus_bleu_metric").
@@ -25,6 +31,8 @@ class BaseBLEU(base_metric.BaseMetric):
         smoothing_method: One of NLTK's SmoothingFunction methods (e.g. "method0", "method1", etc.).
         weights: Optional custom weights for n-gram orders. Must sum to 1.0. If None,
                  defaults to uniform distribution across `n_grams`.
+        project_name: Optional project name to track the metric in for the cases when
+            there are no parent span/trace to inherit project name from.
     """
 
     def __init__(
@@ -34,8 +42,9 @@ class BaseBLEU(base_metric.BaseMetric):
         n_grams: int,
         smoothing_method: str,
         weights: Optional[List[float]],
+        project_name: Optional[str],
     ):
-        super().__init__(name=name, track=track)
+        super().__init__(name=name, track=track, project_name=project_name)
 
         if nltk_bleu_score is None:
             raise ImportError(
@@ -70,6 +79,18 @@ class BaseBLEU(base_metric.BaseMetric):
         return tuple(normalized)
 
 
+@contextmanager
+def _suppress_bleu_warnings() -> Iterator[None]:
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=r"The hypothesis contains 0 counts of 2-gram overlaps\.",
+            category=UserWarning,
+            module="nltk\\.translate\\.bleu_score",
+        )
+        yield
+
+
 class SentenceBLEU(BaseBLEU):
     """
     Computes sentence-level BLEU for a single candidate string vs. one or more references.
@@ -89,6 +110,7 @@ class SentenceBLEU(BaseBLEU):
         n_grams: int = 4,
         smoothing_method: str = "method1",
         weights: Optional[List[float]] = None,
+        project_name: Optional[str] = None,
     ):
         super().__init__(
             name=name,
@@ -96,6 +118,7 @@ class SentenceBLEU(BaseBLEU):
             n_grams=n_grams,
             smoothing_method=smoothing_method,
             weights=weights,
+            project_name=project_name,
         )
 
     def score(
@@ -146,12 +169,13 @@ class SentenceBLEU(BaseBLEU):
         smoothing_func = self._get_smoothing_func()
 
         try:
-            bleu_val = nltk_bleu_score.sentence_bleu(
-                ref_lists,
-                candidate_tokens,
-                weights=used_weights,
-                smoothing_function=smoothing_func,
-            )
+            with _suppress_bleu_warnings():
+                bleu_val = nltk_bleu_score.sentence_bleu(
+                    ref_lists,
+                    candidate_tokens,
+                    weights=used_weights,
+                    smoothing_function=smoothing_func,
+                )
         except ZeroDivisionError:
             bleu_val = 0.0
 
@@ -190,6 +214,7 @@ class CorpusBLEU(BaseBLEU):
         n_grams: int = 4,
         smoothing_method: str = "method1",
         weights: Optional[List[float]] = None,
+        project_name: Optional[str] = None,
     ):
         super().__init__(
             name=name,
@@ -197,6 +222,7 @@ class CorpusBLEU(BaseBLEU):
             n_grams=n_grams,
             smoothing_method=smoothing_method,
             weights=weights,
+            project_name=project_name,
         )
 
     def score(
@@ -261,12 +287,13 @@ class CorpusBLEU(BaseBLEU):
         smoothing_func = self._get_smoothing_func()
 
         try:
-            bleu_val = nltk_bleu_score.corpus_bleu(
-                all_references,
-                all_candidates,
-                weights=used_weights,
-                smoothing_function=smoothing_func,
-            )
+            with _suppress_bleu_warnings():
+                bleu_val = nltk_bleu_score.corpus_bleu(
+                    all_references,
+                    all_candidates,
+                    weights=used_weights,
+                    smoothing_function=smoothing_func,
+                )
         except ZeroDivisionError:
             bleu_val = 0.0
 

opik/evaluation/metrics/heuristics/chrf.py

@@ -0,0 +1,127 @@
+"""Character n-gram F-score (chrF/chrF++) metric wrapper."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Sequence, Union
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+
+try:  # pragma: no cover - optional dependency
+    from nltk.translate import chrf_score as nltk_chrf_score
+except ImportError:  # pragma: no cover - optional dependency
+    nltk_chrf_score = None
+
+ChrFFn = Callable[[Sequence[str], Sequence[str]], float]
+
+
+class ChrF(BaseMetric):
+    """
+    Compute chrF / chrF++ scores between a candidate string and references.
+
+    By default the implementation delegates to ``nltk.translate.chrf_score`` and
+    supports both chrF (character n-gram overlap) and chrF++ (when ``word_order``
+    is non-zero). Scores range from `0.0` (no overlap) to `1.0` (perfect match).
+
+    References:
+      - Popović, "chrF: character n-gram F-score for automatic MT evaluation" (WMT 2015)
+        https://aclanthology.org/W15-3049/
+      - NLTK chrf_score module documentation
+        https://www.nltk.org/api/nltk.translate.chrf_score.html
+      - Hugging Face Evaluate: chrF metric overview
+        https://huggingface.co/spaces/evaluate-metric/chrf
+
+    Args:
+        name: Display name for the metric result. Defaults to ``"chrf_metric"``.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+        beta: Weighting between precision and recall (``beta = 2`` is standard).
+        ignore_whitespace: Whether whitespace is ignored before scoring.
+        char_order: Maximum character n-gram order.
+        word_order: Maximum word n-gram order (set ``>0`` to enable chrF++).
+        lowercase: Whether to lowercase candidate and references prior to scoring.
+        chrf_fn: Optional custom scoring callable for testing or offline usage.
+
+    Example:
+        >>> from opik.evaluation.metrics import ChrF
+        >>> metric = ChrF(beta=2.0, char_order=6, lowercase=True)
+        >>> result = metric.score(
+        ...     output="The quick brown fox",
+        ...     reference="The quick brown fox jumps",
+        ... )
+        >>> round(result.value, 4)  # doctest: +SKIP
+        0.8795
+    """
+
+    def __init__(
+        self,
+        name: str = "chrf_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        beta: float = 2.0,
+        ignore_whitespace: bool = False,
+        char_order: int = 6,
+        word_order: int = 0,
+        lowercase: bool = False,
+        chrf_fn: Optional[ChrFFn] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._beta = beta
+        self._ignore_whitespace = ignore_whitespace
+        self._char_order = char_order
+        self._word_order = word_order
+        self._lowercase = lowercase
+
+        if chrf_fn is not None:
+            self._chrf_fn = chrf_fn
+        else:
+            if nltk_chrf_score is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "chrF metric requires the optional 'nltk' package. Install via"
+                    " `pip install nltk` or provide `chrf_fn`."
+                )
+
+            def _compute(candidate: Sequence[str], references: Sequence[str]) -> float:
+                try:
+                    return float(
+                        nltk_chrf_score.sentence_chrf(
+                            references,
+                            candidate,
+                            beta=self._beta,
+                        )
+                    )
+                except TypeError:
+                    # Older NLTK versions expose the helper with fewer keyword arguments.
+                    return float(nltk_chrf_score.sentence_chrf(references, candidate))
+
+            self._chrf_fn = _compute
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str]],
+        **ignored_kwargs: Any,
+    ) -> ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (chrF metric).")
+        if isinstance(reference, str):
+            references = [reference]
+        else:
+            references = list(reference)
+        if not references or any(not ref.strip() for ref in references):
+            raise MetricComputationError("Reference is empty (chrF metric).")
+
+        if self._lowercase:
+            output_text = output.lower()
+            references = [ref.lower() for ref in references]
+        else:
+            output_text = output
+
+        value = self._chrf_fn(output_text, references)
+
+        return ScoreResult(
+            value=float(value),
+            name=self.name,
+            reason=f"chrF score: {float(value):.4f}",
+        )

opik/evaluation/metrics/heuristics/contains.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Optional
 
 from .. import base_metric, score_result
 
@@ -12,52 +12,91 @@ class Contains(base_metric.BaseMetric):
 
     Args:
         case_sensitive: Whether the comparison should be case-sensitive. Defaults to False.
+        reference: Optional default reference string. If provided, it will be used unless
+            a reference is explicitly passed to `score()`.
         name: The name of the metric. Defaults to "contains_metric".
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when there are
+            no parent span/trace to inherit project name from.
 
-    Example:
-        >>> from opik.evaluation.metrics import Contains
-        >>> contains_metric = Contains(case_sensitive=True)
-        >>> result = contains_metric.score("Hello, World!", "World")
+    Examples:
+        >>> # Using a default reference at initialization
+        >>> contains_metric = Contains(reference="world")
+        >>> result = contains_metric.score("Hello, World!")
         >>> print(result.value)
         1.0
-        >>> result = contains_metric.score("Hello, World!", "world")
+
+        >>> # Overriding the default reference at score time
+        >>> result = contains_metric.score("Hello, World!", reference="there")
         >>> print(result.value)
         0.0
+
+        >>> # If no reference is set at all, score() raises an error
+        >>> contains_metric = Contains()
+        >>> contains_metric.score("Hello")
+        Traceback (most recent call last):
+            ...
+        ValueError: No reference string provided. Either pass `reference` to `score()` or set a default reference when creating the metric.
+
+        >>> # Empty reference string is invalid
+        >>> contains_metric = Contains(reference="")
+        >>> contains_metric.score("Hello")
+        Traceback (most recent call last):
+            ...
+        ValueError: Invalid reference string provided. Reference must be a non-empty string.
     """
 
     def __init__(
         self,
         case_sensitive: bool = False,
+        reference: Optional[str] = None,
         name: str = "contains_metric",
         track: bool = True,
+        project_name: Optional[str] = None,
     ):
         super().__init__(
             name=name,
             track=track,
+            project_name=project_name,
         )
-
         self._case_sensitive = case_sensitive
+        self._default_reference = reference
 
     def score(
-        self, output: str, reference: str, **ignored_kwargs: Any
+        self, output: str, reference: Optional[str] = None, **ignored_kwargs: Any
     ) -> score_result.ScoreResult:
         """
         Calculate the score based on whether the reference string is contained in the output string.
 
         Args:
             output: The output string to check.
-            reference: The reference string to look for in the output.
+            reference: The reference string to look for in the output. If None, falls back to the
+                default reference provided at initialization.
             **ignored_kwargs: Additional keyword arguments that are ignored.
 
         Returns:
             score_result.ScoreResult: A ScoreResult object with a value of 1.0 if the reference
                 is found in the output, 0.0 otherwise.
         """
+        # Use provided reference, else fall back to default
+        ref = reference if reference is not None else self._default_reference
+
+        # Handle missing reference (None) separately
+        if ref is None:
+            raise ValueError(
+                "No reference string provided. Either pass `reference` to `score()` or set a default reference when creating the metric."
+            )
+
+        # Handle empty string separately
+        if ref == "":
+            raise ValueError(
+                "Invalid reference string provided. Reference must be a non-empty string."
+            )
+
         value = output if self._case_sensitive else output.lower()
-        reference = reference if self._case_sensitive else reference.lower()
+        ref = ref if self._case_sensitive else ref.lower()
 
-        if reference in value:
+        if ref in value:
             return score_result.ScoreResult(value=1.0, name=self.name)
 
         return score_result.ScoreResult(value=0.0, name=self.name)

opik/evaluation/metrics/heuristics/distribution_metrics.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+TokenizeFn = Callable[[str], Iterable[str]]
+
+
+class _JSDistanceFn(Protocol):
+    def __call__(
+        self,
+        p: Sequence[float],
+        q: Sequence[float],
+        base: Optional[
+            float
+        ] = ...,  # matches scipy signature allowing positional or keyword use
+    ) -> float: ...
+
+
+def _load_jensen_shannon_distance() -> _JSDistanceFn:
+    try:
+        from scipy.spatial.distance import jensenshannon
+    except ImportError as error:  # pragma: no cover - optional dependency
+        raise ImportError(
+            "Install scipy via `pip install scipy` to use Jensen-Shannon metrics."
+        ) from error
+
+    return jensenshannon
+
+
+def _default_tokenizer(text: str) -> Iterable[str]:
+    return text.lower().split()
+
+
+class _DistributionMetricBase(base_metric.BaseMetric):
+    """
+    Internal helper for metrics that compare token distributions.
+
+    Args:
+        tokenizer: Optional tokenizer returning an iterable of tokens given text.
+        name: Display name for the metric.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project.
+        normalize: When ``True`` the histogram is converted to probabilities.
+        smoothing: Optional additive constant applied during KL-like computations.
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn],
+        name: str,
+        track: bool,
+        project_name: Optional[str],
+        normalize: bool,
+        smoothing: float = 0.0,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._tokenizer = tokenizer or _default_tokenizer
+        self._normalize = normalize
+        self._smoothing = max(0.0, smoothing)
+
+    def _build_distribution(self, text: str) -> Dict[str, float]:
+        tokens = list(self._tokenizer(text))
+        if len(tokens) == 0:
+            raise MetricComputationError(
+                "Tokenized text is empty (distribution-based metric)."
+            )
+
+        counts = Counter(tokens)
+        if not self._normalize:
+            return {token: float(count) for token, count in counts.items()}
+
+        total = float(sum(counts.values()))
+        return {token: count / total for token, count in counts.items()}
+
+    def _smooth(self, value: float) -> float:
+        if self._smoothing == 0.0:
+            return value
+        return value + self._smoothing
+
+
+class JSDivergence(_DistributionMetricBase):
+    """
+    Compute Jensen–Shannon similarity (``1 - JSD``) between two texts.
+
+    Args:
+        tokenizer: Optional tokenizer function. Defaults to whitespace split.
+        base: Logarithm base used when computing divergence (> ``1.0``).
+        normalize: Whether to normalise token counts to probabilities first.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Note:
+        Requires :mod:`scipy` to be installed.
+
+    Example:
+        >>> from opik.evaluation.metrics import JSDivergence
+        >>> metric = JSDivergence()
+        >>> result = metric.score(
+        ...     output="cat cat sat",
+        ...     reference="cat sat on mat",
+        ... )
+        >>> round(result.value, 3)  # doctest: +SKIP
+        0.812
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        base: float = 2.0,
+        normalize: bool = True,
+        name: str = "js_divergence_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        if base <= 1.0:
+            raise ValueError("base must be greater than 1.0")
+        super().__init__(
+            tokenizer=tokenizer,
+            name=name,
+            track=track,
+            project_name=project_name,
+            normalize=normalize,
+        )
+        self._base = base
+        self._js_distance_fn = _load_jensen_shannon_distance()
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError(
+                "Candidate is empty (Jensen-Shannon divergence)."
+            )
+        if not reference.strip():
+            raise MetricComputationError(
+                "Reference is empty (Jensen-Shannon divergence)."
+            )
+
+        output_dist = self._build_distribution(output)
+        reference_dist = self._build_distribution(reference)
+
+        divergence = self._js_divergence(output_dist, reference_dist)
+        score = max(0.0, min(1.0, 1.0 - divergence))
+
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason=(
+                f"Jensen-Shannon similarity (base={self._base:g}): {score:.4f} "
+                f"(divergence={divergence:.4f})"
+            ),
+            metadata={
+                "divergence": divergence,
+                "distance": math.sqrt(divergence),
+                "base": self._base,
+            },
+        )
+
+    def _js_divergence(
+        self,
+        p_dist: Dict[str, float],
+        q_dist: Dict[str, float],
+    ) -> float:
+        vocabulary = sorted(set(p_dist) | set(q_dist))
+        if not vocabulary:
+            return 0.0
+
+        p_vector = [p_dist.get(token, 0.0) for token in vocabulary]
+        q_vector = [q_dist.get(token, 0.0) for token in vocabulary]
+
+        p_probs = self._ensure_probability_vector(p_vector)
+        q_probs = self._ensure_probability_vector(q_vector)
+
+        distance = float(self._js_distance_fn(p_probs, q_probs, base=self._base))
+        return distance**2
+
+    def _ensure_probability_vector(self, values: Sequence[float]) -> List[float]:
+        total = sum(values)
+        if total <= 0.0:
+            raise MetricComputationError(
+                "Distribution is empty after tokenisation (Jensen-Shannon metric)."
+            )
+        return [value / total for value in values]
+
+
+class JSDistance(JSDivergence):
+    """
+    Return the raw Jensen–Shannon divergence instead of similarity.
+
+    Args:
+        tokenizer: Optional tokenizer function.
+        base: Logarithm base used for the divergence calculation.
+        normalize: Whether to normalise counts into probabilities.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Example:
+        >>> from opik.evaluation.metrics import JSDistance
+        >>> metric = JSDistance()
+        >>> result = metric.score("a a b", reference="a b b")
+        >>> round(result.value, 3)  # doctest: +SKIP
+        0.188
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        base: float = 2.0,
+        normalize: bool = True,
+        name: str = "js_distance_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            tokenizer=tokenizer,
+            base=base,
+            normalize=normalize,
+            name=name,
+            track=track,
+            project_name=project_name,
+        )
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        similarity = super().score(output=output, reference=reference)
+        metadata = similarity.metadata or {}
+        divergence = float(metadata.get("divergence", 0.0))
+        distance = float(metadata.get("distance", math.sqrt(divergence)))
+        return score_result.ScoreResult(
+            value=divergence,
+            name=self.name,
+            reason=f"Jensen-Shannon divergence (base={self._base:g}): {divergence:.4f}",
+            metadata={
+                "distance": distance,
+                "base": metadata.get("base", self._base),
+            },
+        )
+
+
+class KLDivergence(_DistributionMetricBase):
+    """
+    Compute the (optionally symmetric) KL divergence between token distributions.
+
+    Args:
+        tokenizer: Optional tokenizer function. Defaults to whitespace split.
+        direction: Direction to compute (``"pq"``, ``"qp"``, or ``"avg"`` for
+            symmetric).
+        normalize: Whether to normalise token counts to probabilities first.
+        smoothing: Additive smoothing constant to avoid divide-by-zero.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Example:
+        >>> from opik.evaluation.metrics import KLDivergence
+        >>> metric = KLDivergence(direction="avg")
+        >>> result = metric.score("hello hello world", reference="hello world")
+        >>> round(result.value, 4)  # doctest: +SKIP
+        0.0583
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        direction: str = "pq",
+        normalize: bool = True,
+        smoothing: float = 1e-12,
+        name: str = "kl_divergence_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        if direction not in {"pq", "qp", "avg"}:
+            raise ValueError("direction must be one of {'pq', 'qp', 'avg'}")
+        super().__init__(
+            tokenizer=tokenizer,
+            name=name,
+            track=track,
+            project_name=project_name,
+            normalize=normalize,
+            smoothing=smoothing,
+        )
+        self._direction = direction
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (KL divergence metric).")
+        if not reference.strip():
+            raise MetricComputationError("Reference is empty (KL divergence metric).")
+
+        p_dist = self._build_distribution(output)
+        q_dist = self._build_distribution(reference)
+
+        if self._direction == "pq":
+            divergence = self._kl(p_dist, q_dist)
+        elif self._direction == "qp":
+            divergence = self._kl(q_dist, p_dist)
+        else:
+            divergence = 0.5 * (self._kl(p_dist, q_dist) + self._kl(q_dist, p_dist))
+
+        return score_result.ScoreResult(
+            value=divergence,
+            name=self.name,
+            reason=f"KL divergence ({self._direction}): {divergence:.4f}",
+        )
+
+    def _kl(self, p_dist: Dict[str, float], q_dist: Dict[str, float]) -> float:
+        divergence = 0.0
+        for token, p_val in p_dist.items():
+            p_val = self._smooth(p_val)
+            q_val = self._smooth(q_dist.get(token, 0.0))
+            divergence += p_val * math.log(p_val / q_val)
+        return divergence