opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/heuristics/bertscore.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Sequence, Tuple, Union
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+try:  # pragma: no cover - optional dependency
+    from bert_score import score as bert_score_fn
+except ImportError:  # pragma: no cover - optional dependency
+    bert_score_fn = None
+
+
+BertScoreFn = Callable[
+    [Sequence[str], Union[Sequence[str], Sequence[Sequence[str]]]], Tuple[Any, Any, Any]
+]
+
+
+class BERTScore(base_metric.BaseMetric):
+    """Wrapper around the `bert-score` library.
+
+    Args:
+        scorer_fn: Optional callable compatible with ``bert_score.score`` for
+            dependency injection or advanced usage.
+        model_type: Model checkpoint to use when loading the scorer. Ignored when
+            ``scorer_fn`` is provided.
+        lang: Two-letter language code used by the default scorer.
+        rescale_with_baseline: Whether to rescale the score using the provided
+            baseline statistics.
+        device: Optional device string forwarded to ``bert_score`` (e.g., "cpu",
+            "cuda").
+    """
+
+    def __init__(
+        self,
+        scorer_fn: Optional[BertScoreFn] = None,
+        model_type: Optional[str] = "bert-base-uncased",
+        lang: Optional[str] = "en",
+        rescale_with_baseline: bool = False,
+        device: Optional[str] = None,
+        name: str = "bertscore_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        **scorer_kwargs: Any,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+
+        if scorer_fn is not None:
+            self._scorer_fn = scorer_fn
+        else:
+            if bert_score_fn is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "BERTScore metric requires the optional 'bert-score' package. "
+                    "Install via `pip install bert-score` or provide `scorer_fn`."
+                )
+
+            def _score(
+                candidates: Sequence[str],
+                references: Union[Sequence[str], Sequence[Sequence[str]]],
+            ) -> Tuple[Any, Any, Any]:
+                return bert_score_fn(
+                    candidates,
+                    references,
+                    model_type=model_type,
+                    lang=lang,
+                    rescale_with_baseline=rescale_with_baseline,
+                    device=device,
+                    **scorer_kwargs,
+                )
+
+            self._scorer_fn = _score
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str], Sequence[Sequence[str]]],
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (BERTScore metric).")
+
+        references: Union[Sequence[str], Sequence[Sequence[str]]]
+        if isinstance(reference, str):
+            references = [reference]
+        else:
+            references = reference
+            if isinstance(reference, Sequence) and len(reference) == 0:
+                raise MetricComputationError("Reference is empty (BERTScore metric).")
+
+        precision, recall, f1 = self._scorer_fn([output], references)
+
+        score_value = float(f1[0].item() if hasattr(f1[0], "item") else f1[0])
+        metadata = {
+            "precision": float(
+                precision[0].item() if hasattr(precision[0], "item") else precision[0]
+            ),
+            "recall": float(
+                recall[0].item() if hasattr(recall[0], "item") else recall[0]
+            ),
+        }
+
+        return score_result.ScoreResult(
+            value=score_value,
+            name=self.name,
+            reason=f"BERTScore F1: {score_value:.4f}",
+            metadata=metadata,
+        )

opik/evaluation/metrics/heuristics/bleu.py

@@ -1,4 +1,6 @@
-from typing import Any, List, Optional, Tuple, Union
+import warnings
+from contextlib import contextmanager
+from typing import Any, Iterator, List, Optional, Tuple, Union
 
 from opik.exceptions import MetricComputationError
 from opik.evaluation.metrics import base_metric, score_result
@@ -15,8 +17,12 @@ class BaseBLEU(base_metric.BaseMetric):
     and weights initialization. This class is not intended to be used directly.
 
     References:
-      - NLTK BLEU smoothing:
-        https://www.nltk.org/api/nltk.translate.bleu_score.html#nltk.translate.bleu_score.SmoothingFunction
+      - BLEU: Papineni et al., "BLEU: a Method for Automatic Evaluation of Machine Translation" (ACL 2002)
+        https://aclanthology.org/P02-1040/
+      - NLTK BLEU documentation
+        https://www.nltk.org/api/nltk.translate.bleu_score.html
+      - Hugging Face Evaluate: BLEU metric overview
+        https://huggingface.co/spaces/evaluate-metric/bleu
 
     Args:
         name: The name of the metric (e.g. "sentence_bleu_metric" or "corpus_bleu_metric").
@@ -73,6 +79,18 @@ class BaseBLEU(base_metric.BaseMetric):
         return tuple(normalized)
 
 
+@contextmanager
+def _suppress_bleu_warnings() -> Iterator[None]:
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=r"The hypothesis contains 0 counts of 2-gram overlaps\.",
+            category=UserWarning,
+            module="nltk\\.translate\\.bleu_score",
+        )
+        yield
+
+
 class SentenceBLEU(BaseBLEU):
     """
     Computes sentence-level BLEU for a single candidate string vs. one or more references.
@@ -151,12 +169,13 @@ class SentenceBLEU(BaseBLEU):
         smoothing_func = self._get_smoothing_func()
 
         try:
-            bleu_val = nltk_bleu_score.sentence_bleu(
-                ref_lists,
-                candidate_tokens,
-                weights=used_weights,
-                smoothing_function=smoothing_func,
-            )
+            with _suppress_bleu_warnings():
+                bleu_val = nltk_bleu_score.sentence_bleu(
+                    ref_lists,
+                    candidate_tokens,
+                    weights=used_weights,
+                    smoothing_function=smoothing_func,
+                )
         except ZeroDivisionError:
             bleu_val = 0.0
 
@@ -268,12 +287,13 @@ class CorpusBLEU(BaseBLEU):
         smoothing_func = self._get_smoothing_func()
 
         try:
-            bleu_val = nltk_bleu_score.corpus_bleu(
-                all_references,
-                all_candidates,
-                weights=used_weights,
-                smoothing_function=smoothing_func,
-            )
+            with _suppress_bleu_warnings():
+                bleu_val = nltk_bleu_score.corpus_bleu(
+                    all_references,
+                    all_candidates,
+                    weights=used_weights,
+                    smoothing_function=smoothing_func,
+                )
         except ZeroDivisionError:
             bleu_val = 0.0
 

opik/evaluation/metrics/heuristics/chrf.py

@@ -0,0 +1,127 @@
+"""Character n-gram F-score (chrF/chrF++) metric wrapper."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Sequence, Union
+
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+
+try:  # pragma: no cover - optional dependency
+    from nltk.translate import chrf_score as nltk_chrf_score
+except ImportError:  # pragma: no cover - optional dependency
+    nltk_chrf_score = None
+
+ChrFFn = Callable[[Sequence[str], Sequence[str]], float]
+
+
+class ChrF(BaseMetric):
+    """
+    Compute chrF / chrF++ scores between a candidate string and references.
+
+    By default the implementation delegates to ``nltk.translate.chrf_score`` and
+    supports both chrF (character n-gram overlap) and chrF++ (when ``word_order``
+    is non-zero). Scores range from `0.0` (no overlap) to `1.0` (perfect match).
+
+    References:
+      - Popović, "chrF: character n-gram F-score for automatic MT evaluation" (WMT 2015)
+        https://aclanthology.org/W15-3049/
+      - NLTK chrf_score module documentation
+        https://www.nltk.org/api/nltk.translate.chrf_score.html
+      - Hugging Face Evaluate: chrF metric overview
+        https://huggingface.co/spaces/evaluate-metric/chrf
+
+    Args:
+        name: Display name for the metric result. Defaults to ``"chrf_metric"``.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+        beta: Weighting between precision and recall (``beta = 2`` is standard).
+        ignore_whitespace: Whether whitespace is ignored before scoring.
+        char_order: Maximum character n-gram order.
+        word_order: Maximum word n-gram order (set ``>0`` to enable chrF++).
+        lowercase: Whether to lowercase candidate and references prior to scoring.
+        chrf_fn: Optional custom scoring callable for testing or offline usage.
+
+    Example:
+        >>> from opik.evaluation.metrics import ChrF
+        >>> metric = ChrF(beta=2.0, char_order=6, lowercase=True)
+        >>> result = metric.score(
+        ...     output="The quick brown fox",
+        ...     reference="The quick brown fox jumps",
+        ... )
+        >>> round(result.value, 4)  # doctest: +SKIP
+        0.8795
+    """
+
+    def __init__(
+        self,
+        name: str = "chrf_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        beta: float = 2.0,
+        ignore_whitespace: bool = False,
+        char_order: int = 6,
+        word_order: int = 0,
+        lowercase: bool = False,
+        chrf_fn: Optional[ChrFFn] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._beta = beta
+        self._ignore_whitespace = ignore_whitespace
+        self._char_order = char_order
+        self._word_order = word_order
+        self._lowercase = lowercase
+
+        if chrf_fn is not None:
+            self._chrf_fn = chrf_fn
+        else:
+            if nltk_chrf_score is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "chrF metric requires the optional 'nltk' package. Install via"
+                    " `pip install nltk` or provide `chrf_fn`."
+                )
+
+            def _compute(candidate: Sequence[str], references: Sequence[str]) -> float:
+                try:
+                    return float(
+                        nltk_chrf_score.sentence_chrf(
+                            references,
+                            candidate,
+                            beta=self._beta,
+                        )
+                    )
+                except TypeError:
+                    # Older NLTK versions expose the helper with fewer keyword arguments.
+                    return float(nltk_chrf_score.sentence_chrf(references, candidate))
+
+            self._chrf_fn = _compute
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str]],
+        **ignored_kwargs: Any,
+    ) -> ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (chrF metric).")
+        if isinstance(reference, str):
+            references = [reference]
+        else:
+            references = list(reference)
+        if not references or any(not ref.strip() for ref in references):
+            raise MetricComputationError("Reference is empty (chrF metric).")
+
+        if self._lowercase:
+            output_text = output.lower()
+            references = [ref.lower() for ref in references]
+        else:
+            output_text = output
+
+        value = self._chrf_fn(output_text, references)
+
+        return ScoreResult(
+            value=float(value),
+            name=self.name,
+            reason=f"chrF score: {float(value):.4f}",
+        )

opik/evaluation/metrics/heuristics/contains.py

@@ -12,24 +12,44 @@ class Contains(base_metric.BaseMetric):
 
     Args:
         case_sensitive: Whether the comparison should be case-sensitive. Defaults to False.
+        reference: Optional default reference string. If provided, it will be used unless
+            a reference is explicitly passed to `score()`.
         name: The name of the metric. Defaults to "contains_metric".
         track: Whether to track the metric. Defaults to True.
-        project_name: Optional project name to track the metric in for the cases when there are no parent span/trace to inherit project name from.
+        project_name: Optional project name to track the metric in for the cases when there are
+            no parent span/trace to inherit project name from.
 
-    Example:
-        >>> from opik.evaluation.metrics import Contains
-        >>> contains_metric = Contains(case_sensitive=True)
-        >>> result = contains_metric.score("Hello, World!", "World")
+    Examples:
+        >>> # Using a default reference at initialization
+        >>> contains_metric = Contains(reference="world")
+        >>> result = contains_metric.score("Hello, World!")
         >>> print(result.value)
         1.0
-        >>> result = contains_metric.score("Hello, World!", "world")
+
+        >>> # Overriding the default reference at score time
+        >>> result = contains_metric.score("Hello, World!", reference="there")
         >>> print(result.value)
         0.0
+
+        >>> # If no reference is set at all, score() raises an error
+        >>> contains_metric = Contains()
+        >>> contains_metric.score("Hello")
+        Traceback (most recent call last):
+            ...
+        ValueError: No reference string provided. Either pass `reference` to `score()` or set a default reference when creating the metric.
+
+        >>> # Empty reference string is invalid
+        >>> contains_metric = Contains(reference="")
+        >>> contains_metric.score("Hello")
+        Traceback (most recent call last):
+            ...
+        ValueError: Invalid reference string provided. Reference must be a non-empty string.
     """
 
     def __init__(
         self,
         case_sensitive: bool = False,
+        reference: Optional[str] = None,
         name: str = "contains_metric",
         track: bool = True,
         project_name: Optional[str] = None,
@@ -39,28 +59,44 @@ class Contains(base_metric.BaseMetric):
             track=track,
             project_name=project_name,
         )
-
         self._case_sensitive = case_sensitive
+        self._default_reference = reference
 
     def score(
-        self, output: str, reference: str, **ignored_kwargs: Any
+        self, output: str, reference: Optional[str] = None, **ignored_kwargs: Any
     ) -> score_result.ScoreResult:
         """
         Calculate the score based on whether the reference string is contained in the output string.
 
         Args:
             output: The output string to check.
-            reference: The reference string to look for in the output.
+            reference: The reference string to look for in the output. If None, falls back to the
+                default reference provided at initialization.
             **ignored_kwargs: Additional keyword arguments that are ignored.
 
         Returns:
             score_result.ScoreResult: A ScoreResult object with a value of 1.0 if the reference
                 is found in the output, 0.0 otherwise.
         """
+        # Use provided reference, else fall back to default
+        ref = reference if reference is not None else self._default_reference
+
+        # Handle missing reference (None) separately
+        if ref is None:
+            raise ValueError(
+                "No reference string provided. Either pass `reference` to `score()` or set a default reference when creating the metric."
+            )
+
+        # Handle empty string separately
+        if ref == "":
+            raise ValueError(
+                "Invalid reference string provided. Reference must be a non-empty string."
+            )
+
         value = output if self._case_sensitive else output.lower()
-        reference = reference if self._case_sensitive else reference.lower()
+        ref = ref if self._case_sensitive else ref.lower()
 
-        if reference in value:
+        if ref in value:
             return score_result.ScoreResult(value=1.0, name=self.name)
 
         return score_result.ScoreResult(value=0.0, name=self.name)