opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/heuristics/distribution_metrics.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+TokenizeFn = Callable[[str], Iterable[str]]
+
+
+class _JSDistanceFn(Protocol):
+    def __call__(
+        self,
+        p: Sequence[float],
+        q: Sequence[float],
+        base: Optional[
+            float
+        ] = ...,  # matches scipy signature allowing positional or keyword use
+    ) -> float: ...
+
+
+def _load_jensen_shannon_distance() -> _JSDistanceFn:
+    try:
+        from scipy.spatial.distance import jensenshannon
+    except ImportError as error:  # pragma: no cover - optional dependency
+        raise ImportError(
+            "Install scipy via `pip install scipy` to use Jensen-Shannon metrics."
+        ) from error
+
+    return jensenshannon
+
+
+def _default_tokenizer(text: str) -> Iterable[str]:
+    return text.lower().split()
+
+
+class _DistributionMetricBase(base_metric.BaseMetric):
+    """
+    Internal helper for metrics that compare token distributions.
+
+    Args:
+        tokenizer: Optional tokenizer returning an iterable of tokens given text.
+        name: Display name for the metric.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project.
+        normalize: When ``True`` the histogram is converted to probabilities.
+        smoothing: Optional additive constant applied during KL-like computations.
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn],
+        name: str,
+        track: bool,
+        project_name: Optional[str],
+        normalize: bool,
+        smoothing: float = 0.0,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._tokenizer = tokenizer or _default_tokenizer
+        self._normalize = normalize
+        self._smoothing = max(0.0, smoothing)
+
+    def _build_distribution(self, text: str) -> Dict[str, float]:
+        tokens = list(self._tokenizer(text))
+        if len(tokens) == 0:
+            raise MetricComputationError(
+                "Tokenized text is empty (distribution-based metric)."
+            )
+
+        counts = Counter(tokens)
+        if not self._normalize:
+            return {token: float(count) for token, count in counts.items()}
+
+        total = float(sum(counts.values()))
+        return {token: count / total for token, count in counts.items()}
+
+    def _smooth(self, value: float) -> float:
+        if self._smoothing == 0.0:
+            return value
+        return value + self._smoothing
+
+
+class JSDivergence(_DistributionMetricBase):
+    """
+    Compute Jensen–Shannon similarity (``1 - JSD``) between two texts.
+
+    Args:
+        tokenizer: Optional tokenizer function. Defaults to whitespace split.
+        base: Logarithm base used when computing divergence (> ``1.0``).
+        normalize: Whether to normalise token counts to probabilities first.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Note:
+        Requires :mod:`scipy` to be installed.
+
+    Example:
+        >>> from opik.evaluation.metrics import JSDivergence
+        >>> metric = JSDivergence()
+        >>> result = metric.score(
+        ...     output="cat cat sat",
+        ...     reference="cat sat on mat",
+        ... )
+        >>> round(result.value, 3)  # doctest: +SKIP
+        0.812
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        base: float = 2.0,
+        normalize: bool = True,
+        name: str = "js_divergence_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        if base <= 1.0:
+            raise ValueError("base must be greater than 1.0")
+        super().__init__(
+            tokenizer=tokenizer,
+            name=name,
+            track=track,
+            project_name=project_name,
+            normalize=normalize,
+        )
+        self._base = base
+        self._js_distance_fn = _load_jensen_shannon_distance()
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError(
+                "Candidate is empty (Jensen-Shannon divergence)."
+            )
+        if not reference.strip():
+            raise MetricComputationError(
+                "Reference is empty (Jensen-Shannon divergence)."
+            )
+
+        output_dist = self._build_distribution(output)
+        reference_dist = self._build_distribution(reference)
+
+        divergence = self._js_divergence(output_dist, reference_dist)
+        score = max(0.0, min(1.0, 1.0 - divergence))
+
+        return score_result.ScoreResult(
+            value=score,
+            name=self.name,
+            reason=(
+                f"Jensen-Shannon similarity (base={self._base:g}): {score:.4f} "
+                f"(divergence={divergence:.4f})"
+            ),
+            metadata={
+                "divergence": divergence,
+                "distance": math.sqrt(divergence),
+                "base": self._base,
+            },
+        )
+
+    def _js_divergence(
+        self,
+        p_dist: Dict[str, float],
+        q_dist: Dict[str, float],
+    ) -> float:
+        vocabulary = sorted(set(p_dist) | set(q_dist))
+        if not vocabulary:
+            return 0.0
+
+        p_vector = [p_dist.get(token, 0.0) for token in vocabulary]
+        q_vector = [q_dist.get(token, 0.0) for token in vocabulary]
+
+        p_probs = self._ensure_probability_vector(p_vector)
+        q_probs = self._ensure_probability_vector(q_vector)
+
+        distance = float(self._js_distance_fn(p_probs, q_probs, base=self._base))
+        return distance**2
+
+    def _ensure_probability_vector(self, values: Sequence[float]) -> List[float]:
+        total = sum(values)
+        if total <= 0.0:
+            raise MetricComputationError(
+                "Distribution is empty after tokenisation (Jensen-Shannon metric)."
+            )
+        return [value / total for value in values]
+
+
+class JSDistance(JSDivergence):
+    """
+    Return the raw Jensen–Shannon divergence instead of similarity.
+
+    Args:
+        tokenizer: Optional tokenizer function.
+        base: Logarithm base used for the divergence calculation.
+        normalize: Whether to normalise counts into probabilities.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Example:
+        >>> from opik.evaluation.metrics import JSDistance
+        >>> metric = JSDistance()
+        >>> result = metric.score("a a b", reference="a b b")
+        >>> round(result.value, 3)  # doctest: +SKIP
+        0.188
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        base: float = 2.0,
+        normalize: bool = True,
+        name: str = "js_distance_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            tokenizer=tokenizer,
+            base=base,
+            normalize=normalize,
+            name=name,
+            track=track,
+            project_name=project_name,
+        )
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        similarity = super().score(output=output, reference=reference)
+        metadata = similarity.metadata or {}
+        divergence = float(metadata.get("divergence", 0.0))
+        distance = float(metadata.get("distance", math.sqrt(divergence)))
+        return score_result.ScoreResult(
+            value=divergence,
+            name=self.name,
+            reason=f"Jensen-Shannon divergence (base={self._base:g}): {divergence:.4f}",
+            metadata={
+                "distance": distance,
+                "base": metadata.get("base", self._base),
+            },
+        )
+
+
+class KLDivergence(_DistributionMetricBase):
+    """
+    Compute the (optionally symmetric) KL divergence between token distributions.
+
+    Args:
+        tokenizer: Optional tokenizer function. Defaults to whitespace split.
+        direction: Direction to compute (``"pq"``, ``"qp"``, or ``"avg"`` for
+            symmetric).
+        normalize: Whether to normalise token counts to probabilities first.
+        smoothing: Additive smoothing constant to avoid divide-by-zero.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Example:
+        >>> from opik.evaluation.metrics import KLDivergence
+        >>> metric = KLDivergence(direction="avg")
+        >>> result = metric.score("hello hello world", reference="hello world")
+        >>> round(result.value, 4)  # doctest: +SKIP
+        0.0583
+    """
+
+    def __init__(
+        self,
+        tokenizer: Optional[TokenizeFn] = None,
+        direction: str = "pq",
+        normalize: bool = True,
+        smoothing: float = 1e-12,
+        name: str = "kl_divergence_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        if direction not in {"pq", "qp", "avg"}:
+            raise ValueError("direction must be one of {'pq', 'qp', 'avg'}")
+        super().__init__(
+            tokenizer=tokenizer,
+            name=name,
+            track=track,
+            project_name=project_name,
+            normalize=normalize,
+            smoothing=smoothing,
+        )
+        self._direction = direction
+
+    def score(
+        self,
+        output: str,
+        reference: str,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (KL divergence metric).")
+        if not reference.strip():
+            raise MetricComputationError("Reference is empty (KL divergence metric).")
+
+        p_dist = self._build_distribution(output)
+        q_dist = self._build_distribution(reference)
+
+        if self._direction == "pq":
+            divergence = self._kl(p_dist, q_dist)
+        elif self._direction == "qp":
+            divergence = self._kl(q_dist, p_dist)
+        else:
+            divergence = 0.5 * (self._kl(p_dist, q_dist) + self._kl(q_dist, p_dist))
+
+        return score_result.ScoreResult(
+            value=divergence,
+            name=self.name,
+            reason=f"KL divergence ({self._direction}): {divergence:.4f}",
+        )
+
+    def _kl(self, p_dist: Dict[str, float], q_dist: Dict[str, float]) -> float:
+        divergence = 0.0
+        for token, p_val in p_dist.items():
+            p_val = self._smooth(p_val)
+            q_val = self._smooth(q_dist.get(token, 0.0))
+            divergence += p_val * math.log(p_val / q_val)
+        return divergence

opik/evaluation/metrics/heuristics/gleu.py

@@ -0,0 +1,113 @@
+from typing import Any, Callable, Optional, Sequence, Union
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+try:
+    from nltk.translate import gleu_score as nltk_gleu_score
+except ImportError:  # pragma: no cover - optional dependency
+    nltk_gleu_score = None
+
+
+GleuFn = Callable[[Sequence[Sequence[str]], Sequence[str]], float]
+
+
+class GLEU(base_metric.BaseMetric):
+    """
+    Sentence-level GLEU metric powered by ``nltk.translate.gleu_score``.
+
+    References:
+      - NLTK Reference Documentation on GLEU
+        https://www.nltk.org/api/nltk.translate.gleu_score.html
+      - OECD Catalogue of Tools & Metrics for Trustworthy AI
+        https://oecd.ai/en/catalogue/metrics/google-bleu-gleu
+      - Hugging Face Evaluate: Google BLEU (GLEU) metric overview
+        https://huggingface.co/spaces/evaluate-metric/google_bleu
+
+    Args:
+        gleu_fn: Optional custom scoring callable compatible with
+            ``nltk.translate.gleu_score.sentence_gleu``. Useful for testing.
+        min_len: Minimum n-gram size considered.
+        max_len: Maximum n-gram size considered.
+        name: Display name for the metric result.
+        track: Whether to automatically track metric results.
+        project_name: Optional tracking project name.
+
+    Example:
+        >>> from opik.evaluation.metrics import GLEU
+        >>> metric = GLEU(min_len=1, max_len=4)
+        >>> result = metric.score(
+        ...     output="The cat sat on the mat",
+        ...     reference="The cat is on the mat",
+        ... )
+        >>> round(result.value, 3)  # doctest: +SKIP
+        0.816
+    """
+
+    def __init__(
+        self,
+        gleu_fn: Optional[GleuFn] = None,
+        min_len: int = 1,
+        max_len: int = 4,
+        name: str = "gleu_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        if min_len <= 0 or max_len <= 0:
+            raise ValueError("min_len and max_len must be positive integers.")
+        if min_len > max_len:
+            raise ValueError("min_len cannot exceed max_len.")
+
+        super().__init__(name=name, track=track, project_name=project_name)
+
+        if gleu_fn is not None:
+            self._gleu_fn = gleu_fn
+        else:
+            if nltk_gleu_score is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "GLEU metric requires the optional 'nltk' package. Install via"
+                    " `pip install nltk` or provide `gleu_fn`."
+                )
+
+            def _scorer(
+                references: Sequence[Sequence[str]], hypothesis: Sequence[str]
+            ) -> float:
+                return float(
+                    nltk_gleu_score.sentence_gleu(
+                        references,
+                        hypothesis,
+                        min_len=min_len,
+                        max_len=max_len,
+                    )
+                )
+
+            self._gleu_fn = _scorer
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str]],
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (GLEU metric).")
+        hypothesis_tokens = output.split()
+        if isinstance(reference, str):
+            references = [reference.split()]
+        else:
+            ref_list = list(reference)
+            if not ref_list:
+                raise MetricComputationError("Reference is empty (GLEU metric).")
+            references = [ref.split() for ref in ref_list]
+
+        if any(len(ref) == 0 for ref in references):
+            raise MetricComputationError(
+                "Reference contains empty segment (GLEU metric)."
+            )
+
+        score = self._gleu_fn(references, hypothesis_tokens)
+        return score_result.ScoreResult(
+            value=float(score),
+            name=self.name,
+            reason=f"GLEU score: {float(score):.4f}",
+        )

opik/evaluation/metrics/heuristics/language_adherence.py

@@ -0,0 +1,123 @@
+"""Language adherence metric leveraging fastText-style language identification."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Tuple
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics.base_metric import BaseMetric
+from opik.evaluation.metrics.score_result import ScoreResult
+
+try:  # optional dependency
+    import fasttext
+except ImportError:  # pragma: no cover
+    fasttext = None  # type: ignore
+
+
+DetectorFn = Callable[[str], Tuple[str, float]]
+
+
+class LanguageAdherenceMetric(BaseMetric):
+    """
+    Check whether text is written in the expected language.
+
+    The metric relies on a fastText language identification model (or a
+    user-supplied detector callable) to predict the language of the evaluated text
+    and compares it with ``expected_language``. It outputs ``1.0`` when the detected
+    language matches and ``0.0`` otherwise, along with the detected label and
+    confidence score in ``metadata``.
+
+    References:
+      - fastText language identification models
+        https://fasttext.cc/docs/en/language-identification.html
+      - Joulin et al., "Bag of Tricks for Efficient Text Classification" (EACL 2017)
+        https://aclanthology.org/E17-2068/
+
+    Args:
+        expected_language: Language code the text should conform to, e.g. ``"en"``.
+        model_path: Path to a fastText language identification model. Required unless
+            ``detector`` is provided.
+        name: Display name for the metric result. Defaults to
+            ``"language_adherence_metric"``.
+        track: Whether to automatically track metric results. Defaults to ``True``.
+        project_name: Optional tracking project name. Defaults to ``None``.
+        detector: Optional callable accepting text and returning a
+            ``(language, confidence)`` tuple. When provided, ``model_path`` is not
+            needed.
+
+    Example:
+        >>> from opik.evaluation.metrics import LanguageAdherenceMetric
+        >>> # Assuming `lid.176.ftz` is available locally for fastText
+        >>> metric = LanguageAdherenceMetric(expected_language="en", model_path="lid.176.ftz")
+        >>> result = metric.score("This response is written in English.")  # doctest: +SKIP
+        >>> result.value  # doctest: +SKIP
+        1.0
+    """
+
+    def __init__(
+        self,
+        expected_language: str,
+        model_path: Optional[str] = None,
+        name: str = "language_adherence_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        detector: Optional[DetectorFn] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._expected_language = expected_language
+        self._detector_fn: DetectorFn
+        self._model_path = model_path
+
+        self._fasttext_model: Optional[Any]
+
+        if detector is not None:
+            self._detector_fn = detector
+            self._fasttext_model = None
+            return
+
+        if fasttext is None:
+            raise ImportError(
+                "Install fasttext via `pip install fasttext` and provide a fastText language"
+                " model (e.g., lid.176.ftz) or supply a custom detector callable."
+            )
+        if model_path is None:
+            raise ValueError(
+                "model_path is required when using the fastText-based detector."
+            )
+        self._fasttext_model = fasttext.load_model(model_path)
+        self._detector_fn = self._predict_with_fasttext
+
+    def score(self, output: str, **ignored_kwargs: Any) -> ScoreResult:
+        processed = output
+        if not processed.strip():
+            raise MetricComputationError("Text is empty for language adherence check.")
+
+        language, confidence = self._detector_fn(processed)
+        adherence = 1.0 if language == self._expected_language else 0.0
+
+        metadata = {
+            "detected_language": language,
+            "confidence": confidence,
+            "expected_language": self._expected_language,
+        }
+
+        reason = (
+            "Language adheres to expectation"
+            if adherence == 1.0
+            else f"Detected language '{language}' differs from expected '{self._expected_language}'"
+        )
+
+        return ScoreResult(
+            value=adherence, name=self.name, reason=reason, metadata=metadata
+        )
+
+    def _predict_with_fasttext(self, text: str) -> tuple[str, float]:
+        if self._fasttext_model is None:
+            raise MetricComputationError(
+                "fastText model is not loaded. Ensure that LanguageAdherenceMetric was initialized with a valid model_path and fastText is installed."
+            )
+        prediction = self._fasttext_model.predict(text)
+        label = prediction[0][0] if prediction[0] else ""
+        language = label.replace("__label__", "")
+        confidence = float(prediction[1][0]) if prediction[1] else 0.0
+        return language, confidence

opik/evaluation/metrics/heuristics/meteor.py

@@ -0,0 +1,119 @@
+from typing import Any, Callable, Optional, Sequence, Union
+
+try:
+    import nltk  # type: ignore
+    from nltk.corpus import wordnet  # type: ignore
+except ImportError:  # pragma: no cover - optional dependency
+    nltk = None
+    wordnet = None
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+try:
+    from nltk.translate import meteor_score as nltk_meteor_score
+except ImportError:  # pragma: no cover - optional dependency
+    nltk_meteor_score = None
+
+
+MeteorFn = Callable[[Sequence[str], str], float]
+
+
+class METEOR(base_metric.BaseMetric):
+    """Computes the METEOR score between output and reference text.
+
+    This implementation wraps ``nltk.translate.meteor_score.meteor_score`` while
+    allowing a custom scoring function to be injected (useful for testing).
+
+    References:
+      - Banerjee & Lavie, "METEOR: An Automatic Metric for MT Evaluation with Improved
+        Correlation with Human Judgments" (ACL Workshop 2005)
+        https://aclanthology.org/W05-0909/
+      - Hugging Face Evaluate: METEOR metric overview
+        https://huggingface.co/spaces/evaluate-metric/meteor
+
+    Args:
+        meteor_fn: Optional callable with the same interface as
+            ``nltk.translate.meteor_score.meteor_score``. When omitted the
+            function from NLTK is used.
+        alpha: Precision weight.
+        beta: Penalty exponent.
+        gamma: Fragmentation penalty weight.
+        name: Optional metric name.
+        track: Whether Opik should track the metric automatically.
+        project_name: Optional project name used when tracking.
+    """
+
+    def __init__(
+        self,
+        meteor_fn: Optional[MeteorFn] = None,
+        alpha: float = 0.9,
+        beta: float = 3.0,
+        gamma: float = 0.5,
+        name: str = "meteor_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+
+        if meteor_fn is not None:
+            self._meteor_fn = meteor_fn
+        else:
+            if nltk_meteor_score is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "METEOR metric requires the optional 'nltk' package. Install via"
+                    " `pip install nltk` or provide `meteor_fn`."
+                )
+
+            if nltk is not None and wordnet is not None:
+                try:
+                    wordnet.ensure_loaded()  # type: ignore[attr-defined]
+                except (
+                    LookupError
+                ):  # pragma: no cover - download path relies on network access
+                    try:
+                        nltk.download("wordnet", quiet=True)
+                        nltk.download("omw-1.4", quiet=True)
+                        wordnet.ensure_loaded()  # type: ignore[attr-defined]
+                    except Exception as download_error:
+                        raise ImportError(
+                            "METEOR metric requires the NLTK corpora 'wordnet' and 'omw-1.4'. "
+                            "Install manually via `python -m nltk.downloader wordnet omw-1.4`."
+                        ) from download_error
+
+            def _scorer(references: Sequence[str], hypothesis: str) -> float:
+                try:
+                    return float(
+                        nltk_meteor_score.meteor_score(
+                            references, hypothesis, alpha=alpha, beta=beta, gamma=gamma
+                        )
+                    )
+                except LookupError as error:
+                    raise MetricComputationError(
+                        "NLTK resource requirement for METEOR not satisfied. "
+                        "Download WordNet via `nltk.download('wordnet')`."
+                    ) from error
+
+            self._meteor_fn = _scorer
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str]],
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (METEOR metric).")
+        if isinstance(reference, str):
+            references: Sequence[str] = [reference]
+        else:
+            references = list(reference)
+        if not references or any(not ref.strip() for ref in references):
+            raise MetricComputationError("Reference is empty (METEOR metric).")
+
+        score = self._meteor_fn(references, output)
+        return score_result.ScoreResult(
+            value=float(score),
+            name=self.name,
+            reason=f"METEOR score: {float(score):.4f}",
+        )