opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/__init__.py

@@ -1,23 +1,81 @@
 from .aggregated_metric import AggregatedMetric
-from .conversation.session_completeness.metric import SessionCompletenessQuality
-from .conversation.conversational_coherence.metric import ConversationalCoherenceMetric
-from .conversation.user_frustration.metric import UserFrustrationMetric
+
+# Keep the canonical import first for the new layout while still tolerating
+# older packaging artefacts (some environments import this module before the
+# conversation package is available).  If the eager import fails we fall back
+# to the lazy getter below, letting legacy entry-points keep working.
+from .conversation.conversation_thread_metric import ConversationThreadMetric
+
+from .conversation import types as conversation_types
+from .conversation.heuristics.degeneration.metric import ConversationDegenerationMetric
+from .conversation.heuristics.knowledge_retention.metric import (
+    KnowledgeRetentionMetric,
+)
+from .conversation.llm_judges.conversational_coherence.metric import (
+    ConversationalCoherenceMetric,
+)
+from .conversation.llm_judges.g_eval_wrappers import (
+    GEvalConversationMetric,
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    ConversationPromptUncertaintyMetric,
+)
+from .conversation.llm_judges.session_completeness.metric import (
+    SessionCompletenessQuality,
+)
+from .conversation.llm_judges.user_frustration.metric import UserFrustrationMetric
 from .heuristics.contains import Contains
 from .heuristics.equals import Equals
+from .heuristics.gleu import GLEU
+from .heuristics.chrf import ChrF
 from .heuristics.is_json import IsJson
+from .heuristics.distribution_metrics import (
+    JSDivergence,
+    JSDistance,
+    KLDivergence,
+)
 from .heuristics.levenshtein_ratio import LevenshteinRatio
+from .heuristics.meteor import METEOR
+from .heuristics.bertscore import BERTScore
+from .heuristics.spearman import SpearmanRanking
+from .heuristics.readability import Readability
+from .heuristics.tone import Tone
+from .heuristics.prompt_injection import PromptInjection
+from .heuristics.language_adherence import LanguageAdherenceMetric
 from .heuristics.regex_match import RegexMatch
 from .heuristics.bleu import SentenceBLEU, CorpusBLEU
 from .heuristics.rouge import ROUGE
 from .heuristics.sentiment import Sentiment
+from .heuristics.vader_sentiment import VADERSentiment
 from .llm_judges.answer_relevance.metric import AnswerRelevance
+from .llm_judges.g_eval_presets import (
+    AgentTaskCompletionJudge,
+    AgentToolCorrectnessJudge,
+    ComplianceRiskJudge,
+    DemographicBiasJudge,
+    DialogueHelpfulnessJudge,
+    GenderBiasJudge,
+    PoliticalBiasJudge,
+    PromptUncertaintyJudge,
+    QARelevanceJudge,
+    RegionalBiasJudge,
+    ReligiousBiasJudge,
+    SummarizationCoherenceJudge,
+    SummarizationConsistencyJudge,
+)
 from .llm_judges.context_precision.metric import ContextPrecision
 from .llm_judges.context_recall.metric import ContextRecall
-from .llm_judges.g_eval.metric import GEval
+from .llm_judges.g_eval.metric import GEval, GEvalPreset
 from .llm_judges.hallucination.metric import Hallucination
 from .llm_judges.moderation.metric import Moderation
+from .llm_judges.llm_juries.metric import LLMJuriesJudge
 from .llm_judges.trajectory_accuracy import TrajectoryAccuracy
+from .llm_judges.syc_eval.metric import SycEval
 from .llm_judges.usefulness.metric import Usefulness
+from .llm_judges.structure_output_compliance.metric import StructuredOutputCompliance
 from .base_metric import BaseMetric
 from .ragas_metric import RagasMetricWrapper
 from opik.exceptions import MetricComputationError
@@ -27,17 +85,51 @@ from opik.exceptions import MetricComputationError
 __all__ = [
     "AggregatedMetric",
     "AnswerRelevance",
+    "AgentTaskCompletionJudge",
+    "AgentToolCorrectnessJudge",
     "BaseMetric",
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+    "GEvalConversationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "ConversationPromptUncertaintyMetric",
+    "conversation_types",
+    "ComplianceRiskJudge",
     "Contains",
     "ContextPrecision",
     "ContextRecall",
     "ConversationalCoherenceMetric",
     "CorpusBLEU",
+    "DemographicBiasJudge",
     "Equals",
     "GEval",
+    "GEvalPreset",
+    "GLEU",
+    "GenderBiasJudge",
     "Hallucination",
     "IsJson",
+    "JSDivergence",
+    "JSDistance",
+    "KLDivergence",
     "LevenshteinRatio",
+    "BERTScore",
+    "METEOR",
+    "ChrF",
+    "Readability",
+    "PromptInjection",
+    "LanguageAdherenceMetric",
+    "PoliticalBiasJudge",
+    "PromptUncertaintyJudge",
+    "SpearmanRanking",
+    "ReligiousBiasJudge",
+    "RegionalBiasJudge",
+    "VADERSentiment",
+    "Tone",
+    "StructuredOutputCompliance",
     "MetricComputationError",
     "Moderation",
     "RagasMetricWrapper",
@@ -46,8 +138,15 @@ __all__ = [
     "SentenceBLEU",
     "Sentiment",
     "SessionCompletenessQuality",
+    "SycEval",
     "Usefulness",
     "UserFrustrationMetric",
     "TrajectoryAccuracy",
+    "DialogueHelpfulnessJudge",
+    "QARelevanceJudge",
+    "SummarizationCoherenceJudge",
+    "SummarizationConsistencyJudge",
+    "LLMJuriesJudge",
+    "ConversationThreadMetric",
     # "Factuality",
 ]

opik/evaluation/metrics/aggregated_metric.py

@@ -8,14 +8,43 @@ from . import arguments_helpers, arguments_validator, base_metric, score_result
 class AggregatedMetric(
     base_metric.BaseMetric, arguments_validator.ScoreArgumentsValidator
 ):
-    """A metric that aggregates results obtained from a list of provided metrics using specified aggregation function.
+    """
+    Combine the output of multiple metrics into a single aggregated ``ScoreResult``.
+
+    Each metric in ``metrics`` is executed with the provided scoring kwargs, then the
+    ``aggregator`` callback decides how to merge the individual results. This is
+    handy for building ensembles such as min/max, weighted averages, or custom
+    pass/fail checks without re-implementing the metrics themselves.
 
     Args:
-        name: The name of the metric.
-        metrics: A list of concrete metric instances that inherit the `opik.evaluation.base_metric.BaseMetric`.
-        aggregator: The aggregation function to use for evaluation.
-        track: Whether to track the metric. Defaults to True.
-        project_name: Optional project name to track the metric in for the cases when there are no parent span/trace to inherit project name from.
+        name: Display name for the aggregated metric result.
+        metrics: Ordered list of metric instances that should be executed.
+        aggregator: Callable receiving the list of ``ScoreResult`` objects and
+            returning the final aggregated ``ScoreResult``.
+        track: Whether to automatically track the metric in Opik. Defaults to
+            ``True``.
+        project_name: Optional tracking project used when no parent context exists.
+
+    Example:
+        >>> from opik.evaluation.metrics import AggregatedMetric, Contains, RegexMatch
+        >>> metrics = [Contains(track=False), RegexMatch(pattern=r"\\d+", track=False)]
+        >>> from opik.evaluation.metrics import score_result
+        >>> def combine(results):
+        ...     score = sum(result.value for result in results) / len(results)
+        ...     return score_result.ScoreResult(
+        ...         name="combined_contains_regex",
+        ...         value=score,
+        ...         reason="Average of contains and regex checks",
+        ...     )
+        >>> metric = AggregatedMetric(
+        ...     name="combined_contains_regex",
+        ...     metrics=metrics,
+        ...     aggregator=combine,
+        ... )
+        >>> response = "Order number 12345 confirmed"
+        >>> result = metric.score(output=response, reference="order")
+        >>> float(result.value)  # doctest: +SKIP
+        1.0
     """
 
     def __init__(

opik/evaluation/metrics/base_metric.py

@@ -1,5 +1,5 @@
 import abc
-from typing import Any, List, Union, Optional
+from typing import Any, List, Optional, Union
 
 import opik
 import opik.config as opik_config

opik/evaluation/metrics/conversation/__init__.py

@@ -0,0 +1,48 @@
+"""Public conversation metrics API."""
+
+from .conversation_thread_metric import ConversationThreadMetric
+from .conversation_turns_factory import build_conversation_turns
+from .helpers import (
+    extract_turns_windows_from_conversation,
+    get_turns_in_sliding_window,
+    merge_turns,
+)
+from .types import Conversation, ConversationDict, ConversationTurn
+
+__all__ = [
+    "ConversationThreadMetric",
+    "Conversation",
+    "ConversationDict",
+    "ConversationTurn",
+    "build_conversation_turns",
+    "extract_turns_windows_from_conversation",
+    "get_turns_in_sliding_window",
+    "merge_turns",
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+    "ConversationalCoherenceMetric",
+    "SessionCompletenessQuality",
+    "UserFrustrationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "GEvalConversationMetric",
+]
+
+from .heuristics.degeneration.metric import ConversationDegenerationMetric
+from .heuristics.knowledge_retention.metric import KnowledgeRetentionMetric
+from .llm_judges.conversational_coherence.metric import ConversationalCoherenceMetric
+from .llm_judges.g_eval_wrappers import (
+    GEvalConversationMetric,
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    ConversationPromptUncertaintyMetric,
+)
+from .llm_judges.session_completeness.metric import SessionCompletenessQuality
+from .llm_judges.user_frustration.metric import UserFrustrationMetric

opik/evaluation/metrics/conversation/conversation_thread_metric.py

@@ -5,11 +5,53 @@ from .. import base_metric, score_result
 
 
 class ConversationThreadMetric(base_metric.BaseMetric):
-    """Abstract base class for all conversation thread metrics."""
+    """
+    Abstract base class for all conversation thread metrics. When creating a custom
+    conversation metric, you should inherit from this class and implement the abstract methods.
+
+    Conversation metrics are designed to evaluate multi-turn conversations rather than
+    single input-output pairs. They accept a conversation as a list of message dictionaries,
+    where each message has a 'role' (either 'user' or 'assistant') and 'content'.
+
+    Args:
+        name: The name of the metric. If not provided, uses the class name as default.
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there is no parent span/trace to inherit project name from.
+
+    Example:
+        >>> from opik.evaluation.metrics.conversation import conversation_thread_metric, types
+        >>> from opik.evaluation.metrics import score_result
+        >>> from typing import Any
+        >>>
+        >>> class ConversationLengthMetric(conversation_thread_metric.ConversationThreadMetric):
+        >>>     def __init__(self, name: str = "conversation_length_score"):
+        >>>         super().__init__(name)
+        >>>
+        >>>     def score(self, conversation: types.Conversation, **kwargs: Any):
+        >>>         num_turns = sum(1 for msg in conversation if msg["role"] == "assistant")
+        >>>         return score_result.ScoreResult(
+        >>>             name=self.name,
+        >>>             value=num_turns,
+        >>>             reason=f"Conversation has {num_turns} turns"
+        >>>         )
+    """
 
     def score(
         self, conversation: types.Conversation, **kwargs: Any
     ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Evaluate a conversation and return a score.
+
+        Args:
+            conversation: A list of conversation messages. Each message is a dictionary
+                with 'role' (either 'user' or 'assistant') and 'content' (the message text).
+            **kwargs: Additional keyword arguments that may be used by specific metric implementations.
+
+        Returns:
+            A ScoreResult object or list of ScoreResult objects containing the evaluation score,
+            metric name, and optional reasoning.
+        """
         raise NotImplementedError(
             "Please use concrete metric classes instead of this one."
         )
@@ -18,7 +60,19 @@ class ConversationThreadMetric(base_metric.BaseMetric):
         self, conversation: types.Conversation, **kwargs: Any
     ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
         """
-        Async public method that can be called independently.
+        Asynchronously evaluate a conversation and return a score.
+
+        This is the async version of the score method. By default, it calls the
+        synchronous score method, but can be overridden for true async implementations.
+
+        Args:
+            conversation: A list of conversation messages. Each message is a dictionary
+                with 'role' (either 'user' or 'assistant') and 'content' (the message text).
+            **kwargs: Additional keyword arguments that may be used by specific metric implementations.
+
+        Returns:
+            A ScoreResult object or list of ScoreResult objects containing the evaluation score,
+            metric name, and optional reasoning.
         """
         raise NotImplementedError(
             "Please use concrete metric classes instead of this one."

opik/evaluation/metrics/conversation/g_eval_wrappers.py

@@ -0,0 +1,19 @@
+from .llm_judges.g_eval_wrappers import (
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationPromptUncertaintyMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    GEvalConversationMetric,
+)
+
+__all__ = [
+    "GEvalConversationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+]

opik/evaluation/metrics/conversation/helpers.py

@@ -1,6 +1,7 @@
 from typing import Any, Generator, List
 
-from . import types, conversation_turns_factory
+from . import types
+from .conversation_turns_factory import build_conversation_turns
 
 
 def get_turns_in_sliding_window(
@@ -8,22 +9,20 @@ def get_turns_in_sliding_window(
 ) -> Generator[List[types.ConversationTurn], Any, None]:
     """
     Generates windows of conversation turns of a fixed size from a list of turns.
-
     This function creates a sliding window over the list of conversation turns.
     Each window includes the current turn and up to `window_size - 1` previous
     conversation turns. If there are fewer turns available than the `window_size`,
     the window will consist of all available turns up to the current turn.
-
     Args:
         turns: List of conversation turn objects representing the interactions
             in a conversation.
         window_size: Integer specifying the maximum number of turns to include
             in each window.
-
     Yields:
         A generator that produces lists of conversation turns, where each list
         represents a sliding window of turns.
     """
+
     for i in range(len(turns)):
         yield turns[max(0, i - window_size + 1) : i + 1]
 
@@ -31,19 +30,17 @@ def get_turns_in_sliding_window(
 def merge_turns(turns: List[types.ConversationTurn]) -> types.Conversation:
     """
     Merges a list of conversation turns into a single conversation.
-
     This function takes a list of conversation turns and combines them
     into a single conversation by extending the output list with the data
     from each turn.
-
     Args:
         turns: A list of conversation turn  objects to be combined.
-
     Returns:
         types.Conversation: A combined conversation object containing all
             the turns from the input list.
     """
-    output = []
+
+    output: types.Conversation = []
     for turn in turns:
         output.extend(turn.as_list())
     return output
@@ -56,24 +53,20 @@ def extract_turns_windows_from_conversation(
     Extracts a list of conversation windows based on turns using a sliding window
     approach. This function divides a conversation into consecutive overlapping
     windows, where each window contains a specified number of turns.
-
     Args:
         conversation: The input conversation from which turns will be processed.
         window_size: The number of turns to include in each sliding window.
-
     Returns:
         A list of conversations, each representing a window of turns specified
         by the given window size.
-
     Raises:
         ValueError: If the conversation is empty or if it has no turns.
     """
+
     if len(conversation) == 0:
         raise ValueError("Conversation is empty")
 
-    turns = conversation_turns_factory.build_conversation_turns(
-        conversation=conversation
-    )
+    turns = build_conversation_turns(conversation=conversation)
     if len(turns) == 0:
         raise ValueError("Conversation has no turns")
 
@@ -81,5 +74,11 @@ def extract_turns_windows_from_conversation(
         merge_turns(turns_window)
         for turns_window in get_turns_in_sliding_window(turns, window_size)
     ]
-
     return turns_windows
+
+
+__all__ = [
+    "get_turns_in_sliding_window",
+    "merge_turns",
+    "extract_turns_windows_from_conversation",
+]

opik/evaluation/metrics/conversation/heuristics/__init__.py

@@ -0,0 +1,14 @@
+"""Heuristic conversation-level metrics.
+
+Exposes the reusable conversation-level heuristics under the public namespace
+``opik.evaluation.metrics.conversation.heuristics.*`` so documentation and downstream
+code can import them directly.
+"""
+
+from .degeneration.metric import ConversationDegenerationMetric
+from .knowledge_retention.metric import KnowledgeRetentionMetric
+
+__all__ = [
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+]

opik/evaluation/metrics/conversation/heuristics/degeneration/__init__.py

@@ -0,0 +1,3 @@
+from .metric import ConversationDegenerationMetric
+
+__all__ = ["ConversationDegenerationMetric"]

opik/evaluation/metrics/conversation/heuristics/degeneration/metric.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Dict, List, Optional
+
+from opik.evaluation.metrics.conversation import types as conversation_types
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+from .phrases import DEFAULT_FALLBACK_PHRASES
+
+
+def _tokenize(text: str) -> List[str]:
+    return re.findall(r"\b\w+\b", text.lower())
+
+
+def _ngram_counts(tokens: List[str], n: int) -> Counter:
+    if len(tokens) < n:
+        return Counter()
+    return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
+
+
+class ConversationDegenerationMetric(ConversationThreadMetric):
+    """
+    Score how strongly an assistant conversation shows degeneration or repetition.
+
+    The metric inspects each assistant turn, measuring repeated n-grams, overlap with
+    the previous reply, low lexical diversity, and presence of known fallback
+    phrases (for example, "as an AI language model..."). Each turn receives a
+    degeneration score between `0.0` and `1.0`; the overall metric reports the peak
+    risk observed so you can quickly flag sections where the assistant got stuck or
+    stopped being helpful. Detailed per-turn diagnostics are returned in the
+    ``ScoreResult.metadata`` payload.
+
+    Args:
+        name: Display name for the metric result. Defaults to
+            ``"conversation_degeneration_metric"``.
+        track: Whether the metric should automatically track to an Opik project.
+            Defaults to ``True``.
+        project_name: Optional project to store tracked results in. Defaults to
+            ``None`` (inherit global setting).
+        ngram_size: Size of the n-grams used to detect repetition within a single
+            response. Must be at least ``2``. Defaults to ``3``.
+        fallback_phrases: Custom list of phrases that should be treated as
+            degeneration signatures. If ``None``, a sensible default list is used.
+
+    Example:
+        >>> from opik.evaluation.metrics import ConversationDegenerationMetric
+        >>> conversation = [
+        ...     {"role": "user", "content": "Can you draft a short bio for Ada?"},
+        ...     {"role": "assistant", "content": "Sure, here is a short bio for Ada."},
+        ...     {"role": "user", "content": "Could you add more detail?"},
+        ...     {"role": "assistant", "content": "Sure, here is a short bio for Ada."},
+        ... ]
+        >>> metric = ConversationDegenerationMetric(ngram_size=3)
+        >>> result = metric.score(conversation)
+        >>> float(result.value)  # doctest: +SKIP
+        0.75
+    """
+
+    def __init__(
+        self,
+        name: str = "conversation_degeneration_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        ngram_size: int = 3,
+        fallback_phrases: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        if ngram_size < 2:
+            raise MetricComputationError("ngram_size must be >= 2")
+        self._ngram_size = ngram_size
+        phrases = (
+            fallback_phrases
+            if fallback_phrases is not None
+            else DEFAULT_FALLBACK_PHRASES
+        )
+        self._fallback_phrases = [phrase.lower() for phrase in phrases]
+
+    def score(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: object,
+    ) -> ScoreResult:
+        assistant_turns = [
+            turn["content"]
+            for turn in conversation
+            if turn.get("role") == "assistant" and turn.get("content")
+        ]
+        if not assistant_turns:
+            raise MetricComputationError("Conversation contains no assistant messages")
+
+        per_turn_metadata: List[Dict[str, float]] = []
+        degeneracy_scores: List[float] = []
+
+        prev_tokens: Optional[List[str]] = None
+        for content in assistant_turns:
+            tokens = _tokenize(content)
+            if not tokens:
+                continue
+
+            entropy_norm = self._token_entropy(tokens)
+            repetition_ratio = self._repetition_ratio(tokens)
+            prev_overlap = self._overlap_with_previous(tokens, prev_tokens)
+            fallback_score = 1.0 if self._contains_fallback_phrase(content) else 0.0
+
+            normalized_entropy = 1.0 - entropy_norm
+            # Combine all four risk factors with equal weight; this keeps the
+            # heuristic interpretable and matches the legacy scoring behaviour.
+            deg_score = min(
+                1.0,
+                (repetition_ratio + prev_overlap + fallback_score + normalized_entropy)
+                / 4.0,
+            )
+
+            per_turn_metadata.append(
+                {
+                    "repetition_ratio": repetition_ratio,
+                    "overlap_previous": prev_overlap,
+                    "fallback_hit": fallback_score,
+                    "normalized_entropy": normalized_entropy,
+                    "degeneration_score": deg_score,
+                }
+            )
+            degeneracy_scores.append(deg_score)
+            prev_tokens = tokens
+
+        if not degeneracy_scores:
+            raise MetricComputationError(
+                "Assistant messages were empty after tokenization"
+            )
+
+        average_score = sum(degeneracy_scores) / len(degeneracy_scores)
+        peak_score = max(degeneracy_scores)
+
+        return ScoreResult(
+            value=peak_score,
+            name=self.name,
+            reason=(
+                f"Peak degeneration risk ({len(degeneracy_scores)} turns):"
+                f" {peak_score:.3f}"
+            ),
+            metadata={
+                "per_turn": per_turn_metadata,
+                "average_score": average_score,
+                "peak_score": peak_score,
+            },
+        )
+
+    def _token_entropy(self, tokens: List[str]) -> float:
+        counts = Counter(tokens)
+        total = float(len(tokens))
+        entropy = 0.0
+        for count in counts.values():
+            prob = count / total
+            entropy -= prob * math.log(prob, 2)
+        max_entropy = math.log(len(counts), 2) if counts else 1.0
+        if max_entropy == 0:
+            return 0.0
+        return min(1.0, entropy / max_entropy)
+
+    def _repetition_ratio(self, tokens: List[str]) -> float:
+        ngram_counts = _ngram_counts(tokens, self._ngram_size)
+        total = sum(ngram_counts.values())
+        if total == 0:
+            return 0.0
+        repeated = sum(count for count in ngram_counts.values() if count > 1)
+        return repeated / total
+
+    def _overlap_with_previous(
+        self, tokens: List[str], prev_tokens: Optional[List[str]]
+    ) -> float:
+        if not prev_tokens:
+            return 0.0
+        current_set = set(tokens)
+        prev_set = set(prev_tokens)
+        if not current_set or not prev_set:
+            return 0.0
+        intersection = len(current_set & prev_set)
+        union = len(current_set | prev_set)
+        return intersection / union
+
+    def _contains_fallback_phrase(self, content: str) -> bool:
+        lowered = content.lower()
+        return any(phrase in lowered for phrase in self._fallback_phrases)

opik/evaluation/metrics/conversation/heuristics/degeneration/phrases.py

@@ -0,0 +1,12 @@
+"""Phrase lists used by the conversation degeneration metric."""
+
+DEFAULT_FALLBACK_PHRASES = [
+    "i'm sorry",
+    "as an ai language model",
+    "i cannot",
+    "i'm unable",
+    "please provide",
+    "i don't have access",
+    "i don't understand",
+    "could you please clarify",
+]

opik/evaluation/metrics/conversation/heuristics/knowledge_retention/__init__.py

@@ -0,0 +1,3 @@
+from .metric import KnowledgeRetentionMetric
+
+__all__ = ["KnowledgeRetentionMetric"]