opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/conversation/helpers.py

@@ -0,0 +1,84 @@
+from typing import Any, Generator, List
+
+from . import types
+from .conversation_turns_factory import build_conversation_turns
+
+
+def get_turns_in_sliding_window(
+    turns: List[types.ConversationTurn], window_size: int
+) -> Generator[List[types.ConversationTurn], Any, None]:
+    """
+    Generates windows of conversation turns of a fixed size from a list of turns.
+    This function creates a sliding window over the list of conversation turns.
+    Each window includes the current turn and up to `window_size - 1` previous
+    conversation turns. If there are fewer turns available than the `window_size`,
+    the window will consist of all available turns up to the current turn.
+    Args:
+        turns: List of conversation turn objects representing the interactions
+            in a conversation.
+        window_size: Integer specifying the maximum number of turns to include
+            in each window.
+    Yields:
+        A generator that produces lists of conversation turns, where each list
+        represents a sliding window of turns.
+    """
+
+    for i in range(len(turns)):
+        yield turns[max(0, i - window_size + 1) : i + 1]
+
+
+def merge_turns(turns: List[types.ConversationTurn]) -> types.Conversation:
+    """
+    Merges a list of conversation turns into a single conversation.
+    This function takes a list of conversation turns and combines them
+    into a single conversation by extending the output list with the data
+    from each turn.
+    Args:
+        turns: A list of conversation turn  objects to be combined.
+    Returns:
+        types.Conversation: A combined conversation object containing all
+            the turns from the input list.
+    """
+
+    output: types.Conversation = []
+    for turn in turns:
+        output.extend(turn.as_list())
+    return output
+
+
+def extract_turns_windows_from_conversation(
+    conversation: types.Conversation, window_size: int
+) -> List[types.Conversation]:
+    """
+    Extracts a list of conversation windows based on turns using a sliding window
+    approach. This function divides a conversation into consecutive overlapping
+    windows, where each window contains a specified number of turns.
+    Args:
+        conversation: The input conversation from which turns will be processed.
+        window_size: The number of turns to include in each sliding window.
+    Returns:
+        A list of conversations, each representing a window of turns specified
+        by the given window size.
+    Raises:
+        ValueError: If the conversation is empty or if it has no turns.
+    """
+
+    if len(conversation) == 0:
+        raise ValueError("Conversation is empty")
+
+    turns = build_conversation_turns(conversation=conversation)
+    if len(turns) == 0:
+        raise ValueError("Conversation has no turns")
+
+    turns_windows: List[types.Conversation] = [
+        merge_turns(turns_window)
+        for turns_window in get_turns_in_sliding_window(turns, window_size)
+    ]
+    return turns_windows
+
+
+__all__ = [
+    "get_turns_in_sliding_window",
+    "merge_turns",
+    "extract_turns_windows_from_conversation",
+]

opik/evaluation/metrics/conversation/heuristics/__init__.py

@@ -0,0 +1,14 @@
+"""Heuristic conversation-level metrics.
+
+Exposes the reusable conversation-level heuristics under the public namespace
+``opik.evaluation.metrics.conversation.heuristics.*`` so documentation and downstream
+code can import them directly.
+"""
+
+from .degeneration.metric import ConversationDegenerationMetric
+from .knowledge_retention.metric import KnowledgeRetentionMetric
+
+__all__ = [
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+]

opik/evaluation/metrics/conversation/heuristics/degeneration/__init__.py

@@ -0,0 +1,3 @@
+from .metric import ConversationDegenerationMetric
+
+__all__ = ["ConversationDegenerationMetric"]

opik/evaluation/metrics/conversation/heuristics/degeneration/metric.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Dict, List, Optional
+
+from opik.evaluation.metrics.conversation import types as conversation_types
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.exceptions import MetricComputationError
+from .phrases import DEFAULT_FALLBACK_PHRASES
+
+
+def _tokenize(text: str) -> List[str]:
+    return re.findall(r"\b\w+\b", text.lower())
+
+
+def _ngram_counts(tokens: List[str], n: int) -> Counter:
+    if len(tokens) < n:
+        return Counter()
+    return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
+
+
+class ConversationDegenerationMetric(ConversationThreadMetric):
+    """
+    Score how strongly an assistant conversation shows degeneration or repetition.
+
+    The metric inspects each assistant turn, measuring repeated n-grams, overlap with
+    the previous reply, low lexical diversity, and presence of known fallback
+    phrases (for example, "as an AI language model..."). Each turn receives a
+    degeneration score between `0.0` and `1.0`; the overall metric reports the peak
+    risk observed so you can quickly flag sections where the assistant got stuck or
+    stopped being helpful. Detailed per-turn diagnostics are returned in the
+    ``ScoreResult.metadata`` payload.
+
+    Args:
+        name: Display name for the metric result. Defaults to
+            ``"conversation_degeneration_metric"``.
+        track: Whether the metric should automatically track to an Opik project.
+            Defaults to ``True``.
+        project_name: Optional project to store tracked results in. Defaults to
+            ``None`` (inherit global setting).
+        ngram_size: Size of the n-grams used to detect repetition within a single
+            response. Must be at least ``2``. Defaults to ``3``.
+        fallback_phrases: Custom list of phrases that should be treated as
+            degeneration signatures. If ``None``, a sensible default list is used.
+
+    Example:
+        >>> from opik.evaluation.metrics import ConversationDegenerationMetric
+        >>> conversation = [
+        ...     {"role": "user", "content": "Can you draft a short bio for Ada?"},
+        ...     {"role": "assistant", "content": "Sure, here is a short bio for Ada."},
+        ...     {"role": "user", "content": "Could you add more detail?"},
+        ...     {"role": "assistant", "content": "Sure, here is a short bio for Ada."},
+        ... ]
+        >>> metric = ConversationDegenerationMetric(ngram_size=3)
+        >>> result = metric.score(conversation)
+        >>> float(result.value)  # doctest: +SKIP
+        0.75
+    """
+
+    def __init__(
+        self,
+        name: str = "conversation_degeneration_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        ngram_size: int = 3,
+        fallback_phrases: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        if ngram_size < 2:
+            raise MetricComputationError("ngram_size must be >= 2")
+        self._ngram_size = ngram_size
+        phrases = (
+            fallback_phrases
+            if fallback_phrases is not None
+            else DEFAULT_FALLBACK_PHRASES
+        )
+        self._fallback_phrases = [phrase.lower() for phrase in phrases]
+
+    def score(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: object,
+    ) -> ScoreResult:
+        assistant_turns = [
+            turn["content"]
+            for turn in conversation
+            if turn.get("role") == "assistant" and turn.get("content")
+        ]
+        if not assistant_turns:
+            raise MetricComputationError("Conversation contains no assistant messages")
+
+        per_turn_metadata: List[Dict[str, float]] = []
+        degeneracy_scores: List[float] = []
+
+        prev_tokens: Optional[List[str]] = None
+        for content in assistant_turns:
+            tokens = _tokenize(content)
+            if not tokens:
+                continue
+
+            entropy_norm = self._token_entropy(tokens)
+            repetition_ratio = self._repetition_ratio(tokens)
+            prev_overlap = self._overlap_with_previous(tokens, prev_tokens)
+            fallback_score = 1.0 if self._contains_fallback_phrase(content) else 0.0
+
+            normalized_entropy = 1.0 - entropy_norm
+            # Combine all four risk factors with equal weight; this keeps the
+            # heuristic interpretable and matches the legacy scoring behaviour.
+            deg_score = min(
+                1.0,
+                (repetition_ratio + prev_overlap + fallback_score + normalized_entropy)
+                / 4.0,
+            )
+
+            per_turn_metadata.append(
+                {
+                    "repetition_ratio": repetition_ratio,
+                    "overlap_previous": prev_overlap,
+                    "fallback_hit": fallback_score,
+                    "normalized_entropy": normalized_entropy,
+                    "degeneration_score": deg_score,
+                }
+            )
+            degeneracy_scores.append(deg_score)
+            prev_tokens = tokens
+
+        if not degeneracy_scores:
+            raise MetricComputationError(
+                "Assistant messages were empty after tokenization"
+            )
+
+        average_score = sum(degeneracy_scores) / len(degeneracy_scores)
+        peak_score = max(degeneracy_scores)
+
+        return ScoreResult(
+            value=peak_score,
+            name=self.name,
+            reason=(
+                f"Peak degeneration risk ({len(degeneracy_scores)} turns):"
+                f" {peak_score:.3f}"
+            ),
+            metadata={
+                "per_turn": per_turn_metadata,
+                "average_score": average_score,
+                "peak_score": peak_score,
+            },
+        )
+
+    def _token_entropy(self, tokens: List[str]) -> float:
+        counts = Counter(tokens)
+        total = float(len(tokens))
+        entropy = 0.0
+        for count in counts.values():
+            prob = count / total
+            entropy -= prob * math.log(prob, 2)
+        max_entropy = math.log(len(counts), 2) if counts else 1.0
+        if max_entropy == 0:
+            return 0.0
+        return min(1.0, entropy / max_entropy)
+
+    def _repetition_ratio(self, tokens: List[str]) -> float:
+        ngram_counts = _ngram_counts(tokens, self._ngram_size)
+        total = sum(ngram_counts.values())
+        if total == 0:
+            return 0.0
+        repeated = sum(count for count in ngram_counts.values() if count > 1)
+        return repeated / total
+
+    def _overlap_with_previous(
+        self, tokens: List[str], prev_tokens: Optional[List[str]]
+    ) -> float:
+        if not prev_tokens:
+            return 0.0
+        current_set = set(tokens)
+        prev_set = set(prev_tokens)
+        if not current_set or not prev_set:
+            return 0.0
+        intersection = len(current_set & prev_set)
+        union = len(current_set | prev_set)
+        return intersection / union
+
+    def _contains_fallback_phrase(self, content: str) -> bool:
+        lowered = content.lower()
+        return any(phrase in lowered for phrase in self._fallback_phrases)

opik/evaluation/metrics/conversation/heuristics/degeneration/phrases.py

@@ -0,0 +1,12 @@
+"""Phrase lists used by the conversation degeneration metric."""
+
+DEFAULT_FALLBACK_PHRASES = [
+    "i'm sorry",
+    "as an ai language model",
+    "i cannot",
+    "i'm unable",
+    "please provide",
+    "i don't have access",
+    "i don't understand",
+    "could you please clarify",
+]

opik/evaluation/metrics/conversation/heuristics/knowledge_retention/__init__.py

@@ -0,0 +1,3 @@
+from .metric import KnowledgeRetentionMetric
+
+__all__ = ["KnowledgeRetentionMetric"]

opik/evaluation/metrics/conversation/heuristics/knowledge_retention/metric.py

@@ -0,0 +1,172 @@
+"""Heuristic metric for knowledge retention across conversation turns."""
+
+from __future__ import annotations
+
+import functools
+from typing import Callable, List, Optional, Sequence, Set
+
+from opik.evaluation.metrics.conversation import types as conversation_types
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.evaluation.preprocessing import normalize_text
+
+_MIN_TOKEN_LENGTH = 4
+_STOPWORDS = {
+    "the",
+    "this",
+    "that",
+    "with",
+    "have",
+    "from",
+    "about",
+    "there",
+    "which",
+    "would",
+    "could",
+    "should",
+    "name",
+    "number",
+}
+
+_REQUEST_KEYWORDS = {
+    "need",
+    "please",
+    "help",
+    "can",
+    "could",
+    "would",
+    "should",
+    "shall",
+    "may",
+    "might",
+    "want",
+    "tell",
+    "explain",
+    "show",
+    "give",
+    "provide",
+    "let",
+    "allow",
+    "request",
+}
+
+
+class KnowledgeRetentionMetric(ConversationThreadMetric):
+    """
+    Measure how many user-provided facts resurface in the closing assistant reply.
+
+    The metric extracts salient, non-stopword terms from earlier user turns,
+    excluding explicit requests (for example, questions or pleas for help). When the
+    final assistant response omits those key terms, the score drops, signalling the
+    assistant may have forgotten essential context. Scores range from `0.0`
+    (nothing retained) to `1.0` (all extracted facts referenced).
+
+    Args:
+        name: Display name for the metric result. Defaults to
+            ``"knowledge_retention_metric"``.
+        track: Whether the metric should automatically track results. Defaults to
+            ``True``.
+        project_name: Optional Opik project name for tracking. Defaults to ``None``.
+        turns_to_consider: How many of the earliest user turns should be mined for
+            facts. Defaults to ``5``.
+
+    Example:
+        >>> from opik.evaluation.metrics import KnowledgeRetentionMetric
+        >>> conversation = [
+        ...     {"role": "user", "content": "My new router is a Netgear Nighthawk."},
+        ...     {"role": "assistant", "content": "Great choice!"},
+        ...     {"role": "user", "content": "Please remind me about the Netgear router setup."},
+        ...     {
+        ...         "role": "assistant",
+        ...         "content": "Be sure to update the Netgear Nighthawk firmware first.",
+        ...     },
+        ... ]
+        >>> metric = KnowledgeRetentionMetric(turns_to_consider=3)
+        >>> result = metric.score(conversation)
+        >>> float(result.value)  # doctest: +SKIP
+        1.0
+    """
+
+    def __init__(
+        self,
+        name: str = "knowledge_retention_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        turns_to_consider: int = 5,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._turns_to_consider = turns_to_consider
+        self._normalizer: Callable[[str], str] = functools.partial(
+            normalize_text,
+            keep_emoji=False,
+            remove_punctuation=True,
+        )
+
+    def score(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: object,
+    ) -> ScoreResult:
+        user_turns: List[str] = []
+        assistant_turns: List[str] = []
+        for message in conversation:
+            content = message.get("content")
+            role = message.get("role")
+            if not content or not role:
+                continue
+            if role == "user":
+                if self._is_user_request(content):
+                    continue
+                user_turns.append(content)
+            elif role == "assistant":
+                assistant_turns.append(content)
+
+        if not assistant_turns:
+            return ScoreResult(
+                value=0.0, name=self.name, reason="No assistant turns", metadata={}
+            )
+
+        reference_facts = self._extract_terms(user_turns[: self._turns_to_consider])
+        if not reference_facts:
+            return ScoreResult(
+                value=1.0, name=self.name, reason="No facts to retain", metadata={}
+            )
+
+        final_response = assistant_turns[-1]
+        final_terms = self._extract_terms([final_response])
+
+        retained = reference_facts & final_terms
+        score = len(retained) / len(reference_facts)
+
+        metadata = {
+            "reference_terms": sorted(reference_facts),
+            "retained_terms": sorted(retained),
+        }
+
+        return ScoreResult(
+            value=score,
+            name=self.name,
+            reason=f"Retained {len(retained)} of {len(reference_facts)} reference terms",
+            metadata=metadata,
+        )
+
+    def _extract_terms(self, texts: Sequence[str]) -> Set[str]:
+        tokens: Set[str] = set()
+        for text in texts:
+            normalized = self._normalizer(text)
+            for token in normalized.split():
+                token = token.strip()
+                if len(token) < _MIN_TOKEN_LENGTH or token in _STOPWORDS:
+                    continue
+                tokens.add(token)
+        return tokens
+
+    def _is_user_request(self, text: str) -> bool:
+        if "?" in text:
+            return True
+
+        normalized = self._normalizer(text)
+        tokens = set(normalized.split())
+        return any(token in _REQUEST_KEYWORDS for token in tokens)

opik/evaluation/metrics/conversation/llm_judges/__init__.py

@@ -0,0 +1,32 @@
+"""
+LLM-judge conversation adapters.
+
+Expose the conversation-focused judges under
+`opik.evaluation.metrics.conversation.llm_judges.*`.
+"""
+
+from .conversational_coherence.metric import ConversationalCoherenceMetric
+from .g_eval_wrappers import (
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationPromptUncertaintyMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    GEvalConversationMetric,
+)
+from .session_completeness.metric import SessionCompletenessQuality
+from .user_frustration.metric import UserFrustrationMetric
+
+__all__ = [
+    "ConversationalCoherenceMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "GEvalConversationMetric",
+    "SessionCompletenessQuality",
+    "UserFrustrationMetric",
+]