opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/conversation/heuristics/knowledge_retention/metric.py

@@ -0,0 +1,172 @@
+"""Heuristic metric for knowledge retention across conversation turns."""
+
+from __future__ import annotations
+
+import functools
+from typing import Callable, List, Optional, Sequence, Set
+
+from opik.evaluation.metrics.conversation import types as conversation_types
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.score_result import ScoreResult
+from opik.evaluation.preprocessing import normalize_text
+
+_MIN_TOKEN_LENGTH = 4
+_STOPWORDS = {
+    "the",
+    "this",
+    "that",
+    "with",
+    "have",
+    "from",
+    "about",
+    "there",
+    "which",
+    "would",
+    "could",
+    "should",
+    "name",
+    "number",
+}
+
+_REQUEST_KEYWORDS = {
+    "need",
+    "please",
+    "help",
+    "can",
+    "could",
+    "would",
+    "should",
+    "shall",
+    "may",
+    "might",
+    "want",
+    "tell",
+    "explain",
+    "show",
+    "give",
+    "provide",
+    "let",
+    "allow",
+    "request",
+}
+
+
+class KnowledgeRetentionMetric(ConversationThreadMetric):
+    """
+    Measure how many user-provided facts resurface in the closing assistant reply.
+
+    The metric extracts salient, non-stopword terms from earlier user turns,
+    excluding explicit requests (for example, questions or pleas for help). When the
+    final assistant response omits those key terms, the score drops, signalling the
+    assistant may have forgotten essential context. Scores range from `0.0`
+    (nothing retained) to `1.0` (all extracted facts referenced).
+
+    Args:
+        name: Display name for the metric result. Defaults to
+            ``"knowledge_retention_metric"``.
+        track: Whether the metric should automatically track results. Defaults to
+            ``True``.
+        project_name: Optional Opik project name for tracking. Defaults to ``None``.
+        turns_to_consider: How many of the earliest user turns should be mined for
+            facts. Defaults to ``5``.
+
+    Example:
+        >>> from opik.evaluation.metrics import KnowledgeRetentionMetric
+        >>> conversation = [
+        ...     {"role": "user", "content": "My new router is a Netgear Nighthawk."},
+        ...     {"role": "assistant", "content": "Great choice!"},
+        ...     {"role": "user", "content": "Please remind me about the Netgear router setup."},
+        ...     {
+        ...         "role": "assistant",
+        ...         "content": "Be sure to update the Netgear Nighthawk firmware first.",
+        ...     },
+        ... ]
+        >>> metric = KnowledgeRetentionMetric(turns_to_consider=3)
+        >>> result = metric.score(conversation)
+        >>> float(result.value)  # doctest: +SKIP
+        1.0
+    """
+
+    def __init__(
+        self,
+        name: str = "knowledge_retention_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        turns_to_consider: int = 5,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self._turns_to_consider = turns_to_consider
+        self._normalizer: Callable[[str], str] = functools.partial(
+            normalize_text,
+            keep_emoji=False,
+            remove_punctuation=True,
+        )
+
+    def score(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: object,
+    ) -> ScoreResult:
+        user_turns: List[str] = []
+        assistant_turns: List[str] = []
+        for message in conversation:
+            content = message.get("content")
+            role = message.get("role")
+            if not content or not role:
+                continue
+            if role == "user":
+                if self._is_user_request(content):
+                    continue
+                user_turns.append(content)
+            elif role == "assistant":
+                assistant_turns.append(content)
+
+        if not assistant_turns:
+            return ScoreResult(
+                value=0.0, name=self.name, reason="No assistant turns", metadata={}
+            )
+
+        reference_facts = self._extract_terms(user_turns[: self._turns_to_consider])
+        if not reference_facts:
+            return ScoreResult(
+                value=1.0, name=self.name, reason="No facts to retain", metadata={}
+            )
+
+        final_response = assistant_turns[-1]
+        final_terms = self._extract_terms([final_response])
+
+        retained = reference_facts & final_terms
+        score = len(retained) / len(reference_facts)
+
+        metadata = {
+            "reference_terms": sorted(reference_facts),
+            "retained_terms": sorted(retained),
+        }
+
+        return ScoreResult(
+            value=score,
+            name=self.name,
+            reason=f"Retained {len(retained)} of {len(reference_facts)} reference terms",
+            metadata=metadata,
+        )
+
+    def _extract_terms(self, texts: Sequence[str]) -> Set[str]:
+        tokens: Set[str] = set()
+        for text in texts:
+            normalized = self._normalizer(text)
+            for token in normalized.split():
+                token = token.strip()
+                if len(token) < _MIN_TOKEN_LENGTH or token in _STOPWORDS:
+                    continue
+                tokens.add(token)
+        return tokens
+
+    def _is_user_request(self, text: str) -> bool:
+        if "?" in text:
+            return True
+
+        normalized = self._normalizer(text)
+        tokens = set(normalized.split())
+        return any(token in _REQUEST_KEYWORDS for token in tokens)

opik/evaluation/metrics/conversation/llm_judges/__init__.py

@@ -0,0 +1,32 @@
+"""
+LLM-judge conversation adapters.
+
+Expose the conversation-focused judges under
+`opik.evaluation.metrics.conversation.llm_judges.*`.
+"""
+
+from .conversational_coherence.metric import ConversationalCoherenceMetric
+from .g_eval_wrappers import (
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationPromptUncertaintyMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    GEvalConversationMetric,
+)
+from .session_completeness.metric import SessionCompletenessQuality
+from .user_frustration.metric import UserFrustrationMetric
+
+__all__ = [
+    "ConversationalCoherenceMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "GEvalConversationMetric",
+    "SessionCompletenessQuality",
+    "UserFrustrationMetric",
+]

opik/evaluation/metrics/conversation/{conversational_coherence → llm_judges/conversational_coherence}/metric.py

@@ -4,23 +4,21 @@ from typing import Optional, Union, Any, List, Dict
 
 import pydantic
 
+from opik.evaluation.metrics.conversation import helpers as conversation_helpers
+from opik.evaluation.metrics.conversation import types as conversation_types
 import opik.exceptions as exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.llm_judges import parsing_helpers
 from opik.evaluation.models import base_model, models_factory
 from . import schema, templates
-from .. import (
-    types as conversation_types,
-    conversation_thread_metric,
-    helpers,
-)
-from ... import score_result
-from ...llm_judges import parsing_helpers
 
 LOGGER = logging.getLogger(__name__)
 
 
-class ConversationalCoherenceMetric(
-    conversation_thread_metric.ConversationThreadMetric
-):
+class ConversationalCoherenceMetric(ConversationThreadMetric):
     """
     Calculates the conversational coherence metric for a given conversation thread.
     This metric assesses the coherence and relevance across a series of conversation
@@ -51,6 +49,7 @@ class ConversationalCoherenceMetric(
         window_size: The window size to use for calculating the score. It defines the
             maximal number of historical turns to include in each window when assessing
             the coherence of the current turn in the conversation. Default is 10.
+        temperature: The temperature to use for the model. Defaults to 1e-8.
 
     Example:
         >>> from opik.evaluation.metrics import ConversationalCoherenceMetric
@@ -76,23 +75,25 @@ class ConversationalCoherenceMetric(
         track: bool = True,
         project_name: Optional[str] = None,
         window_size: int = 10,
+        temperature: float = 1e-8,
     ):
         super().__init__(
             name=name,
             track=track,
             project_name=project_name,
         )
-        self._init_model(model)
         self._include_reason = include_reason
         self._window_size = window_size
 
+        self._init_model(model, temperature=temperature)
+
     def _init_model(
-        self, model: Optional[Union[str, base_model.OpikBaseModel]]
+        self, model: Optional[Union[str, base_model.OpikBaseModel]], temperature: float
     ) -> None:
         if isinstance(model, base_model.OpikBaseModel):
             self._model = model
         else:
-            self._model = models_factory.get(model_name=model)
+            self._model = models_factory.get(model_name=model, temperature=temperature)
 
     def score(
         self,
@@ -113,8 +114,10 @@ class ConversationalCoherenceMetric(
         conversation: conversation_types.Conversation,
     ) -> score_result.ScoreResult:
         try:
-            turns_windows = helpers.extract_turns_windows_from_conversation(
-                conversation=conversation, window_size=self._window_size
+            turns_windows = (
+                conversation_helpers.extract_turns_windows_from_conversation(
+                    conversation=conversation, window_size=self._window_size
+                )
             )
 
             verdicts = [
@@ -144,8 +147,10 @@ class ConversationalCoherenceMetric(
         conversation: conversation_types.Conversation,
     ) -> score_result.ScoreResult:
         try:
-            turns_windows = helpers.extract_turns_windows_from_conversation(
-                conversation=conversation, window_size=self._window_size
+            turns_windows = (
+                conversation_helpers.extract_turns_windows_from_conversation(
+                    conversation=conversation, window_size=self._window_size
+                )
             )
 
             verdicts = await asyncio.gather(

opik/evaluation/metrics/conversation/{conversational_coherence → llm_judges/conversational_coherence}/templates.py

@@ -1,6 +1,6 @@
 from typing import List, Dict
 
-from .. import types as conversation_types
+from opik.evaluation.metrics.conversation import types as conversation_types
 
 
 def evaluate_conversation(sliding_window: conversation_types.Conversation) -> str: