opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/conversation/llm_judges/user_frustration/metric.py

@@ -0,0 +1,277 @@
+import asyncio
+import logging
+from typing import Optional, Union, Any, List, Dict
+
+import pydantic
+
+from opik.evaluation.metrics.conversation import helpers as conversation_helpers
+from opik.evaluation.metrics.conversation import types as conversation_types
+import opik.exceptions as exceptions
+from opik.evaluation.metrics import score_result
+from opik.evaluation.metrics.conversation.conversation_thread_metric import (
+    ConversationThreadMetric,
+)
+from opik.evaluation.metrics.llm_judges import parsing_helpers
+from opik.evaluation.models import base_model, models_factory
+from . import schema, templates
+
+LOGGER = logging.getLogger(__name__)
+
+
+class UserFrustrationMetric(ConversationThreadMetric):
+    """
+    A heuristic score estimating the likelihood that the user experienced confusion, annoyance,
+    or disengagement during the session — due to repetition, lack of adaptation, ignored
+    intent signals, or failure to smoothly conclude.
+
+    The ``UserFrustrationMetric`` class integrates with LLM models to analyze
+    conversation data in sliding windows and produce a numerical score along with an optional
+    reason for the calculated score. It provides both synchronous and asynchronous methods for
+    calculation and supports customization through attributes like window size and reason inclusion.
+
+    This metric can be used to monitor and track user frustration levels during conversations, enabling
+    insights into user experience. The metric makes use of LLM models to score conversational
+    windows and summarize results. It returns a score between `0.0` and `1.0`. The higher the score,
+    the more frustrated the user is likely to be.
+
+    Args:
+        model: The model to use for evaluating the conversation. If a string is provided, it will be used to
+            fetch the model from the LiteLLM API. If a base_model.OpikBaseModel is
+            provided, it will be used directly. Default is None.
+        name: The name of the metric. The default is "user_frustration_score".
+        include_reason: Whether to include the reason for the score in the
+            result. Default is True.
+        track: Whether to track the metric. Default is True.
+        project_name: The name of the project to track the metric in.
+            Default is None.
+        window_size: The window size to use for calculating the score. It defines the
+            maximal number of historical turns to include in each window when assessing
+            the frustration of the current turn in the conversation. Default is 10.
+        temperature: The temperature to use for the model. Defaults to 1e-8.
+
+    Example:
+        >>> from opik.evaluation.metrics import UserFrustrationMetric
+        >>> conversation = [
+        >>>     {"role": "user", "content": "How do I center a div using CSS?"},
+        >>>     {"role": "assistant", "content": "There are many ways to center elements in CSS."},
+        >>>     {"role": "user", "content": "Okay... can you show me one?"},
+        >>>     {"role": "assistant", "content": "Sure. It depends on the context — are you centering horizontally, vertically, or both?"},
+        >>>     {"role": "user", "content": "Both. Just give me a basic example."},
+        >>> ]
+        >>> metric = UserFrustrationMetric()
+        >>> result = metric.score(conversation)
+        >>> if result.scoring_failed:
+        >>>     print(f"Scoring failed: {result.reason}")
+        >>> else:
+        >>>     print(result.value)
+    """
+
+    def __init__(
+        self,
+        model: Optional[Union[str, base_model.OpikBaseModel]] = None,
+        name: str = "user_frustration_score",
+        include_reason: bool = True,
+        track: bool = True,
+        project_name: Optional[str] = None,
+        window_size: int = 10,
+        temperature: float = 1e-8,
+    ):
+        super().__init__(
+            name=name,
+            track=track,
+            project_name=project_name,
+        )
+        self._include_reason = include_reason
+        self._window_size = window_size
+
+        self._init_model(model, temperature=temperature)
+
+    def _init_model(
+        self, model: Optional[Union[str, base_model.OpikBaseModel]], temperature: float
+    ) -> None:
+        if isinstance(model, base_model.OpikBaseModel):
+            self._model = model
+        else:
+            self._model = models_factory.get(model_name=model, temperature=temperature)
+
+    def score(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        return self._calculate_score(conversation=conversation)
+
+    async def ascore(
+        self,
+        conversation: conversation_types.Conversation,
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        return await self._a_calculate_score(conversation=conversation)
+
+    def _calculate_score(
+        self,
+        conversation: conversation_types.Conversation,
+    ) -> score_result.ScoreResult:
+        try:
+            turns_windows = (
+                conversation_helpers.extract_turns_windows_from_conversation(
+                    conversation=conversation, window_size=self._window_size
+                )
+            )
+
+            verdicts = [
+                self._evaluate_conversation(conversation_sliding_window=window)
+                for window in turns_windows
+            ]
+
+            score = _score_from_verdicts(verdicts=verdicts)
+            reason = (
+                self._reason_from_verdicts(score=score, verdicts=verdicts)
+                if self._include_reason
+                else None
+            )
+            return score_result.ScoreResult(
+                name=self.name,
+                value=score,
+                reason=reason,
+            )
+        except Exception as e:
+            LOGGER.error(f"Failed to calculate user frustration score: {e}")
+            raise exceptions.MetricComputationError(
+                f"Failed to calculate user frustration score: {e}"
+            ) from e
+
+    async def _a_calculate_score(
+        self,
+        conversation: conversation_types.Conversation,
+    ) -> score_result.ScoreResult:
+        try:
+            turns_windows = (
+                conversation_helpers.extract_turns_windows_from_conversation(
+                    conversation=conversation, window_size=self._window_size
+                )
+            )
+
+            verdicts = await asyncio.gather(
+                *[
+                    self._a_evaluate_conversation(conversation_sliding_window=window)
+                    for window in turns_windows
+                ]
+            )
+
+            score = _score_from_verdicts(verdicts=verdicts)
+            reason = (
+                await self._a_reason_from_verdicts(score=score, verdicts=verdicts)
+                if self._include_reason
+                else None
+            )
+            return score_result.ScoreResult(
+                name=self.name,
+                value=score,
+                reason=reason,
+            )
+        except Exception as e:
+            LOGGER.error(f"Failed to calculate user frustration score: {e}")
+            raise exceptions.MetricComputationError(
+                f"Failed to calculate user frustration score: {e}"
+            ) from e
+
+    def _evaluate_conversation(
+        self,
+        conversation_sliding_window: conversation_types.Conversation,
+    ) -> schema.EvaluateUserFrustrationResponse:
+        llm_query = templates.evaluate_conversation(
+            sliding_window=conversation_sliding_window
+        )
+        model_output = self._model.generate_string(
+            input=llm_query,
+            response_format=schema.EvaluateUserFrustrationResponse,
+        )
+        return _evaluate_conversation_from_model_output(model_output=model_output)
+
+    async def _a_evaluate_conversation(
+        self,
+        conversation_sliding_window: conversation_types.Conversation,
+    ) -> schema.EvaluateUserFrustrationResponse:
+        llm_query = templates.evaluate_conversation(
+            sliding_window=conversation_sliding_window
+        )
+        model_output = await self._model.agenerate_string(
+            input=llm_query,
+            response_format=schema.EvaluateUserFrustrationResponse,
+        )
+        return _evaluate_conversation_from_model_output(model_output=model_output)
+
+    def _reason_from_verdicts(
+        self, score: float, verdicts: List[schema.EvaluateUserFrustrationResponse]
+    ) -> str:
+        frustrations: List[Dict[str, str]] = _extract_frustrations_from_verdicts(
+            verdicts
+        )
+
+        llm_query = templates.generate_reason(score=score, frustrations=frustrations)
+
+        model_output = self._model.generate_string(
+            input=llm_query, response_format=schema.ScoreReasonResponse
+        )
+        return _generate_reason_from_model_output(model_output=model_output)
+
+    async def _a_reason_from_verdicts(
+        self, score: float, verdicts: List[schema.EvaluateUserFrustrationResponse]
+    ) -> str:
+        frustrations: List[Dict[str, str]] = _extract_frustrations_from_verdicts(
+            verdicts
+        )
+
+        llm_query = templates.generate_reason(score=score, frustrations=frustrations)
+
+        model_output = await self._model.agenerate_string(
+            input=llm_query, response_format=schema.ScoreReasonResponse
+        )
+        return _generate_reason_from_model_output(model_output=model_output)
+
+
+def _evaluate_conversation_from_model_output(
+    model_output: str,
+) -> schema.EvaluateUserFrustrationResponse:
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(model_output)
+        return schema.EvaluateUserFrustrationResponse.model_validate(dict_content)
+    except pydantic.ValidationError as e:
+        LOGGER.warning(
+            f"Failed to parse user's frustration evaluation results from the LLM output: {model_output}, reason: {e}",
+            exc_info=True,
+        )
+        raise e
+
+
+def _score_from_verdicts(
+    verdicts: List[schema.EvaluateUserFrustrationResponse],
+) -> float:
+    if len(verdicts) == 0:
+        return 0.0
+
+    frustrated_count = sum(v.verdict.strip().lower() == "yes" for v in verdicts)
+    return frustrated_count / len(verdicts)
+
+
+def _extract_frustrations_from_verdicts(
+    verdicts: List[schema.EvaluateUserFrustrationResponse],
+) -> List[Dict[str, str]]:
+    return [
+        {"message_number": f"{index + 1}", "reason": verdict.reason}
+        for index, verdict in enumerate(verdicts)
+        if verdict.verdict.strip().lower() == "yes" and verdict.reason is not None
+    ]
+
+
+def _generate_reason_from_model_output(model_output: str) -> str:
+    try:
+        dict_content = parsing_helpers.extract_json_content_or_raise(model_output)
+        return schema.ScoreReasonResponse.model_validate(dict_content).reason
+    except pydantic.ValidationError as e:
+        LOGGER.warning(
+            f"Failed to parse frustration score reason from the LLM output: {model_output}, reason: {e}",
+            exc_info=True,
+        )
+        raise e

opik/evaluation/metrics/conversation/llm_judges/user_frustration/schema.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+import pydantic
+
+
+class EvaluateUserFrustrationResponse(pydantic.BaseModel):
+    verdict: str
+    reason: Optional[str] = pydantic.Field(default=None)
+
+    __hash__ = object.__hash__
+
+
+class ScoreReasonResponse(pydantic.BaseModel):
+    reason: str
+
+    __hash__ = object.__hash__

opik/evaluation/metrics/conversation/llm_judges/user_frustration/templates.py

@@ -0,0 +1,135 @@
+from typing import List, Dict
+
+from opik.evaluation.metrics.conversation import types as conversation_types
+
+
+def evaluate_conversation(sliding_window: conversation_types.Conversation) -> str:
+    return f"""Based on the given list of message exchanges between a user and an LLM, generate a JSON object to indicate whether the LAST `user` message is showing that the user experiences confusion, annoyance, or disengagement during the conversation session given in the context of the last messages.
+
+** Guidelines: **
+- Make sure to only return in JSON format.
+- The JSON must have only 2 fields: 'verdict' and 'reason'.
+- The 'verdict' key should STRICTLY be either 'yes' or 'no', which states whether the last `user` message is showing that the user experienced confusion, annoyance, or disengagement during the conversation session given in the context of the last messages.
+- Provide a 'reason' ONLY if the answer is 'yes'.
+- You DON'T have to provide a reason if the answer is 'no'.
+- You MUST USE the previous messages (if any) provided in the list of messages to make an informed judgement on user frustration.
+- You MUST ONLY provide a verdict for the LAST message on the list but MUST USE context from the previous messages.
+- ONLY provide a 'yes' answer if the LLM response caused the user to express COMPLETE frustration or confusion in his/her input messages.
+- Vague LLM responses to vague inputs, such as greetings DOES NOT count as causes of frustration!
+- You should mention LLM response instead of `assistant`, and User instead of `user`.
+
+===== Start OF EXAMPLE ======
+** Example Turns: **
+[
+    {{
+        "role": "user",
+        "content": "Hello!"
+    }},
+    {{
+        "role": "assistant",
+        "content": "Hi there!"
+    }},
+    {{
+        "role": "user",
+        "content": "Can you help me write a function to parse dates in Python?"
+    }},
+    {{
+        "role": "assistant",
+        "content": "Of course! Here's a basic example using `datetime.strptime`:\n\n```python\nfrom datetime import datetime\n\ndef parse_date(date_string):\n    return datetime.strptime(date_string, '%Y-%m-%d')\n```"
+    }},
+    {{
+        "role": "user",
+        "content": "That's not what I meant. I need it to handle multiple formats."
+    }},
+    {{
+        "role": "assistant",
+        "content": "Ah, got it. Here's an updated version that tries multiple formats:\n\n```python\nfrom datetime import datetime\n\ndef parse_date(date_string):\n    for fmt in ('%Y-%m-%d', '%d/%m/%Y', '%m-%d-%Y'):\n        try:\n            return datetime.strptime(date_string, fmt)\n        except ValueError:\n            continue\n    raise ValueError('No valid date format found')\n```"
+    }},
+    {{
+        "role": "user",
+        "content": "Why didn’t you just give me this the first time? This is basic."
+    }},
+    {{
+        "role": "assistant",
+        "content": "content": "Apologies for the oversight. I always aim to provide the most helpful answer based on the initial question."
+    }},
+    {{
+        "role": "user",
+        "content": "Well, it's kind of frustrating to keep asking for obvious things. Can you at least include error handling too next time?"
+    }},
+    {{
+        "role": "assistant",
+        "content": "You're right, I’ll be more thorough. Let me know if you'd like to expand the function with logging or timezone support as well."
+
+    }},
+    {{
+        "role": "user",
+        "content": "Now you're overcomplicating it. I just wanted a simple, robust function."
+    }},
+    {{
+        "role": "assistant",
+        "content": "Understood. Here's a simpler version with basic error handling included:\n\n```python\nfrom datetime import datetime\n\ndef parse_date(date_string):\n    formats = ('%Y-%m-%d', '%d/%m/%Y', '%m-%d-%Y')\n    for fmt in formats:\n        try:\n            return datetime.strptime(date_string, fmt)\n        except ValueError:\n            continue\n    return None  # Or raise an error if preferred\n```"
+    }},
+    {{
+        "role": "user", "content": "You could’ve just started with this. Why is it so hard to get a straight answer?"
+    }}
+]
+
+** Example output JSON **
+{{
+    "verdict": "yes",
+    "reason": "The User repeatedly clarifies their intent and expresses dissatisfaction with the LLM’s initial responses, indicating a mismatch between the User’s expectations and the LLM’s output. Despite asking a clear question, the LLM provides an overly simplistic solution at first, requiring the User to iterate and request obvious improvements (e.g., support for multiple formats, error handling). The User’s tone becomes increasingly critical, with statements like “Why didn’t you just give me this the first time?” and “Why is it so hard to get a straight answer?”, signaling rising frustration due to perceived inefficiency and lack of responsiveness from the assistant."
+}}
+===== END OF EXAMPLE ======
+
+** Turns: **
+{sliding_window}
+
+** JSON: **
+"""
+
+
+def generate_reason(score: float, frustrations: List[Dict[str, str]]) -> str:
+    return f"""Below is a list of user frustrations drawn from some messages in a conversation, which you have minimal knowledge of. It is a list of strings explaining why the 'assistant' messages caused the 'user' to be frustrated.
+Given the frustration score, which is a 0-1 score indicating how frustrating the OVERALL AI 'assistant' messages are in a conversation (higher the more frustrating).
+
+** Guidelines: **
+- Make sure to only return in JSON format, with the 'reason' key providing the reason.
+- Always quote WHICH MESSAGE and the INFORMATION in the reason in your final reason.
+- Be confident in your reasoning, as if you’re aware of the `assistant` messages from the messages in a conversation that led to the frustrations.
+- You should CONCISELY summarize the user frustrations to justify the score.
+- You should NOT mention concrete frustration in your reason, and make the reason sound convincing.
+- You should mention LLM response instead of `assistant`, and User instead of `user`.
+- You should format <frustration_score> to use 1 decimal place in the reason.
+
+===== Start OF EXAMPLE ======
+** Example frustrations: **
+[
+    {{
+        "message_number": "1",
+        "reason": "The User repeatedly clarifies their intent and expresses dissatisfaction with the LLM’s initial responses, indicating a mismatch between the User’s expectations and the LLM’s output."
+    }},
+    {{
+        "message_number": "2",
+        "reason": "Despite asking a clear question, the LLM provides an overly simplistic solution at first, requiring the User to iterate and request obvious improvements (e.g., support for multiple formats, error handling)."
+    }},
+    {{
+        "message_number": "3",
+        "reason": "The User’s tone becomes increasingly critical, with statements like “Why didn’t you just give me this the first time?” and “Why is it so hard to get a straight answer?”, signaling rising frustration due to perceived inefficiency and lack of responsiveness from the assistant."
+    }}
+]
+
+** Example output JSON **
+{{
+    "reason": "The score is <frustration_score> because <your_reason>."
+}}
+===== END OF EXAMPLE ======
+
+** Frustration Score: **
+{score}
+
+** Frustrations: **
+{frustrations}
+
+** JSON: **
+"""

opik/evaluation/metrics/conversation/types.py

@@ -0,0 +1,34 @@
+from typing import Dict, List, Literal, Optional
+
+import pydantic
+
+ConversationDict = Dict[Literal["role", "content"], str]
+Conversation = List[ConversationDict]
+
+
+class ConversationTurn(pydantic.BaseModel):
+    """
+    Representation of a single turn in a conversation.
+    This class defines a model for encapsulating a single conversational
+    turn consisting of an input user's message and an output LLM message. It is
+    designed to handle the exchange of messages in a structured format.
+    Args:
+        input: The input message of the conversation turn.
+        output: The output message of the conversation turn.
+    Example:
+        >>> conversation_turn = ConversationTurn(
+        >>>     input={"role": "user", "content": "Hello!"},
+        >>>     output={"role": "assistant", "content": "Hi there! How can I help you today?"}
+        >>> )
+    """
+
+    input: ConversationDict
+    output: Optional[ConversationDict]
+
+    def as_list(self) -> List[ConversationDict]:
+        if self.output is None:
+            return [self.input]
+        return [self.input, self.output]
+
+
+__all__ = ["ConversationDict", "Conversation", "ConversationTurn"]

opik/evaluation/metrics/conversation_types.py

@@ -0,0 +1,9 @@
+"""Public aliases for conversation type helpers."""
+
+from .conversation.types import (
+    Conversation,
+    ConversationDict,
+    ConversationTurn,
+)
+
+__all__ = ["Conversation", "ConversationDict", "ConversationTurn"]

opik/evaluation/metrics/heuristics/bertscore.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional, Sequence, Tuple, Union
+
+from opik.exceptions import MetricComputationError
+from opik.evaluation.metrics import base_metric, score_result
+
+try:  # pragma: no cover - optional dependency
+    from bert_score import score as bert_score_fn
+except ImportError:  # pragma: no cover - optional dependency
+    bert_score_fn = None
+
+
+BertScoreFn = Callable[
+    [Sequence[str], Union[Sequence[str], Sequence[Sequence[str]]]], Tuple[Any, Any, Any]
+]
+
+
+class BERTScore(base_metric.BaseMetric):
+    """Wrapper around the `bert-score` library.
+
+    Args:
+        scorer_fn: Optional callable compatible with ``bert_score.score`` for
+            dependency injection or advanced usage.
+        model_type: Model checkpoint to use when loading the scorer. Ignored when
+            ``scorer_fn`` is provided.
+        lang: Two-letter language code used by the default scorer.
+        rescale_with_baseline: Whether to rescale the score using the provided
+            baseline statistics.
+        device: Optional device string forwarded to ``bert_score`` (e.g., "cpu",
+            "cuda").
+    """
+
+    def __init__(
+        self,
+        scorer_fn: Optional[BertScoreFn] = None,
+        model_type: Optional[str] = "bert-base-uncased",
+        lang: Optional[str] = "en",
+        rescale_with_baseline: bool = False,
+        device: Optional[str] = None,
+        name: str = "bertscore_metric",
+        track: bool = True,
+        project_name: Optional[str] = None,
+        **scorer_kwargs: Any,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+
+        if scorer_fn is not None:
+            self._scorer_fn = scorer_fn
+        else:
+            if bert_score_fn is None:  # pragma: no cover - optional dependency
+                raise ImportError(
+                    "BERTScore metric requires the optional 'bert-score' package. "
+                    "Install via `pip install bert-score` or provide `scorer_fn`."
+                )
+
+            def _score(
+                candidates: Sequence[str],
+                references: Union[Sequence[str], Sequence[Sequence[str]]],
+            ) -> Tuple[Any, Any, Any]:
+                return bert_score_fn(
+                    candidates,
+                    references,
+                    model_type=model_type,
+                    lang=lang,
+                    rescale_with_baseline=rescale_with_baseline,
+                    device=device,
+                    **scorer_kwargs,
+                )
+
+            self._scorer_fn = _score
+
+    def score(
+        self,
+        output: str,
+        reference: Union[str, Sequence[str], Sequence[Sequence[str]]],
+        **ignored_kwargs: Any,
+    ) -> score_result.ScoreResult:
+        if not output.strip():
+            raise MetricComputationError("Candidate is empty (BERTScore metric).")
+
+        references: Union[Sequence[str], Sequence[Sequence[str]]]
+        if isinstance(reference, str):
+            references = [reference]
+        else:
+            references = reference
+            if isinstance(reference, Sequence) and len(reference) == 0:
+                raise MetricComputationError("Reference is empty (BERTScore metric).")
+
+        precision, recall, f1 = self._scorer_fn([output], references)
+
+        score_value = float(f1[0].item() if hasattr(f1[0], "item") else f1[0])
+        metadata = {
+            "precision": float(
+                precision[0].item() if hasattr(precision[0], "item") else precision[0]
+            ),
+            "recall": float(
+                recall[0].item() if hasattr(recall[0], "item") else recall[0]
+            ),
+        }
+
+        return score_result.ScoreResult(
+            value=score_value,
+            name=self.name,
+            reason=f"BERTScore F1: {score_value:.4f}",
+            metadata=metadata,
+        )