opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/metrics/__init__.py

@@ -1,37 +1,152 @@
+from .aggregated_metric import AggregatedMetric
+
+# Keep the canonical import first for the new layout while still tolerating
+# older packaging artefacts (some environments import this module before the
+# conversation package is available).  If the eager import fails we fall back
+# to the lazy getter below, letting legacy entry-points keep working.
+from .conversation.conversation_thread_metric import ConversationThreadMetric
+
+from .conversation import types as conversation_types
+from .conversation.heuristics.degeneration.metric import ConversationDegenerationMetric
+from .conversation.heuristics.knowledge_retention.metric import (
+    KnowledgeRetentionMetric,
+)
+from .conversation.llm_judges.conversational_coherence.metric import (
+    ConversationalCoherenceMetric,
+)
+from .conversation.llm_judges.g_eval_wrappers import (
+    GEvalConversationMetric,
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    ConversationPromptUncertaintyMetric,
+)
+from .conversation.llm_judges.session_completeness.metric import (
+    SessionCompletenessQuality,
+)
+from .conversation.llm_judges.user_frustration.metric import UserFrustrationMetric
 from .heuristics.contains import Contains
 from .heuristics.equals import Equals
+from .heuristics.gleu import GLEU
+from .heuristics.chrf import ChrF
 from .heuristics.is_json import IsJson
+from .heuristics.distribution_metrics import (
+    JSDivergence,
+    JSDistance,
+    KLDivergence,
+)
 from .heuristics.levenshtein_ratio import LevenshteinRatio
+from .heuristics.meteor import METEOR
+from .heuristics.bertscore import BERTScore
+from .heuristics.spearman import SpearmanRanking
+from .heuristics.readability import Readability
+from .heuristics.tone import Tone
+from .heuristics.prompt_injection import PromptInjection
+from .heuristics.language_adherence import LanguageAdherenceMetric
 from .heuristics.regex_match import RegexMatch
 from .heuristics.bleu import SentenceBLEU, CorpusBLEU
+from .heuristics.rouge import ROUGE
+from .heuristics.sentiment import Sentiment
+from .heuristics.vader_sentiment import VADERSentiment
 from .llm_judges.answer_relevance.metric import AnswerRelevance
+from .llm_judges.g_eval_presets import (
+    AgentTaskCompletionJudge,
+    AgentToolCorrectnessJudge,
+    ComplianceRiskJudge,
+    DemographicBiasJudge,
+    DialogueHelpfulnessJudge,
+    GenderBiasJudge,
+    PoliticalBiasJudge,
+    PromptUncertaintyJudge,
+    QARelevanceJudge,
+    RegionalBiasJudge,
+    ReligiousBiasJudge,
+    SummarizationCoherenceJudge,
+    SummarizationConsistencyJudge,
+)
 from .llm_judges.context_precision.metric import ContextPrecision
 from .llm_judges.context_recall.metric import ContextRecall
-from .llm_judges.g_eval.metric import GEval
+from .llm_judges.g_eval.metric import GEval, GEvalPreset
 from .llm_judges.hallucination.metric import Hallucination
 from .llm_judges.moderation.metric import Moderation
+from .llm_judges.llm_juries.metric import LLMJuriesJudge
+from .llm_judges.trajectory_accuracy import TrajectoryAccuracy
+from .llm_judges.syc_eval.metric import SycEval
 from .llm_judges.usefulness.metric import Usefulness
+from .llm_judges.structure_output_compliance.metric import StructuredOutputCompliance
 from .base_metric import BaseMetric
+from .ragas_metric import RagasMetricWrapper
 from opik.exceptions import MetricComputationError
 
 # from .llm_judges.factuality.metric import Factuality
 
 __all__ = [
+    "AggregatedMetric",
     "AnswerRelevance",
+    "AgentTaskCompletionJudge",
+    "AgentToolCorrectnessJudge",
+    "BaseMetric",
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+    "GEvalConversationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "ConversationPromptUncertaintyMetric",
+    "conversation_types",
+    "ComplianceRiskJudge",
     "Contains",
     "ContextPrecision",
     "ContextRecall",
+    "ConversationalCoherenceMetric",
+    "CorpusBLEU",
+    "DemographicBiasJudge",
     "Equals",
-    # "Factuality",
     "GEval",
+    "GEvalPreset",
+    "GLEU",
+    "GenderBiasJudge",
     "Hallucination",
     "IsJson",
+    "JSDivergence",
+    "JSDistance",
+    "KLDivergence",
     "LevenshteinRatio",
+    "BERTScore",
+    "METEOR",
+    "ChrF",
+    "Readability",
+    "PromptInjection",
+    "LanguageAdherenceMetric",
+    "PoliticalBiasJudge",
+    "PromptUncertaintyJudge",
+    "SpearmanRanking",
+    "ReligiousBiasJudge",
+    "RegionalBiasJudge",
+    "VADERSentiment",
+    "Tone",
+    "StructuredOutputCompliance",
+    "MetricComputationError",
     "Moderation",
-    "Usefulness",
+    "RagasMetricWrapper",
     "RegexMatch",
-    "MetricComputationError",
-    "BaseMetric",
+    "ROUGE",
     "SentenceBLEU",
-    "CorpusBLEU",
+    "Sentiment",
+    "SessionCompletenessQuality",
+    "SycEval",
+    "Usefulness",
+    "UserFrustrationMetric",
+    "TrajectoryAccuracy",
+    "DialogueHelpfulnessJudge",
+    "QARelevanceJudge",
+    "SummarizationCoherenceJudge",
+    "SummarizationConsistencyJudge",
+    "LLMJuriesJudge",
+    "ConversationThreadMetric",
+    # "Factuality",
 ]

opik/evaluation/metrics/aggregated_metric.py

@@ -0,0 +1,92 @@
+from typing import List, Callable, Any, Dict, Optional
+
+from .. import types as evaluation_types
+
+from . import arguments_helpers, arguments_validator, base_metric, score_result
+
+
+class AggregatedMetric(
+    base_metric.BaseMetric, arguments_validator.ScoreArgumentsValidator
+):
+    """
+    Combine the output of multiple metrics into a single aggregated ``ScoreResult``.
+
+    Each metric in ``metrics`` is executed with the provided scoring kwargs, then the
+    ``aggregator`` callback decides how to merge the individual results. This is
+    handy for building ensembles such as min/max, weighted averages, or custom
+    pass/fail checks without re-implementing the metrics themselves.
+
+    Args:
+        name: Display name for the aggregated metric result.
+        metrics: Ordered list of metric instances that should be executed.
+        aggregator: Callable receiving the list of ``ScoreResult`` objects and
+            returning the final aggregated ``ScoreResult``.
+        track: Whether to automatically track the metric in Opik. Defaults to
+            ``True``.
+        project_name: Optional tracking project used when no parent context exists.
+
+    Example:
+        >>> from opik.evaluation.metrics import AggregatedMetric, Contains, RegexMatch
+        >>> metrics = [Contains(track=False), RegexMatch(pattern=r"\\d+", track=False)]
+        >>> from opik.evaluation.metrics import score_result
+        >>> def combine(results):
+        ...     score = sum(result.value for result in results) / len(results)
+        ...     return score_result.ScoreResult(
+        ...         name="combined_contains_regex",
+        ...         value=score,
+        ...         reason="Average of contains and regex checks",
+        ...     )
+        >>> metric = AggregatedMetric(
+        ...     name="combined_contains_regex",
+        ...     metrics=metrics,
+        ...     aggregator=combine,
+        ... )
+        >>> response = "Order number 12345 confirmed"
+        >>> result = metric.score(output=response, reference="order")
+        >>> float(result.value)  # doctest: +SKIP
+        1.0
+    """
+
+    def __init__(
+        self,
+        name: str,
+        metrics: List[base_metric.BaseMetric],
+        aggregator: Callable[
+            [List[score_result.ScoreResult]], score_result.ScoreResult
+        ],
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ):
+        super().__init__(name=name, track=track, project_name=project_name)
+        self.metrics = metrics
+        self.aggregator = aggregator
+
+        if self.metrics is None or len(self.metrics) == 0:
+            raise ValueError("No metrics provided")
+
+        if aggregator is None:
+            raise ValueError("No aggregator provided")
+
+    def score(self, **kwargs: Any) -> score_result.ScoreResult:
+        score_results: List[score_result.ScoreResult] = []
+        for metric in self.metrics:
+            metric_result = metric.score(**kwargs)
+            if isinstance(metric_result, list):
+                score_results.extend(metric_result)
+            else:
+                score_results.append(metric_result)
+
+        return self.aggregator(score_results)
+
+    def validate_score_arguments(
+        self,
+        score_kwargs: Dict[str, Any],
+        key_mapping: Optional[evaluation_types.ScoringKeyMappingType],
+    ) -> None:
+        for metric in self.metrics:
+            arguments_helpers.raise_if_score_arguments_are_missing(
+                score_function=metric.score,
+                score_name=metric.name,
+                kwargs=score_kwargs,
+                scoring_key_mapping=key_mapping,
+            )

opik/evaluation/metrics/arguments_helpers.py

@@ -1,9 +1,9 @@
 import inspect
 import logging
-from typing import Any, Callable, Dict, List, Optional, Set
+from typing import Any, Callable, Dict, List, Optional
 
-from opik import exceptions
-from opik.evaluation.types import ScoringKeyMappingType
+import opik.exceptions as exceptions
+from .. import types as evaluation_types
 
 LOGGER = logging.getLogger(__name__)
 
@@ -12,7 +12,7 @@ def raise_if_score_arguments_are_missing(
     score_function: Callable,
     score_name: str,
     kwargs: Dict[str, Any],
-    scoring_key_mapping: Optional[ScoringKeyMappingType],
+    scoring_key_mapping: Optional[evaluation_types.ScoringKeyMappingType],
 ) -> None:
     signature = inspect.signature(score_function)
 
@@ -32,31 +32,25 @@ def raise_if_score_arguments_are_missing(
                 missing_required_arguments.append(name)
 
     if len(missing_required_arguments) > 0:
-        message = (
-            f"The scoring method {score_name} is missing arguments: {missing_required_arguments}. "
-            f"These keys were not present in either the dataset item or the dictionary returned by the evaluation task. "
-            f"You can either update the dataset or evaluation task to return this key or use the `scoring_key_mapping` to map existing items to the expected arguments."
-            f"The available keys found in the dataset item and evaluation task output are: {list(kwargs.keys())}."
-        )
-
-        unused_mapping_arguments: Set[str] = set()
+        unused_mapping_arguments: List[str] = []
         if scoring_key_mapping:
-            unused_mapping_arguments = set(
-                key for key in scoring_key_mapping.values() if not callable(key)
+            unused_mapping_arguments = list(
+                set(key for key in scoring_key_mapping.values() if not callable(key))
+                - set(kwargs.keys())
             )
 
-        unused_mapping_arguments -= set(kwargs.keys())
-
-        if len(unused_mapping_arguments) > 0:
-            message += f" Some keys in `scoring_key_mapping` didn't match anything: {unused_mapping_arguments}"
-
-        raise exceptions.ScoreMethodMissingArguments(message)
+        raise exceptions.ScoreMethodMissingArguments(
+            score_name,
+            missing_required_arguments,
+            list(kwargs.keys()),
+            unused_mapping_arguments,
+        )
 
 
 def create_scoring_inputs(
     dataset_item: Dict[str, Any],
     task_output: Dict[str, Any],
-    scoring_key_mapping: Optional[ScoringKeyMappingType],
+    scoring_key_mapping: Optional[evaluation_types.ScoringKeyMappingType],
 ) -> Dict[str, Any]:
     mapped_inputs = {**dataset_item, **task_output}
 

opik/evaluation/metrics/arguments_validator.py

@@ -0,0 +1,38 @@
+import abc
+from typing import Dict, Any, Optional
+
+from . import base_metric, arguments_helpers
+from .. import types as evaluation_types
+
+
+class ScoreArgumentsValidator(abc.ABC):
+    """Abstract base class for all metrics allowing validation of score method's arguments.
+    It validates if arguments to be provided to the score method are valid, otherwise it raises an error.
+    """
+
+    @abc.abstractmethod
+    def validate_score_arguments(
+        self,
+        score_kwargs: Dict[str, Any],
+        key_mapping: Optional[evaluation_types.ScoringKeyMappingType],
+    ) -> None:
+        """The subclasses must implement this method to provide actual validation logic."""
+        pass
+
+
+def validate_score_arguments(
+    metric: base_metric.BaseMetric,
+    kwargs: Dict[str, Any],
+    scoring_key_mapping: Optional[evaluation_types.ScoringKeyMappingType],
+) -> None:
+    if isinstance(metric, ScoreArgumentsValidator):
+        metric.validate_score_arguments(
+            score_kwargs=kwargs, key_mapping=scoring_key_mapping
+        )
+    else:
+        arguments_helpers.raise_if_score_arguments_are_missing(
+            score_function=metric.score,
+            score_name=metric.name,
+            kwargs=kwargs,
+            scoring_key_mapping=scoring_key_mapping,
+        )

opik/evaluation/metrics/base_metric.py

@@ -1,8 +1,8 @@
 import abc
-from typing import Any, List, Union
+from typing import Any, List, Optional, Union
 
 import opik
-from opik import config as opik_config
+import opik.config as opik_config
 from ..metrics import score_result
 
 
@@ -12,19 +12,20 @@ class BaseMetric(abc.ABC):
     from this class and implement the abstract methods.
 
     Args:
-        name: The name of the metric.
+        name: The name of the metric. If not provided, uses the class name as default.
         track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there is no parent span/trace to inherit project name from.
 
     Example:
         >>> from opik.evaluation.metrics import base_metric, score_result
         >>>
         >>> class MyCustomMetric(base_metric.BaseMetric):
         >>>     def __init__(self, name: str, track: bool = True):
-        >>>         self.name = name
-        >>>         self.track = track
+        >>>         super().__init__(name=name, track=track)
         >>>
         >>>     def score(self, input: str, output: str, **ignored_kwargs: Any):
-        >>>         # Add you logic here
+        >>>         # Add your logic here
         >>>
         >>>         return score_result.ScoreResult(
         >>>             value=0,
@@ -33,15 +34,24 @@ class BaseMetric(abc.ABC):
         >>>         )
     """
 
-    def __init__(self, name: str, track: bool = True) -> None:
-        self.name = name
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        self.name = name if name is not None else self.__class__.__name__
         self.track = track
 
         config = opik_config.OpikConfig()
 
+        if not track and project_name is not None:
+            raise ValueError("project_name can be set only when `track` is set to True")
+
         if track and config.check_for_known_misconfigurations() is False:
-            self.score = opik.track(name=self.name)(self.score)  # type: ignore
-            self.ascore = opik.track(name=self.name)(self.ascore)  # type: ignore
+            track_decorator = opik.track(name=self.name, project_name=project_name)
+            self.score = track_decorator(self.score)  # type: ignore
+            self.ascore = track_decorator(self.ascore)  # type: ignore
 
     @abc.abstractmethod
     def score(

opik/evaluation/metrics/conversation/__init__.py

@@ -0,0 +1,48 @@
+"""Public conversation metrics API."""
+
+from .conversation_thread_metric import ConversationThreadMetric
+from .conversation_turns_factory import build_conversation_turns
+from .helpers import (
+    extract_turns_windows_from_conversation,
+    get_turns_in_sliding_window,
+    merge_turns,
+)
+from .types import Conversation, ConversationDict, ConversationTurn
+
+__all__ = [
+    "ConversationThreadMetric",
+    "Conversation",
+    "ConversationDict",
+    "ConversationTurn",
+    "build_conversation_turns",
+    "extract_turns_windows_from_conversation",
+    "get_turns_in_sliding_window",
+    "merge_turns",
+    "ConversationDegenerationMetric",
+    "KnowledgeRetentionMetric",
+    "ConversationalCoherenceMetric",
+    "SessionCompletenessQuality",
+    "UserFrustrationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+    "GEvalConversationMetric",
+]
+
+from .heuristics.degeneration.metric import ConversationDegenerationMetric
+from .heuristics.knowledge_retention.metric import KnowledgeRetentionMetric
+from .llm_judges.conversational_coherence.metric import ConversationalCoherenceMetric
+from .llm_judges.g_eval_wrappers import (
+    GEvalConversationMetric,
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    ConversationPromptUncertaintyMetric,
+)
+from .llm_judges.session_completeness.metric import SessionCompletenessQuality
+from .llm_judges.user_frustration.metric import UserFrustrationMetric

opik/evaluation/metrics/conversation/conversation_thread_metric.py

@@ -0,0 +1,79 @@
+from typing import Union, List, Any
+
+from . import types
+from .. import base_metric, score_result
+
+
+class ConversationThreadMetric(base_metric.BaseMetric):
+    """
+    Abstract base class for all conversation thread metrics. When creating a custom
+    conversation metric, you should inherit from this class and implement the abstract methods.
+
+    Conversation metrics are designed to evaluate multi-turn conversations rather than
+    single input-output pairs. They accept a conversation as a list of message dictionaries,
+    where each message has a 'role' (either 'user' or 'assistant') and 'content'.
+
+    Args:
+        name: The name of the metric. If not provided, uses the class name as default.
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name to track the metric in for the cases when
+            there is no parent span/trace to inherit project name from.
+
+    Example:
+        >>> from opik.evaluation.metrics.conversation import conversation_thread_metric, types
+        >>> from opik.evaluation.metrics import score_result
+        >>> from typing import Any
+        >>>
+        >>> class ConversationLengthMetric(conversation_thread_metric.ConversationThreadMetric):
+        >>>     def __init__(self, name: str = "conversation_length_score"):
+        >>>         super().__init__(name)
+        >>>
+        >>>     def score(self, conversation: types.Conversation, **kwargs: Any):
+        >>>         num_turns = sum(1 for msg in conversation if msg["role"] == "assistant")
+        >>>         return score_result.ScoreResult(
+        >>>             name=self.name,
+        >>>             value=num_turns,
+        >>>             reason=f"Conversation has {num_turns} turns"
+        >>>         )
+    """
+
+    def score(
+        self, conversation: types.Conversation, **kwargs: Any
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Evaluate a conversation and return a score.
+
+        Args:
+            conversation: A list of conversation messages. Each message is a dictionary
+                with 'role' (either 'user' or 'assistant') and 'content' (the message text).
+            **kwargs: Additional keyword arguments that may be used by specific metric implementations.
+
+        Returns:
+            A ScoreResult object or list of ScoreResult objects containing the evaluation score,
+            metric name, and optional reasoning.
+        """
+        raise NotImplementedError(
+            "Please use concrete metric classes instead of this one."
+        )
+
+    async def ascore(
+        self, conversation: types.Conversation, **kwargs: Any
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Asynchronously evaluate a conversation and return a score.
+
+        This is the async version of the score method. By default, it calls the
+        synchronous score method, but can be overridden for true async implementations.
+
+        Args:
+            conversation: A list of conversation messages. Each message is a dictionary
+                with 'role' (either 'user' or 'assistant') and 'content' (the message text).
+            **kwargs: Additional keyword arguments that may be used by specific metric implementations.
+
+        Returns:
+            A ScoreResult object or list of ScoreResult objects containing the evaluation score,
+            metric name, and optional reasoning.
+        """
+        raise NotImplementedError(
+            "Please use concrete metric classes instead of this one."
+        )

opik/evaluation/metrics/conversation/conversation_turns_factory.py

@@ -0,0 +1,39 @@
+from typing import List
+
+from . import types
+
+
+def build_conversation_turns(
+    conversation: types.Conversation,
+) -> List[types.ConversationTurn]:
+    """
+    Builds a list of conversation turns from a given conversation object by grouping
+    user and assistant message pairs together. Each turn is represented by a
+    `ConversationTurn` object containing a single user's input and the assistant's
+    corresponding output.
+
+    Args:
+        conversation (types.Conversation): A conversation object containing a list
+            of dictionaries. Each dictionary represents a message in the conversation
+            with a "role" key indicating the sender ("user" or "assistant") and
+            message content.
+
+    Returns:
+        List[types.ConversationTurn]: A list of `ConversationTurn` objects, where
+            each object represents a pair of user input and assistant output messages.
+    """
+    turns = []
+    user_input = None
+    for message_dict in conversation:
+        if message_dict["role"] == "user":
+            user_input = message_dict
+        elif message_dict["role"] == "assistant" and user_input is not None:
+            current_turn = types.ConversationTurn(input=user_input, output=message_dict)
+            turns.append(current_turn)
+            user_input = None
+
+    # append the last user input if it exists
+    if user_input is not None:
+        turns.append(types.ConversationTurn(input=user_input, output=None))
+
+    return turns

opik/evaluation/metrics/conversation/g_eval_wrappers.py

@@ -0,0 +1,19 @@
+from .llm_judges.g_eval_wrappers import (
+    ConversationComplianceRiskMetric,
+    ConversationDialogueHelpfulnessMetric,
+    ConversationPromptUncertaintyMetric,
+    ConversationQARelevanceMetric,
+    ConversationSummarizationCoherenceMetric,
+    ConversationSummarizationConsistencyMetric,
+    GEvalConversationMetric,
+)
+
+__all__ = [
+    "GEvalConversationMetric",
+    "ConversationComplianceRiskMetric",
+    "ConversationDialogueHelpfulnessMetric",
+    "ConversationPromptUncertaintyMetric",
+    "ConversationQARelevanceMetric",
+    "ConversationSummarizationCoherenceMetric",
+    "ConversationSummarizationConsistencyMetric",
+]