opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/score_statistics.py

@@ -0,0 +1,66 @@
+import dataclasses
+import math
+import statistics
+from collections import defaultdict
+from typing import List, Optional, Dict
+
+from opik.evaluation import test_result
+
+
+@dataclasses.dataclass
+class ScoreStatistics:
+    """Statistics for a single score metric across multiple trials."""
+
+    mean: float
+    max: float
+    min: float
+    values: List[float]
+    std: Optional[float] = None  # Standard deviation (None if count < 2)
+
+
+def calculate_aggregated_statistics(
+    evaluation_results: List[test_result.TestResult],
+) -> Dict[str, ScoreStatistics]:
+    """
+    Calculate mean, max, and min scores for each score name in the evaluation test results.
+
+    Args:
+        evaluation_results: List of TestResult objects to be aggregated
+
+    Returns:
+        Dict mapping score names to their aggregated statistics
+    """
+    if not evaluation_results:
+        return {}
+
+    # Group scores by name across all trials
+    scores_by_name = defaultdict(list)
+
+    for test_result_ in evaluation_results:
+        for score_result in test_result_.score_results:
+            # Only include successful scores with valid values
+            if not score_result.scoring_failed and _is_valid_score_value(
+                score_result.value
+            ):
+                scores_by_name[score_result.name].append(score_result.value)
+
+    # Calculate aggregated statistics for each score name
+    aggregated_scores = {}
+    for score_name, values in scores_by_name.items():
+        if values:
+            std = statistics.stdev(values) if len(values) >= 2 else None
+
+            aggregated_scores[score_name] = ScoreStatistics(
+                mean=statistics.mean(values),
+                max=max(values),
+                min=min(values),
+                values=values.copy(),  # Store the actual values used
+                std=std,
+            )
+
+    return aggregated_scores
+
+
+def _is_valid_score_value(value: float) -> bool:
+    """Check if a score value is valid for statistical calculations."""
+    return isinstance(value, (int, float)) and math.isfinite(value)

opik/evaluation/scorers/__init__.py

@@ -0,0 +1,4 @@
+from .scorer_function import ScorerFunction
+from .scorer_wrapper_metric import ScorerWrapperMetric
+
+__all__ = ["ScorerFunction", "ScorerWrapperMetric"]

opik/evaluation/scorers/scorer_function.py

@@ -0,0 +1,55 @@
+import inspect
+from typing import Any, Dict, Optional, Protocol, Union, List
+
+from opik.evaluation.metrics import score_result
+from opik.message_processing.emulation import models
+
+
+class ScorerFunctionProtocol(Protocol):
+    """
+    Represents a protocol defining the structure for a scorer function.
+
+    This protocol serves as a contract for implementing scorer functions used in
+    evaluating tasks. A scorer function adhering to this protocol should take
+    dataset item data, task outputs, and optionally a task span model as input
+    parameters and return a scoring result.
+    """
+
+    def __call__(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        task_span: Optional[models.SpanModel] = None,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]: ...
+
+
+ScorerFunction = ScorerFunctionProtocol
+
+
+EXPECTED_SCORER_FUNCTION_PARAMETERS = ["dataset_item", "task_outputs"]
+
+
+def validate_scorer_function(scorer_function: ScorerFunction) -> None:
+    if not callable(scorer_function):
+        raise ValueError("scorer_function must be a callable function")
+
+    parameters = inspect.signature(scorer_function).parameters
+    names = set(parameters.keys())
+
+    # Check if it has both dataset_item and task_outputs
+    has_dataset_item_and_task_outputs = all(
+        param in names for param in EXPECTED_SCORER_FUNCTION_PARAMETERS
+    )
+
+    # Check if it has at least one task_span parameter
+    has_task_span = "task_span" in names
+
+    if not (has_dataset_item_and_task_outputs or has_task_span):
+        raise ValueError(
+            f"scorer_function must have either both 'dataset_item' and 'task_outputs' parameters "
+            f"or at least one 'task_span' parameter. Found parameters: {list(names)}"
+        )
+
+
+def has_task_span_in_parameters(scorer_function: ScorerFunction) -> bool:
+    return "task_span" in inspect.signature(scorer_function).parameters

opik/evaluation/scorers/scorer_wrapper_metric.py

@@ -0,0 +1,130 @@
+from typing import Any, Callable, Dict, Optional, List, Union
+
+from opik.evaluation.metrics import base_metric, score_result
+
+from . import scorer_function
+from ...message_processing.emulation import models
+
+
+class ScorerWrapperMetric(base_metric.BaseMetric):
+    """
+    A wrapper metric that adapts a ScorerFunction to the BaseMetric interface.
+
+    This class allows using ScorerFunction instances as BaseMetric instances,
+    providing compatibility between the two interfaces.
+
+    Args:
+        scorer: The ScorerFunction to wrap
+        name: Optional name for the metric. If not provided, uses the class name.
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name for tracking.
+
+    Raises:
+        ValueError if the scorer function is invalid.
+
+    Example:
+        >>> def my_scorer(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]) -> score_result.ScoreResult:
+        >>>     return score_result.ScoreResult(name="my_metric", value=1.0)
+        >>>
+        >>> wrapper = ScorerWrapperMetric(scorer_function=my_scorer, name="wrapped_scorer")
+        >>> result = wrapper.score(dataset_item={"text": "hello"}, task_outputs={"text": "hello"})
+    """
+
+    def __init__(
+        self,
+        scorer: scorer_function.ScorerFunction,
+        name: str,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self.scorer = scorer
+
+        # validate scorer function
+        scorer_function.validate_scorer_function(scorer)
+
+    def score(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        **kwargs: Any,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Score using the wrapped ScorerFunction.
+
+        Args:
+            dataset_item: The dataset item data to score against
+            task_outputs: The output dictionary to be scored - can be the output of LLM task, etc.
+            **kwargs: Additional keyword arguments (ignored by the scorer function)
+
+        Returns:
+            ScoreResult from the wrapped scorer function
+        """
+        return self.scorer(dataset_item=dataset_item, task_outputs=task_outputs)
+
+
+class ScorerWrapperMetricTaskSpan(ScorerWrapperMetric):
+    def __init__(
+        self,
+        scorer: scorer_function.ScorerFunction,
+        name: str,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            scorer=scorer, name=name, track=track, project_name=project_name
+        )
+
+    def score(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        task_span: Optional[models.SpanModel] = None,
+        **kwargs: Any,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Score using the wrapped ScorerFunction.
+
+        Args:
+            dataset_item: The dataset item data to score against
+            task_outputs: The output dictionary to be scored - can be the output of LLM task, etc.
+            task_span: The collected task span data.
+            **kwargs: Additional keyword arguments (ignored by the scorer function)
+
+        Returns:
+            ScoreResult from the wrapped scorer function
+        """
+        if task_span is not None and scorer_function.has_task_span_in_parameters(
+            self.scorer
+        ):
+            return self.scorer(
+                dataset_item=dataset_item,
+                task_outputs=task_outputs,
+                task_span=task_span,
+            )
+
+        return self.scorer(dataset_item=dataset_item, task_outputs=task_outputs)
+
+
+def _scorer_name(scorer: Callable) -> str:
+    return scorer.__name__
+
+
+def wrap_scorer_functions(
+    scorer_functions: List[scorer_function.ScorerFunction], project_name: Optional[str]
+) -> List[base_metric.BaseMetric]:
+    metrics: List[base_metric.BaseMetric] = []
+    for f in scorer_functions:
+        name = _scorer_name(f)
+        if scorer_function.has_task_span_in_parameters(f):
+            metrics.append(
+                ScorerWrapperMetricTaskSpan(
+                    scorer=f, project_name=project_name, name=name
+                )
+            )
+        else:
+            metrics.append(
+                ScorerWrapperMetric(scorer=f, project_name=project_name, name=name)
+            )
+
+    return metrics

opik/evaluation/test_case.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 import dataclasses
 
 
@@ -6,5 +6,6 @@ import dataclasses
 class TestCase:
     trace_id: str
     dataset_item_id: str
-    scoring_inputs: Dict[str, Any]
     task_output: Dict[str, Any]
+    dataset_item_content: Dict[str, Any] = dataclasses.field(default_factory=dict)
+    mapped_scoring_inputs: Optional[Dict[str, Any]] = None

opik/evaluation/test_result.py

@@ -10,3 +10,4 @@ from .metrics import score_result
 class TestResult:
     test_case: test_case.TestCase
     score_results: List[score_result.ScoreResult]
+    trial_id: int

opik/evaluation/threads/context_helper.py

@@ -0,0 +1,32 @@
+import contextlib
+from typing import Iterator, Optional
+
+import opik.context_storage as context_storage
+from opik.api_objects import trace, opik_client
+from opik.decorator import error_info_collector
+from opik.types import ErrorInfoDict
+
+
+@contextlib.contextmanager
+def evaluate_llm_conversation_context(
+    trace_data: trace.TraceData,
+    client: opik_client.Opik,
+) -> Iterator[None]:
+    error_info: Optional[ErrorInfoDict] = None
+    try:
+        context_storage.set_trace_data(trace_data)
+        yield
+    except Exception as exception:
+        error_info = error_info_collector.collect(exception)
+        raise
+    finally:
+        trace_data = context_storage.pop_trace_data()  # type: ignore
+
+        assert trace_data is not None
+
+        if error_info is not None:
+            trace_data.error_info = error_info
+
+        trace_data.init_end_time()
+
+        client.trace(**trace_data.as_parameters)

opik/evaluation/threads/evaluation_engine.py

@@ -0,0 +1,181 @@
+import functools
+import logging
+from typing import Optional, List, Callable, Dict, Literal
+
+import opik
+import opik.exceptions as exceptions
+import opik.opik_context as opik_context
+from opik.evaluation.metrics.conversation import conversation_thread_metric
+from opik.rest_api import JsonListStringPublic, TraceThread
+
+from . import evaluation_result, helpers
+from ..engine import evaluation_tasks_executor
+from ..engine import types as engine_types
+from ..metrics import score_result
+from ...api_objects import trace
+from ...api_objects.threads import threads_client
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ThreadsEvaluationEngine:
+    def __init__(
+        self,
+        client: threads_client.ThreadsClient,
+        project_name: Optional[str],
+        number_of_workers: int,
+        verbose: int,
+    ) -> None:
+        self._client = client
+        self._project_name = project_name
+        self._number_of_workers = number_of_workers
+        self._verbose = verbose
+
+        self._threads_client = client
+
+    def evaluate_threads(
+        self,
+        filter_string: Optional[str],
+        eval_project_name: Optional[str],
+        metrics: List[conversation_thread_metric.ConversationThreadMetric],
+        trace_input_transform: Callable[[JsonListStringPublic], str],
+        trace_output_transform: Callable[[JsonListStringPublic], str],
+        max_traces_per_thread: int = 1000,
+    ) -> evaluation_result.ThreadsEvaluationResult:
+        if len(metrics) == 0:
+            raise ValueError("No metrics provided")
+
+        threads = self._threads_client.search_threads(
+            project_name=self._project_name,
+            filter_string=filter_string,
+        )
+        if len(threads) == 0:
+            raise exceptions.EvaluationError(
+                f"No threads found with filter_string: {filter_string}"
+            )
+
+        inactive_threads = [thread for thread in threads if thread.status == "inactive"]
+        if len(inactive_threads) == 0:
+            raise exceptions.EvaluationError(
+                f"No closed threads found with filter_string: {filter_string}. Only closed threads can be evaluated."
+            )
+        elif len(inactive_threads) < len(threads):
+            active_threads_ids = [
+                thread.id for thread in threads if thread.status == "active"
+            ]
+            inactive_threads_ids = [thread.id for thread in inactive_threads]
+            LOGGER.warning(
+                f"Some threads are active: {active_threads_ids} with filter_string: {filter_string}. Only closed threads will be evaluated: {inactive_threads_ids}."
+            )
+
+        evaluation_tasks: List[
+            engine_types.EvaluationTask[evaluation_result.ThreadEvaluationResult]
+        ] = [
+            functools.partial(
+                self.evaluate_thread,
+                thread=thread,
+                eval_project_name=eval_project_name,
+                metrics=metrics,
+                trace_input_transform=trace_input_transform,
+                trace_output_transform=trace_output_transform,
+                max_traces_per_thread=max_traces_per_thread,
+            )
+            for thread in inactive_threads
+        ]
+
+        results = evaluation_tasks_executor.execute(
+            evaluation_tasks, workers=self._number_of_workers, verbose=self._verbose
+        )
+
+        helpers.log_feedback_scores(
+            results, project_name=self._project_name, client=self._threads_client
+        )
+
+        return evaluation_result.ThreadsEvaluationResult(results=results)
+
+    def evaluate_thread(
+        self,
+        thread: TraceThread,
+        eval_project_name: Optional[str],
+        metrics: List[conversation_thread_metric.ConversationThreadMetric],
+        trace_input_transform: Callable[[JsonListStringPublic], str],
+        trace_output_transform: Callable[[JsonListStringPublic], str],
+        max_traces_per_thread: int,
+    ) -> evaluation_result.ThreadEvaluationResult:
+        conversation_dict = helpers.load_conversation_thread(
+            thread=thread,
+            trace_input_transform=trace_input_transform,
+            trace_output_transform=trace_output_transform,
+            max_results=max_traces_per_thread,
+            project_name=self._project_name,
+            client=self._client.opik_client,
+        ).model_dump()
+
+        conversation = conversation_dict["discussion"]
+        if len(conversation) == 0:
+            LOGGER.warning(
+                f"Thread '{thread.id}' has no conversation traces. Skipping evaluation."
+            )
+            return evaluation_result.ThreadEvaluationResult(
+                thread_id=thread.id, scores=[]
+            )
+
+        if eval_project_name is None:
+            eval_project_name = self._project_name
+
+        # Create a new trace for the evaluation
+        trace_data = trace.TraceData(
+            input={"conversation": conversation, "metrics": metrics},
+            name="evaluation_task",
+            created_by="evaluation",
+            project_name=eval_project_name,
+        )
+
+        with opik_context.trace_context(
+            trace_data=trace_data,
+            client=self._client.opik_client,
+        ):
+            results = self._evaluate_conversation(conversation, metrics)
+
+            # Update the current trace with the evaluation results
+            outputs = [result.__dict__ for result in results]
+            opik_context.update_current_trace(output={"evaluation_results": outputs})
+
+        return evaluation_result.ThreadEvaluationResult(
+            thread_id=thread.id,
+            scores=results,
+        )
+
+    @opik.track(name="metrics_calculation")  # type: ignore[attr-defined,has-type]
+    def _evaluate_conversation(
+        self,
+        conversation: List[Dict[Literal["role", "content"], str]],
+        metrics: List[conversation_thread_metric.ConversationThreadMetric],
+    ) -> List[score_result.ScoreResult]:
+        score_results: List[score_result.ScoreResult] = []
+        for metric in metrics:
+            try:
+                LOGGER.debug("Metric %s score started", metric.name)
+                result = metric.score(conversation)
+                LOGGER.debug("Metric %s score ended", metric.name)
+
+                if isinstance(result, list):
+                    score_results.extend(result)
+                else:
+                    score_results.append(result)
+            except Exception as e:
+                LOGGER.error(
+                    "Failed to compute metric %s. Score result will be marked as failed.",
+                    metric.name,
+                    exc_info=True,
+                )
+                score_results.append(
+                    score_result.ScoreResult(
+                        name=metric.name,
+                        value=0.0,
+                        reason=str(e),
+                        scoring_failed=True,
+                    )
+                )
+
+        return score_results

opik/evaluation/threads/evaluation_result.py

@@ -0,0 +1,18 @@
+from typing import List
+
+import pydantic
+
+from ..metrics import score_result
+
+
+class ThreadEvaluationResult(pydantic.BaseModel):
+    """Evaluation result for a particular thread."""
+
+    thread_id: str
+    scores: List[score_result.ScoreResult] = pydantic.Field(default_factory=list)
+
+
+class ThreadsEvaluationResult(pydantic.BaseModel):
+    """Threads evaluation results"""
+
+    results: List[ThreadEvaluationResult] = pydantic.Field(default_factory=list)

opik/evaluation/threads/evaluator.py

@@ -0,0 +1,120 @@
+from typing import Optional, List, Callable
+
+from .. import asyncio_support
+from ...api_objects import opik_client
+from ...api_objects.threads import threads_client
+from ..metrics.conversation import conversation_thread_metric
+from . import evaluation_engine, evaluation_result
+from opik.rest_api import JsonListStringPublic
+
+
+def evaluate_threads(
+    project_name: str,
+    filter_string: Optional[str],
+    eval_project_name: Optional[str],
+    metrics: List[conversation_thread_metric.ConversationThreadMetric],
+    trace_input_transform: Callable[[JsonListStringPublic], str],
+    trace_output_transform: Callable[[JsonListStringPublic], str],
+    verbose: int = 1,
+    num_workers: int = 8,
+    max_traces_per_thread: int = 1000,
+) -> evaluation_result.ThreadsEvaluationResult:
+    """Evaluate conversation threads using specified metrics.
+
+    This function evaluates conversation threads from a project using the provided metrics.
+    It creates a ThreadsEvaluationEngine to fetch threads matching the filter string,
+    converts them to conversation threads, applies the metrics, and logs feedback scores.
+
+    Args:
+        project_name: The name of the project containing the threads to evaluate.
+        filter_string: Optional filter string to select specific threads for evaluation using Opik Query Language (OQL).
+            The format is: "<COLUMN> <OPERATOR> <VALUE> [AND <COLUMN> <OPERATOR> <VALUE>]*"
+
+            Supported columns include:
+            - `id`, `name`, `created_by`, `thread_id`, `type`, `model`, `provider`: String fields with full operator support
+            - `status`: String field (=, contains, not_contains only)
+            - `start_time`, `end_time`: DateTime fields (use ISO 8601 format, e.g., "2024-01-01T00:00:00Z")
+            - `input`, `output`: String fields for content (=, contains, not_contains only)
+            - `metadata`: Dictionary field (use dot notation, e.g., "metadata.model")
+            - `feedback_scores`: Numeric field (use dot notation, e.g., "feedback_scores.accuracy")
+            - `tags`: List field (use "contains" operator only)
+            - `usage.total_tokens`, `usage.prompt_tokens`, `usage.completion_tokens`: Numeric usage fields
+            - `duration`, `number_of_messages`, `total_estimated_cost`: Numeric fields
+
+            Examples: 'status = "inactive"', 'id = "thread_123"', 'duration > 300'
+            If None, all threads in the project will be evaluated.
+        eval_project_name: Optional name for the evaluation project where evaluation traces will be stored.
+            If None, the same project_name will be used.
+        metrics: List of ConversationThreadMetric instances to apply to each thread.
+            Must contain at least one metric.
+        trace_input_transform: Function to transform trace input JSON to string representation.
+            This function extracts the relevant user message from your trace's input structure.
+            The function receives the raw trace input as a dictionary and should return a string.
+
+            Example: If your trace input is {"content": {"user_question": "Hello"}},
+            use: lambda x: x["content"]["user_question"]
+
+            This transformation is essential because trace inputs vary by framework, but metrics
+            expect a standardized string format representing the user's message.
+        trace_output_transform: Function to transform trace output JSON to string representation.
+            This function extracts the relevant agent response from your trace's output structure.
+            The function receives the raw trace output as a dictionary and should return a string.
+
+            Example: If your trace output is {"response": {"text": "Hi there"}},
+            use: lambda x: x["response"]["text"]
+
+            This transformation is essential because trace outputs vary by framework, but metrics
+            expect a standardized string format representing the agent's response.
+        verbose: Verbosity level for progress reporting (0=silent, 1=progress).
+            Default is 1.
+        num_workers: Number of concurrent workers for thread evaluation.
+            Default is 8.
+        max_traces_per_thread: Maximum number of traces to fetch per thread.
+            Default is 1000.
+
+    Returns:
+        ThreadsEvaluationResult containing evaluation scores for each thread.
+
+    Raises:
+        ValueError: If no metrics are provided.
+        MetricComputationError: If no threads are found or if evaluation fails.
+
+    Example:
+        >>> from opik.evaluation import evaluate_threads
+        >>> from opik.evaluation.metrics import ConversationalCoherenceMetric, UserFrustrationMetric
+        >>>
+        >>> # Initialize the evaluation metrics
+        >>> conversation_coherence_metric = ConversationalCoherenceMetric()
+        >>> user_frustration_metric = UserFrustrationMetric()
+        >>>
+        >>> # Run the threads evaluation
+        >>> results = evaluate_threads(
+        >>>     project_name="ai_team",
+        >>>     filter_string='thread_id = "0197ad2a-cf5c-75af-be8b-20e8a23304fe"',
+        >>>     eval_project_name="ai_team_evaluation",
+        >>>     metrics=[
+        >>>         conversation_coherence_metric,
+        >>>         user_frustration_metric,
+        >>>     ],
+        >>>     trace_input_transform=lambda x: x["input"],
+        >>>     trace_output_transform=lambda x: x["output"],
+        >>> )
+    """
+    client = opik_client.get_client_cached()
+    threads_client_ = threads_client.ThreadsClient(client)
+
+    with asyncio_support.async_http_connections_expire_immediately():
+        engine = evaluation_engine.ThreadsEvaluationEngine(
+            client=threads_client_,
+            project_name=project_name,
+            number_of_workers=num_workers,
+            verbose=verbose,
+        )
+        return engine.evaluate_threads(
+            filter_string=filter_string,
+            eval_project_name=eval_project_name,
+            metrics=metrics,
+            trace_input_transform=trace_input_transform,
+            trace_output_transform=trace_output_transform,
+            max_traces_per_thread=max_traces_per_thread,
+        )

opik/evaluation/threads/helpers.py

@@ -0,0 +1,51 @@
+from typing import List, Callable, Optional
+
+from . import evaluation_result
+from ...api_objects import opik_client
+from ...api_objects.conversation import conversation_thread, conversation_factory
+from ...rest_api import TraceThread, JsonListStringPublic
+from ...types import BatchFeedbackScoreDict
+from ...api_objects.threads import threads_client
+
+
+def log_feedback_scores(
+    results: List[evaluation_result.ThreadEvaluationResult],
+    project_name: Optional[str],
+    client: threads_client.ThreadsClient,
+) -> None:
+    for result in results:
+        feedback_scores = [
+            BatchFeedbackScoreDict(
+                id=result.thread_id,
+                name=score.name,
+                value=score.value,
+                reason=score.reason,
+            )
+            for score in result.scores
+            if not score.scoring_failed
+        ]
+        client.log_threads_feedback_scores(
+            scores=feedback_scores,
+            project_name=project_name,
+        )
+
+
+def load_conversation_thread(
+    thread: TraceThread,
+    trace_input_transform: Callable[[JsonListStringPublic], str],
+    trace_output_transform: Callable[[JsonListStringPublic], str],
+    max_results: int,
+    project_name: Optional[str],
+    client: opik_client.Opik,
+) -> conversation_thread.ConversationThread:
+    traces = client.search_traces(
+        project_name=project_name,
+        filter_string=f'thread_id = "{thread.id}"',
+        max_results=max_results,
+        truncate=False,
+    )
+    return conversation_factory.create_conversation_from_traces(
+        traces=traces,
+        input_transform=trace_input_transform,
+        output_transform=trace_output_transform,
+    )