opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/engine/evaluation_tasks_executor.py

@@ -1,22 +1,27 @@
 from concurrent import futures
-from typing import List
+from typing import List, TypeVar
 
-import tqdm
-
-from .. import test_result
+from ...environment import get_tqdm_for_current_environment
 from .types import EvaluationTask
 
+_tqdm = get_tqdm_for_current_environment()
+
+T = TypeVar("T")
+
 
 def execute(
-    evaluation_tasks: List[EvaluationTask], workers: int, verbose: int
-) -> List[test_result.TestResult]:
+    evaluation_tasks: List[EvaluationTask[T]],
+    workers: int,
+    verbose: int,
+    desc: str = "Evaluation",
+) -> List[T]:
     if workers == 1:
         test_results = [
             evaluation_task()
-            for evaluation_task in tqdm.tqdm(
+            for evaluation_task in _tqdm(
                 evaluation_tasks,
                 disable=(verbose < 1),
-                desc="Evaluation",
+                desc=desc,
                 total=len(evaluation_tasks),
             )
         ]
@@ -30,12 +35,12 @@ def execute(
 
         test_results = [
             test_result_future.result()
-            for test_result_future in tqdm.tqdm(
+            for test_result_future in _tqdm(
                 futures.as_completed(
                     test_result_futures,
                 ),
                 disable=(verbose < 1),
-                desc="Evaluation",
+                desc=desc,
                 total=len(test_result_futures),
             )
         ]

opik/evaluation/engine/helpers.py

@@ -6,18 +6,18 @@ from opik.api_objects.experiment import experiment_item
 from opik.decorator import error_info_collector
 from opik.types import ErrorInfoDict
 
-from opik import context_storage
+import opik.context_storage as context_storage
 
 
 @contextlib.contextmanager
 def evaluate_llm_task_context(
-    experiment: experiment.Experiment,
+    experiment: Optional[experiment.Experiment],
     dataset_item_id: str,
     trace_data: trace.TraceData,
     client: opik_client.Opik,
 ) -> Iterator[None]:
+    error_info: Optional[ErrorInfoDict] = None
     try:
-        error_info: Optional[ErrorInfoDict] = None
         context_storage.set_trace_data(trace_data)
         yield
     except Exception as exception:
@@ -34,11 +34,36 @@ def evaluate_llm_task_context(
         trace_data.init_end_time()
 
         client = client if client is not None else opik_client.get_client_cached()
-        client.trace(**trace_data.__dict__)
+        client.trace(**trace_data.as_parameters)
 
-        experiment_item_ = experiment_item.ExperimentItemReferences(
-            dataset_item_id=dataset_item_id,
-            trace_id=trace_data.id,
-        )
+        # Only insert experiment item if an experiment is provided
+        if experiment is not None:
+            experiment_item_ = experiment_item.ExperimentItemReferences(
+                dataset_item_id=dataset_item_id,
+                trace_id=trace_data.id,
+            )
+            experiment.insert(experiment_items_references=[experiment_item_])
 
-        experiment.insert(experiment_items_references=[experiment_item_])
+
+@contextlib.contextmanager
+def evaluate_llm_task_result_spans_context(
+    trace_data: trace.TraceData,
+    client: opik_client.Opik,
+) -> Iterator[None]:
+    error_info: Optional[ErrorInfoDict] = None
+    try:
+        context_storage.set_trace_data(trace_data)
+        yield
+    except Exception as exception:
+        error_info = error_info_collector.collect(exception)
+        raise
+    finally:
+        trace_data = context_storage.pop_trace_data()  # type: ignore
+
+        assert trace_data is not None
+
+        if error_info is not None:
+            trace_data.error_info = error_info
+
+        trace_data.init_end_time()
+        client.trace(**trace_data.as_parameters)

opik/evaluation/engine/metrics_evaluator.py

@@ -0,0 +1,237 @@
+import inspect
+import logging
+from typing import List, Dict, Any, Optional, Callable, Tuple
+
+import opik.exceptions as exceptions
+import opik.logging_messages as logging_messages
+from opik.evaluation.metrics import (
+    arguments_helpers,
+    base_metric,
+    score_result,
+    arguments_validator,
+)
+from opik.evaluation.scorers import scorer_wrapper_metric
+from opik.evaluation.types import ScoringKeyMappingType
+from opik.message_processing.emulation import models
+
+from . import exception_analyzer
+
+
+LOGGER = logging.getLogger(__name__)
+
+EVALUATION_SPAN_PARAMETER_NAME = "task_span"
+
+
+def _has_evaluation_span_parameter(func: Callable) -> bool:
+    """Check if a scoring function expects the task_span parameter."""
+    try:
+        sig = inspect.signature(func)
+        return EVALUATION_SPAN_PARAMETER_NAME in sig.parameters
+    except (ValueError, TypeError):
+        return False
+
+
+def _compute_metric_scores(
+    scoring_metrics: List[base_metric.BaseMetric],
+    mapped_scoring_inputs: Dict[str, Any],
+    scoring_key_mapping: Optional[ScoringKeyMappingType],
+    dataset_item_content: Dict[str, Any],
+    task_output: Dict[str, Any],
+) -> List[score_result.ScoreResult]:
+    """
+    Compute scores using given metrics.
+
+    Args:
+        scoring_metrics: List of metrics to compute
+        mapped_scoring_inputs: Scoring inputs after key mapping (will be used for regular metrics)
+        scoring_key_mapping: Optional mapping for renaming score arguments
+        dataset_item_content: Dataset item content (will be used for ScorerWrapperMetric)
+        task_output: Task output (will be used for ScorerWrapperMetric)
+
+    Returns:
+        List of computed score results
+    """
+    score_results: List[score_result.ScoreResult] = []
+
+    for metric in scoring_metrics:
+        try:
+            LOGGER.debug("Metric %s score started", metric.name)
+
+            if isinstance(metric, scorer_wrapper_metric.ScorerWrapperMetric):
+                # ScorerWrapperMetric uses original dataset item and task output without mappings
+                if (
+                    task_span := mapped_scoring_inputs.get(
+                        EVALUATION_SPAN_PARAMETER_NAME
+                    )
+                ) is not None:
+                    result = metric.score(
+                        dataset_item=dataset_item_content,
+                        task_outputs=task_output,
+                        task_span=task_span,
+                    )
+                else:
+                    result = metric.score(
+                        dataset_item=dataset_item_content,
+                        task_outputs=task_output,
+                    )
+            else:
+                arguments_validator.validate_score_arguments(
+                    metric=metric,
+                    kwargs=mapped_scoring_inputs,
+                    scoring_key_mapping=scoring_key_mapping,
+                )
+                result = metric.score(**mapped_scoring_inputs)
+
+            LOGGER.debug("Metric %s score ended", metric.name)
+
+            if isinstance(result, list):
+                score_results += result
+            else:
+                score_results.append(result)
+
+        except exceptions.ScoreMethodMissingArguments:
+            raise
+        except Exception as exception:
+            LOGGER.error(
+                "Failed to compute metric %s. Score result will be marked as failed.",
+                metric.name,
+                exc_info=True,
+            )
+
+            if exception_analyzer.is_llm_provider_rate_limit_error(exception):
+                LOGGER.error(
+                    logging_messages.LLM_PROVIDER_RATE_LIMIT_ERROR_DETECTED_IN_EVALUATE_FUNCTION
+                )
+
+            score_results.append(
+                score_result.ScoreResult(
+                    name=metric.name,
+                    value=0.0,
+                    reason=str(exception),
+                    scoring_failed=True,
+                )
+            )
+
+    return score_results
+
+
+class MetricsEvaluator:
+    """
+    Handles metric computation and scoring.
+
+    Separates metrics into:
+    - Regular metrics: Score based on inputs/outputs
+    - Task span metrics: Score based on LLM call metadata (tokens, latency, etc)
+    """
+
+    def __init__(
+        self,
+        scoring_metrics: List[base_metric.BaseMetric],
+        scoring_key_mapping: Optional[ScoringKeyMappingType],
+    ):
+        self._scoring_key_mapping = scoring_key_mapping
+        self._regular_metrics: List[base_metric.BaseMetric] = []
+        self._task_span_metrics: List[base_metric.BaseMetric] = []
+
+        self._analyze_metrics(scoring_metrics)
+
+    @property
+    def has_task_span_metrics(self) -> bool:
+        """Check if any task span scoring metrics are configured."""
+        return len(self._task_span_metrics) > 0
+
+    @property
+    def task_span_metrics(self) -> List[base_metric.BaseMetric]:
+        """Get list of task span scoring metrics."""
+        return self._task_span_metrics
+
+    @property
+    def regular_metrics(self) -> List[base_metric.BaseMetric]:
+        """Get list of regular scoring metrics."""
+        return self._regular_metrics
+
+    def _analyze_metrics(
+        self,
+        scoring_metrics: List[base_metric.BaseMetric],
+    ) -> None:
+        """Separate metrics into regular and task-span categories."""
+        for metric in scoring_metrics:
+            if _has_evaluation_span_parameter(metric.score):
+                self._task_span_metrics.append(metric)
+            else:
+                self._regular_metrics.append(metric)
+
+        if self.has_task_span_metrics:
+            LOGGER.debug(
+                "Detected %d LLM task span scoring metrics.",
+                len(self._task_span_metrics),
+            )
+
+    def compute_regular_scores(
+        self,
+        dataset_item_content: Dict[str, Any],
+        task_output: Dict[str, Any],
+    ) -> Tuple[List[score_result.ScoreResult], Dict[str, Any]]:
+        """
+        Compute scores using regular metrics.
+
+        Args:
+            dataset_item_content: Dataset item content
+            task_output: Task output
+
+        Returns:
+            Tuple of (score results, mapped scoring inputs used for scoring regular non-wrapper metrics)
+        """
+        mapped_scoring_inputs = arguments_helpers.create_scoring_inputs(
+            dataset_item=dataset_item_content,
+            task_output=task_output,
+            scoring_key_mapping=self._scoring_key_mapping,
+        )
+
+        score_results = _compute_metric_scores(
+            scoring_metrics=self._regular_metrics,
+            mapped_scoring_inputs=mapped_scoring_inputs,
+            scoring_key_mapping=self._scoring_key_mapping,
+            dataset_item_content=dataset_item_content,
+            task_output=task_output,
+        )
+
+        return score_results, mapped_scoring_inputs
+
+    def compute_task_span_scores(
+        self,
+        dataset_item_content: Dict[str, Any],
+        task_output: Dict[str, Any],
+        task_span: models.SpanModel,
+    ) -> Tuple[List[score_result.ScoreResult], Dict[str, Any]]:
+        """
+        Compute scores using task span metrics.
+
+        Args:
+            dataset_item_content: Dataset item content
+            task_output: Task output
+            task_span: Span model containing task execution metadata
+
+        Returns:
+            Tuple of (score results, mapped scoring inputs used for scoring regular non-wrapper metrics)
+        """
+        mapped_scoring_inputs = arguments_helpers.create_scoring_inputs(
+            dataset_item=dataset_item_content,
+            task_output=task_output,
+            scoring_key_mapping=self._scoring_key_mapping,
+        )
+
+        mapped_scoring_inputs_with_span = {
+            **mapped_scoring_inputs,
+            EVALUATION_SPAN_PARAMETER_NAME: task_span,
+        }
+
+        score_results = _compute_metric_scores(
+            scoring_metrics=self._task_span_metrics,
+            mapped_scoring_inputs=mapped_scoring_inputs_with_span,
+            scoring_key_mapping=self._scoring_key_mapping,
+            dataset_item_content=dataset_item_content,
+            task_output=task_output,
+        )
+
+        return score_results, mapped_scoring_inputs_with_span

opik/evaluation/engine/types.py

@@ -1,7 +1,8 @@
-from .. import test_result
-from typing import Protocol
+from typing import Protocol, TypeVar
 
+T = TypeVar("T", covariant=True)
 
-class EvaluationTask(Protocol):
-    def __call__(self) -> test_result.TestResult:
+
+class EvaluationTask(Protocol[T]):
+    def __call__(self) -> T:
         pass

opik/evaluation/evaluation_result.py

@@ -1,12 +1,179 @@
-from typing import List, Optional
+from typing import List, Optional, Dict, TYPE_CHECKING
+from collections import defaultdict
+import logging
 
 import dataclasses
 
-from . import test_result
+from . import score_statistics, test_result
+from .metrics import score_result
+
+if TYPE_CHECKING:
+    pass
+
+LOGGER = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class DatasetItemResults:
+    """Results for a single dataset item across all trials."""
+
+    test_results: List[test_result.TestResult]
+    scores: Dict[str, score_statistics.ScoreStatistics]
+
+
+@dataclasses.dataclass
+class EvaluationResultGroupByDatasetItemsView:
+    """View of evaluation results grouped by dataset items."""
+
+    experiment_id: str
+    dataset_id: str
+    experiment_name: Optional[str]
+    experiment_url: Optional[str]
+    trial_count: int
+    dataset_items: Dict[str, DatasetItemResults]
+
+
+@dataclasses.dataclass
+class EvaluationResultAggregatedScoresView:
+    """
+    Represents an aggregated view of evaluation results and scores for an
+    experiment.
+
+    This class is designed to encapsulate information about an experiment,
+    its related dataset, trial counts, test results, and aggregated score
+    statistics. It serves as a comprehensive representation useful in
+    evaluation pipelines or result summaries.
+
+    Attributes:
+        experiment_id: Unique identifier for the experiment.
+        dataset_id: Unique identifier for the associated dataset.
+        experiment_name: Human-readable name of the experiment.
+        experiment_url: URL link to the experiment for easy access.
+        trial_count: Number of trials conducted in the experiment.
+        test_results: Collection of test results from the experiment.
+        aggregated_scores: Aggregated statistical scores for evaluation
+            results keyed by score name.
+    """
+
+    experiment_id: str
+    dataset_id: str
+    experiment_name: Optional[str]
+    experiment_url: Optional[str]
+    trial_count: int
+    test_results: List[test_result.TestResult]
+    aggregated_scores: Dict[str, score_statistics.ScoreStatistics]
 
 
 @dataclasses.dataclass
 class EvaluationResult:
     experiment_id: str
+    dataset_id: str
     experiment_name: Optional[str]
     test_results: List[test_result.TestResult]
+    experiment_url: Optional[str]
+    trial_count: int
+    experiment_scores: List[score_result.ScoreResult] = dataclasses.field(
+        default_factory=list
+    )
+
+    def aggregate_evaluation_scores(self) -> EvaluationResultAggregatedScoresView:
+        """
+        Aggregates evaluation scores from test results and returns the aggregated scores view.
+
+        The method calculates aggregated scores from test results and encapsulates the results
+        in an EvaluationResultAggregatedScoresView object, which contains information about
+        the experiment and computed aggregated scores.
+
+        The aggregated scores dictionary has keys for each found score name and values containing
+        the statistics for that score.
+
+        Returns:
+            EvaluationResultAggregatedScoresView object containing details about the
+            experiment and the aggregated scores calculated from test results.
+        """
+        aggregated_scores = score_statistics.calculate_aggregated_statistics(
+            self.test_results
+        )
+        return EvaluationResultAggregatedScoresView(
+            experiment_id=self.experiment_id,
+            dataset_id=self.dataset_id,
+            experiment_name=self.experiment_name,
+            experiment_url=self.experiment_url,
+            trial_count=self.trial_count,
+            test_results=self.test_results,
+            aggregated_scores=aggregated_scores,
+        )
+
+    def group_by_dataset_item_view(self) -> EvaluationResultGroupByDatasetItemsView:
+        """
+        Create a view of evaluation results grouped by dataset items.
+
+        Returns:
+            EvaluationResultGroupByDatasetItemsView containing organized results with aggregated score statistics
+        """
+        dataset_items = self._build_results_per_dataset_item()
+
+        return EvaluationResultGroupByDatasetItemsView(
+            experiment_id=self.experiment_id,
+            dataset_id=self.dataset_id,
+            experiment_name=self.experiment_name,
+            experiment_url=self.experiment_url,
+            trial_count=self.trial_count,
+            dataset_items=dataset_items,
+        )
+
+    def _build_results_per_dataset_item(self) -> Dict[str, DatasetItemResults]:
+        """
+        Build dataset item results with aggregated score statistics.
+
+        Returns:
+            Dict mapping dataset item IDs to their results and aggregated score statistics
+        """
+        if not self.test_results:
+            LOGGER.debug("No test results available for aggregation")
+            return {}
+
+        results_by_dataset_item = defaultdict(list)
+        for test_result_ in self.test_results:
+            dataset_item_id = test_result_.test_case.dataset_item_id
+            results_by_dataset_item[dataset_item_id].append(test_result_)
+
+        dataset_items_results = {}
+        for dataset_item_id, dataset_item_results in results_by_dataset_item.items():
+            dataset_item_results.sort(key=lambda x: x.trial_id)
+            aggregated_scores = score_statistics.calculate_aggregated_statistics(
+                dataset_item_results
+            )
+
+            dataset_items_results[dataset_item_id] = DatasetItemResults(
+                test_results=dataset_item_results, scores=aggregated_scores
+            )
+
+        return dataset_items_results
+
+
+@dataclasses.dataclass
+class EvaluationResultOnDictItems:
+    """
+    Evaluation result for dict items evaluation without experiment tracking.
+
+    This class provides a similar interface to EvaluationResult but is designed
+    for lightweight evaluations that don't require experiment or dataset management.
+    It can aggregate scores across test results just like the regular evaluation.
+
+    Attributes:
+        test_results: Collection of test results from the evaluation.
+    """
+
+    test_results: List[test_result.TestResult]
+
+    def aggregate_evaluation_scores(
+        self,
+    ) -> Dict[str, score_statistics.ScoreStatistics]:
+        """
+        Aggregates evaluation scores from test results.
+
+        Returns:
+            Dictionary mapping score names to their aggregated statistics.
+        """
+        return score_statistics.calculate_aggregated_statistics(self.test_results)