opik 1.9.26__py3-none-any.whl → 1.9.39__py3-none-any.whl

opik/decorator/base_track_decorator.py

@@ -14,7 +14,7 @@ from typing import (
     NamedTuple,
 )
 
-from .. import context_storage, logging_messages
+from .. import context_storage, logging_messages, tracing_runtime_config
 from ..api_objects import opik_client, span, trace
 from ..types import DistributedTraceHeadersDict, ErrorInfoDict, SpanType
 from . import (
@@ -24,7 +24,6 @@ from . import (
     inspect_helpers,
     opik_args,
     span_creation_handler,
-    tracing_runtime_config,
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -337,25 +336,24 @@ class BaseTrackDecorator(abc.ABC):
                 )
                 error_info = error_info_collector.collect(exception)
                 func_exception = exception
-            finally:
-                stream_or_stream_manager = self._streams_handler(
-                    result,
-                    track_options.capture_output,
-                    track_options.generations_aggregator,
-                )
-                if stream_or_stream_manager is not None:
-                    return stream_or_stream_manager
-
-                self._after_call(
-                    output=result,
-                    error_info=error_info,
-                    capture_output=track_options.capture_output,
-                    flush=track_options.flush,
-                )
-                if func_exception is not None:
-                    raise func_exception
-                else:
-                    return result
+
+            stream_or_stream_manager = self._streams_handler(
+                result,
+                track_options.capture_output,
+                track_options.generations_aggregator,
+            )
+            if stream_or_stream_manager is not None:
+                return stream_or_stream_manager
+
+            self._after_call(
+                output=result,
+                error_info=error_info,
+                capture_output=track_options.capture_output,
+                flush=track_options.flush,
+            )
+            if func_exception is not None:
+                raise func_exception
+            return result
 
         wrapper.opik_tracked = True  # type: ignore
 
@@ -391,25 +389,24 @@ class BaseTrackDecorator(abc.ABC):
                 )
                 error_info = error_info_collector.collect(exception)
                 func_exception = exception
-            finally:
-                stream_or_stream_manager = self._streams_handler(
-                    result,
-                    track_options.capture_output,
-                    track_options.generations_aggregator,
-                )
-                if stream_or_stream_manager is not None:
-                    return stream_or_stream_manager
-
-                self._after_call(
-                    output=result,
-                    error_info=error_info,
-                    capture_output=track_options.capture_output,
-                    flush=track_options.flush,
-                )
-                if func_exception is not None:
-                    raise func_exception
-                else:
-                    return result
+
+            stream_or_stream_manager = self._streams_handler(
+                result,
+                track_options.capture_output,
+                track_options.generations_aggregator,
+            )
+            if stream_or_stream_manager is not None:
+                return stream_or_stream_manager
+
+            self._after_call(
+                output=result,
+                error_info=error_info,
+                capture_output=track_options.capture_output,
+                flush=track_options.flush,
+            )
+            if func_exception is not None:
+                raise func_exception
+            return result
 
         wrapper.opik_tracked = True  # type: ignore
         return wrapper

opik/evaluation/__init__.py

@@ -1,4 +1,15 @@
-from .evaluator import evaluate, evaluate_prompt, evaluate_experiment
+from .evaluator import (
+    evaluate,
+    evaluate_prompt,
+    evaluate_experiment,
+    evaluate_on_dict_items,
+)
 from .threads.evaluator import evaluate_threads
 
-__all__ = ["evaluate", "evaluate_prompt", "evaluate_experiment", "evaluate_threads"]
+__all__ = [
+    "evaluate",
+    "evaluate_prompt",
+    "evaluate_experiment",
+    "evaluate_on_dict_items",
+    "evaluate_threads",
+]

opik/evaluation/engine/engine.py

@@ -1,13 +1,11 @@
 import functools
-import inspect
 import logging
-from typing import List, Optional, Callable, Any, Dict
+from typing import List, Optional, Any, Dict
 
-import opik.exceptions as exceptions
 import opik.logging_messages as logging_messages
 import opik.opik_context as opik_context
 import opik
-from opik.api_objects import opik_client, trace
+from opik.api_objects import opik_client, trace, local_recording
 from opik.api_objects.dataset import dataset, dataset_item
 from opik.api_objects.experiment import experiment
 from opik.evaluation import (
@@ -18,18 +16,15 @@ from opik.evaluation import (
 )
 from opik.evaluation.types import LLMTask, ScoringKeyMappingType
 
-from . import evaluation_tasks_executor, exception_analyzer, helpers
+from . import evaluation_tasks_executor, exception_analyzer, helpers, metrics_evaluator
 from .types import EvaluationTask
-from ..metrics import arguments_validator, arguments_helpers, base_metric, score_result
-from ..scorers import scorer_wrapper_metric
-from ...message_processing import message_processors_chain
+from ..metrics import base_metric, score_result
 from ...message_processing.emulation import models
 
 
 LOGGER = logging.getLogger(__name__)
 
 EVALUATION_TASK_NAME = "evaluation_task"
-EVALUATION_SPAN_PARAMETER_NAME = "task_span"
 
 
 class EvaluationEngine:
@@ -37,7 +32,6 @@ class EvaluationEngine:
         self,
         client: opik_client.Opik,
         project_name: Optional[str],
-        experiment_: experiment.Experiment,
         scoring_metrics: List[base_metric.BaseMetric],
         workers: int,
         verbose: int,
@@ -45,41 +39,28 @@ class EvaluationEngine:
     ) -> None:
         self._client = client
         self._project_name = project_name
-        self._experiment = experiment_
         self._workers = workers
         self._verbose = verbose
-        self._scoring_metrics: List[base_metric.BaseMetric] = []
-        self._task_span_scoring_metrics: List[base_metric.BaseMetric] = []
-        self._scoring_key_mapping = scoring_key_mapping
 
-        # Analyze metrics
-        self._analyze_metrics(scoring_metrics)
-
-        if len(self._task_span_scoring_metrics) > 0:
-            LOGGER.info(
-                "Detected %d LLM task span scoring metrics — enabling handling of the LLM task evaluation span.",
-                len(self._task_span_scoring_metrics),
-            )
-
-    def _analyze_metrics(self, scoring_metrics: List[base_metric.BaseMetric]) -> None:
-        for metric in scoring_metrics:
-            if _has_evaluation_span_parameter(metric.score):
-                self._task_span_scoring_metrics.append(metric)
-            else:
-                self._scoring_metrics.append(metric)
+        # Delegate metric analysis to MetricsEvaluator
+        self._metrics_evaluator = metrics_evaluator.MetricsEvaluator(
+            scoring_metrics=scoring_metrics,
+            scoring_key_mapping=scoring_key_mapping,
+        )
 
     @opik.track(name="metrics_calculation")  # type: ignore[attr-defined,has-type]
-    def _evaluate_test_case(
+    def _compute_test_result_for_test_case(
         self,
         test_case_: test_case.TestCase,
         trial_id: int = 0,
     ) -> test_result.TestResult:
-        score_results = _scores_by_metrics(
-            scoring_metrics=self._scoring_metrics,
-            score_kwargs=test_case_.scoring_inputs,
-            scoring_key_mapping=self._scoring_key_mapping,
-            test_case_=test_case_,
+        score_results, mapped_scoring_inputs = (
+            self._metrics_evaluator.compute_regular_scores(
+                dataset_item_content=test_case_.dataset_item_content,
+                task_output=test_case_.task_output,
+            )
         )
+        test_case_.mapped_scoring_inputs = mapped_scoring_inputs
 
         test_result_ = test_result.TestResult(
             test_case=test_case_,
@@ -94,11 +75,40 @@ class EvaluationEngine:
         )
         return test_result_
 
-    def _evaluate_llm_task(
+    @opik.track(  # type: ignore[attr-defined,has-type]
+        name="task_span_metrics_calculation",
+        ignore_arguments=["test_case_"],
+    )
+    def _compute_scores_for_test_case_with_task_span(
+        self,
+        trace_id: str,
+        task_span: models.SpanModel,
+        test_case_: test_case.TestCase,
+    ) -> List[score_result.ScoreResult]:
+        score_results, mapped_scoring_inputs = (
+            self._metrics_evaluator.compute_task_span_scores(
+                dataset_item_content=test_case_.dataset_item_content,
+                task_output=test_case_.task_output,
+                task_span=task_span,
+            )
+        )
+        test_case_.mapped_scoring_inputs = mapped_scoring_inputs
+
+        # log feedback scores
+        rest_operations.log_test_result_feedback_scores(
+            client=self._client,
+            score_results=score_results,
+            trace_id=trace_id,
+            project_name=self._project_name,
+        )
+        return score_results
+
+    def _compute_test_result_for_llm_task(
         self,
         item: dataset_item.DatasetItem,
         task: LLMTask,
         trial_id: int,
+        experiment_: Optional[experiment.Experiment],
     ) -> test_result.TestResult:
         if not hasattr(task, "opik_tracked"):
             name = task.__name__ if hasattr(task, "__name__") else "llm_task"
@@ -113,7 +123,7 @@ class EvaluationEngine:
         )
 
         with helpers.evaluate_llm_task_context(
-            experiment=self._experiment,
+            experiment=experiment_,
             dataset_item_id=item.id,
             trace_data=trace_data,
             client=self._client,
@@ -132,121 +142,53 @@ class EvaluationEngine:
 
             opik_context.update_current_trace(output=task_output_)
 
-            scoring_inputs = arguments_helpers.create_scoring_inputs(
-                dataset_item=item_content,
-                task_output=task_output_,
-                scoring_key_mapping=self._scoring_key_mapping,
-            )
-
             test_case_ = test_case.TestCase(
                 trace_id=trace_data.id,
                 dataset_item_id=item.id,
-                scoring_inputs=scoring_inputs,
                 task_output=task_output_,
                 dataset_item_content=item_content,
             )
-            test_result_ = self._evaluate_test_case(
+            test_result_ = self._compute_test_result_for_test_case(
                 test_case_=test_case_,
                 trial_id=trial_id,
             )
 
         return test_result_
 
-    def evaluate_llm_tasks(
+    def _compute_test_results_for_llm_task(
         self,
-        dataset_: dataset.Dataset,
+        dataset_items: List[dataset_item.DatasetItem],
         task: LLMTask,
-        nb_samples: Optional[int],
-        dataset_item_ids: Optional[List[str]],
-        dataset_sampler: Optional[samplers.BaseDatasetSampler],
+        experiment_: Optional[experiment.Experiment],
         trial_count: int,
+        description: str,
     ) -> List[test_result.TestResult]:
-        task_span_scoring_enabled = False
-        if len(self._task_span_scoring_metrics) > 0:
-            message_processors_chain.toggle_local_emulator_message_processor(
-                active=True, chain=self._client._message_processor
-            )
-            task_span_scoring_enabled = True
-
-        dataset_items = dataset_.__internal_api__get_items_as_dataclasses__(
-            nb_samples=nb_samples,
-            dataset_item_ids=dataset_item_ids,
-        )
-
-        if dataset_sampler is not None:
-            dataset_items = dataset_sampler.sample(dataset_items)
-
         test_results: List[test_result.TestResult] = []
 
         for trial_id in range(trial_count):
             evaluation_tasks: List[EvaluationTask[test_result.TestResult]] = [
                 functools.partial(
-                    self._evaluate_llm_task,
+                    self._compute_test_result_for_llm_task,
                     item=item,
                     task=task,
                     trial_id=trial_id,
+                    experiment_=experiment_,
                 )
                 for item in dataset_items
             ]
 
             test_results += evaluation_tasks_executor.execute(
-                evaluation_tasks,
-                self._workers,
-                self._verbose,
-                desc=f"Evaluation trial {trial_id}"
+                evaluation_tasks=evaluation_tasks,
+                workers=self._workers,
+                verbose=self._verbose,
+                desc=f"{description} trial {trial_id}"
                 if trial_count > 1
-                else "Evaluation",
-            )
-
-        if task_span_scoring_enabled:
-            # flush Opik client to make sure all spans are collected
-            self._client.flush()
-
-            self._evaluate_llm_tasks_spans(test_results)
-
-            LOGGER.info(
-                "Task evaluation span handling is disabled — the evaluation has been completed."
-            )
-            message_processors_chain.toggle_local_emulator_message_processor(
-                active=False, chain=self._client._message_processor
+                else description,
             )
 
         return test_results
 
-    def _evaluate_llm_tasks_spans(
-        self, test_results: List[test_result.TestResult]
-    ) -> None:
-        local = message_processors_chain.get_local_emulator_message_processor(
-            chain=self._client._message_processor
-        )
-        if local is None:
-            LOGGER.warning("Local emulator message processor not found in the chain.")
-            return
-
-        # get trace trees from a local emulator
-        trace_trees = local.trace_trees
-        if len(trace_trees) == 0:
-            LOGGER.warning("No trace trees found in the local emulator.")
-            return
-
-        # create span evaluation tasks from LLM tasks evaluation results and evaluate them in parallel
-        span_evaluation_tasks: List[EvaluationTask[test_result.TestResult]] = [
-            functools.partial(
-                self._evaluate_llm_task_result_span,
-                evaluation_task_result=test_result_,
-                trace_trees=trace_trees,
-            )
-            for test_result_ in test_results
-        ]
-
-        evaluation_tasks_executor.execute(
-            span_evaluation_tasks,
-            self._workers,
-            self._verbose,
-            desc="LLM task spans evaluation",
-        )
-
-    def _evaluate_llm_task_result_span(
+    def _update_test_result_with_task_span_metrics(
         self,
         evaluation_task_result: test_result.TestResult,
         trace_trees: List[models.TraceModel],
@@ -288,7 +230,7 @@ class EvaluationEngine:
             ),
             client=self._client,
         ):
-            score_results = self._score_llm_task_result_span(
+            score_results = self._compute_scores_for_test_case_with_task_span(
                 trace_id=trace_id,
                 task_span=evaluation_span,
                 test_case_=evaluation_task_result.test_case,
@@ -297,129 +239,159 @@ class EvaluationEngine:
             evaluation_task_result.score_results += score_results
             return evaluation_task_result
 
-    @opik.track(  # type: ignore[attr-defined,has-type]
-        name="task_span_metrics_calculation",
-        ignore_arguments=["test_case_"],
-    )
-    def _score_llm_task_result_span(
+    def _update_test_results_with_task_span_metrics(
         self,
-        trace_id: str,
-        task_span: models.SpanModel,
-        test_case_: test_case.TestCase,
-    ) -> List[score_result.ScoreResult]:
-        score_kwargs = {
-            **test_case_.scoring_inputs,
-            EVALUATION_SPAN_PARAMETER_NAME: task_span,
-        }
-
-        score_results = _scores_by_metrics(
-            scoring_metrics=self._task_span_scoring_metrics,
-            score_kwargs=score_kwargs,
-            scoring_key_mapping=self._scoring_key_mapping,
-            test_case_=test_case_,
+        test_results: List[test_result.TestResult],
+        recording: local_recording._LocalRecordingHandle,
+    ) -> None:
+        """Evaluate task spans from a local recording."""
+        # Get trace trees from the recording (this flushes automatically)
+        trace_trees = recording.trace_trees
+        if len(trace_trees) == 0:
+            LOGGER.warning("No trace trees found in the local recording.")
+            return
+
+        # Create span evaluation tasks from LLM tasks evaluation results and evaluate them in parallel
+        span_evaluation_tasks: List[EvaluationTask[test_result.TestResult]] = [
+            functools.partial(
+                self._update_test_result_with_task_span_metrics,
+                evaluation_task_result=test_result_,
+                trace_trees=trace_trees,
+            )
+            for test_result_ in test_results
+        ]
+
+        evaluation_tasks_executor.execute(
+            evaluation_tasks=span_evaluation_tasks,
+            workers=self._workers,
+            verbose=self._verbose,
+            desc="LLM task spans evaluation",
         )
 
-        # log feedback scores
-        rest_operations.log_test_result_feedback_scores(
-            client=self._client,
-            score_results=score_results,
-            trace_id=trace_id,
-            project_name=self._project_name,
+        LOGGER.debug(
+            "Task evaluation span handling is disabled — the evaluation has been completed."
         )
-        return score_results
 
-    def evaluate_test_cases(
+    def evaluate_llm_task_on_dataset(
         self,
-        test_cases: List[test_case.TestCase],
+        dataset_: dataset.Dataset,
+        task: LLMTask,
+        nb_samples: Optional[int],
+        dataset_item_ids: Optional[List[str]],
+        dataset_sampler: Optional[samplers.BaseDatasetSampler],
+        trial_count: int,
+        experiment_: Optional[experiment.Experiment],
     ) -> List[test_result.TestResult]:
-        evaluation_tasks: List[EvaluationTask[test_result.TestResult]] = [
-            functools.partial(
-                self._evaluate_test_case,
-                test_case_=test_case_,
+        dataset_items = dataset_.__internal_api__get_items_as_dataclasses__(
+            nb_samples=nb_samples,
+            dataset_item_ids=dataset_item_ids,
+        )
+
+        if dataset_sampler is not None:
+            dataset_items = dataset_sampler.sample(dataset_items)
+
+        if not self._metrics_evaluator.has_task_span_metrics:
+            return self._compute_test_results_for_llm_task(
+                dataset_items=dataset_items,
+                task=task,
+                experiment_=experiment_,
+                trial_count=trial_count,
+                description="Evaluation",
             )
-            for test_case_ in test_cases
-        ]
 
-        test_results = evaluation_tasks_executor.execute(
-            evaluation_tasks, self._workers, self._verbose
+        LOGGER.debug(
+            "Detected %d LLM task span scoring metrics — enabling handling of the LLM task evaluation span.",
+            len(self._metrics_evaluator.task_span_metrics),
         )
 
+        with local_recording.record_traces_locally(client=self._client) as recording:
+            test_results = self._compute_test_results_for_llm_task(
+                dataset_items=dataset_items,
+                task=task,
+                experiment_=experiment_,
+                trial_count=trial_count,
+                description="Evaluation",
+            )
+            self._update_test_results_with_task_span_metrics(
+                test_results=test_results,
+                recording=recording,
+            )
+
         return test_results
 
+    def evaluate_llm_task_on_dict_items(
+        self,
+        items: List[Dict[str, Any]],
+        task: LLMTask,
+    ) -> List[test_result.TestResult]:
+        """
+        Evaluate an LLM task on a list of dict items.
+
+        This method creates traces for each evaluation but doesn't require a Dataset object
+        or experiment. It's useful for optimization scenarios where you have items in memory
+        and want to evaluate them with a task function.
+
+        Args:
+            items: List of dataset item contents (dictionaries).
+            task: A callable that takes a dataset item dict and returns a dict with outputs.
+
+        Returns:
+            List of TestResult objects containing scores for each item.
+        """
+        # Convert raw items to DatasetItem objects for compatibility
+        dataset_items = [
+            dataset_item.DatasetItem(
+                id=f"temp_item_{idx}",
+                **item,
+            )
+            for idx, item in enumerate(items)
+        ]
 
-def _scores_by_metrics(
-    scoring_metrics: List[base_metric.BaseMetric],
-    score_kwargs: Dict[str, Any],
-    scoring_key_mapping: Optional[ScoringKeyMappingType],
-    test_case_: test_case.TestCase,
-) -> List[score_result.ScoreResult]:
-    score_results: List[score_result.ScoreResult] = []
-    for metric in scoring_metrics:
-        try:
-            LOGGER.debug("Metric %s score started", metric.name)
-
-            if isinstance(metric, scorer_wrapper_metric.ScorerWrapperMetric):
-                # use original dataset item content without any mappings applied
-                if (
-                    task_span := score_kwargs.get(EVALUATION_SPAN_PARAMETER_NAME)
-                ) is not None:
-                    result = metric.score(
-                        dataset_item=test_case_.dataset_item_content,
-                        task_outputs=test_case_.task_output,
-                        task_span=task_span,
-                    )
-                else:
-                    result = metric.score(
-                        dataset_item=test_case_.dataset_item_content,
-                        task_outputs=test_case_.task_output,
-                    )
-            else:
-                arguments_validator.validate_score_arguments(
-                    metric=metric,
-                    kwargs=score_kwargs,
-                    scoring_key_mapping=scoring_key_mapping,
-                )
-                result = metric.score(**score_kwargs)
-
-            LOGGER.debug("Metric %s score ended", metric.name)
-
-            if isinstance(result, list):
-                score_results += result
-            else:
-                score_results.append(result)
-        except exceptions.ScoreMethodMissingArguments:
-            raise
-        except Exception as exception:
-            # This can be problematic if the metric returns a list of strings as we will not know the name of the metrics that have failed
-            LOGGER.error(
-                "Failed to compute metric %s. Score result will be marked as failed.",
-                metric.name,
-                exc_info=True,
+        if not self._metrics_evaluator.has_task_span_metrics:
+            return self._compute_test_results_for_llm_task(
+                dataset_items=dataset_items,
+                task=task,
+                experiment_=None,
+                trial_count=1,
+                description="Items evaluation",
             )
 
-            if exception_analyzer.is_llm_provider_rate_limit_error(exception):
-                LOGGER.error(
-                    logging_messages.LLM_PROVIDER_RATE_LIMIT_ERROR_DETECTED_IN_EVALUATE_FUNCTION
-                )
+        LOGGER.debug(
+            "Detected %d LLM task span scoring metrics — enabling handling of the LLM task evaluation span.",
+            len(self._metrics_evaluator.task_span_metrics),
+        )
 
-            score_results.append(
-                score_result.ScoreResult(
-                    name=metric.name,
-                    value=0.0,
-                    reason=str(exception),
-                    scoring_failed=True,
-                )
+        with local_recording.record_traces_locally(client=self._client) as recording:
+            test_results = self._compute_test_results_for_llm_task(
+                dataset_items=dataset_items,
+                task=task,
+                experiment_=None,
+                trial_count=1,
+                description="Items evaluation",
+            )
+            self._update_test_results_with_task_span_metrics(
+                test_results=test_results,
+                recording=recording,
             )
 
-    return score_results
+        return test_results
 
+    def evaluate_test_cases(
+        self,
+        test_cases: List[test_case.TestCase],
+    ) -> List[test_result.TestResult]:
+        evaluation_tasks: List[EvaluationTask[test_result.TestResult]] = [
+            functools.partial(
+                self._compute_test_result_for_test_case,
+                test_case_=test_case_,
+            )
+            for test_case_ in test_cases
+        ]
 
-def _has_evaluation_span_parameter(func: Callable) -> bool:
-    try:
-        sig = inspect.signature(func)
-        has_param = EVALUATION_SPAN_PARAMETER_NAME in sig.parameters
-    except (ValueError, TypeError):
-        # If we can't inspect the signature, assume no parameter
-        has_param = False
+        test_results = evaluation_tasks_executor.execute(
+            evaluation_tasks=evaluation_tasks,
+            workers=self._workers,
+            verbose=self._verbose,
+        )
 
-    return has_param
+        return test_results