opik 1.9.39__py3-none-any.whl → 1.9.86__py3-none-any.whl

opik/file_upload/file_uploader.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from typing import Optional
 
 import httpx
@@ -28,6 +29,10 @@ def upload_attachment(
             httpx_client=upload_httpx_client,
             monitor=monitor,
         )
+
+        # delete the file after upload if requested
+        if upload_options.delete_after_upload:
+            _delete_attachment_file(upload_options.file_path)
     except Exception as e:
         LOGGER.error(
             "Failed to upload attachment: '%s' from file: [%s] with size: [%s]. Error: %s",
@@ -40,6 +45,14 @@ def upload_attachment(
         raise
 
 
+def _delete_attachment_file(file_path: str) -> None:
+    try:
+        os.unlink(file_path)
+    except OSError as e:
+        LOGGER.info(f"Failed to delete attachment file: '{file_path}'. Reason: {e}.")
+        pass
+
+
 def _do_upload_attachment(
     upload_options: file_upload_options.FileUploadOptions,
     rest_client: rest_api_client.OpikApi,

opik/file_upload/upload_options.py

@@ -16,6 +16,7 @@ class FileUploadOptions:
     entity_id: str
     project_name: str
     encoded_url_override: str
+    delete_after_upload: bool
 
 
 def file_upload_options_from_attachment(
@@ -32,4 +33,5 @@ def file_upload_options_from_attachment(
         entity_id=attachment.entity_id,
         project_name=attachment.project_name,
         encoded_url_override=attachment.encoded_url_override,
+        delete_after_upload=attachment.delete_after_upload,
     )

opik/integrations/adk/legacy_opik_tracer.py

@@ -158,15 +158,13 @@ class LegacyOpikTracer:
                     input=user_input,
                     type="general",
                 )
-                _, opik_span_data = (
-                    span_creation_handler.create_span_respecting_context(
-                        start_span_arguments=start_span_arguments,
-                        distributed_trace_headers=None,
-                        opik_context_storage=self._context_storage,
-                    )
+                result = span_creation_handler.create_span_respecting_context(
+                    start_span_arguments=start_span_arguments,
+                    distributed_trace_headers=None,
+                    opik_context_storage=self._context_storage,
                 )
 
-                self._start_span(span_data=opik_span_data)
+                self._start_span(span_data=result.span_data)
         except Exception as e:
             LOGGER.error(f"Failed during before_agent_callback(): {e}", exc_info=True)
 
@@ -212,7 +210,7 @@ class LegacyOpikTracer:
             if provider is None:
                 provider = adk_helpers.get_adk_provider()
 
-            _, span_data = span_creation_handler.create_span_respecting_context(
+            result = span_creation_handler.create_span_respecting_context(
                 start_span_arguments=arguments_helpers.StartSpanParameters(
                     name=llm_request.model,
                     project_name=self.project_name,
@@ -226,7 +224,7 @@ class LegacyOpikTracer:
                 opik_context_storage=self._context_storage,
             )
 
-            self._start_span(span_data=span_data)
+            self._start_span(span_data=result.span_data)
 
         except Exception as e:
             LOGGER.error(f"Failed during before_model_callback(): {e}", exc_info=True)
@@ -300,7 +298,7 @@ class LegacyOpikTracer:
                 **self.metadata,
             }
 
-            _, span_data = span_creation_handler.create_span_respecting_context(
+            result = span_creation_handler.create_span_respecting_context(
                 start_span_arguments=arguments_helpers.StartSpanParameters(
                     name=tool.name,
                     project_name=self.project_name,
@@ -312,7 +310,7 @@ class LegacyOpikTracer:
                 opik_context_storage=self._context_storage,
             )
 
-            self._start_span(span_data=span_data)
+            self._start_span(span_data=result.span_data)
 
         except Exception as e:
             LOGGER.error(f"Failed during before_tool_callback(): {e}", exc_info=True)

opik/integrations/adk/opik_tracer.py

@@ -173,7 +173,7 @@ class OpikTracer:
             # ADK runs `before_model_callback` before running `start_as_current_span` function for the LLM call,
             # which makes it impossible to update the Opik span from this method.
             # So we create a span manually here. This flow is handled inside ADKTracerWrapper.
-            _, span_data = span_creation_handler.create_span_respecting_context(
+            result = span_creation_handler.create_span_respecting_context(
                 start_span_arguments=arguments_helpers.StartSpanParameters(
                     name=model,
                     project_name=self.project_name,
@@ -189,7 +189,7 @@ class OpikTracer:
                 distributed_trace_headers=None,
             )
 
-            context_storage.add_span_data(span_data)
+            context_storage.add_span_data(result.span_data)
         except Exception as e:
             LOGGER.error(f"Failed during before_model_callback(): {e}", exc_info=True)
 

opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py

@@ -190,11 +190,11 @@ def _prepare_trace_and_span_to_be_finalized(
             type="general",
         )
 
-        _, span_to_close_in_finally_block = (
+        span_to_close_in_finally_block = (
             span_creation_handler.create_span_respecting_context(
                 start_span_arguments=start_span_arguments,
                 distributed_trace_headers=None,
-            )
+            ).span_data
         )
         opik.context_storage.add_span_data(span_to_close_in_finally_block)
 

opik/integrations/dspy/callback.py

@@ -1,14 +1,16 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Union
 import logging
 
 import dspy
 from dspy.utils import callback as dspy_callback
 
-from opik import context_storage, opik_context, tracing_runtime_config, types
+from opik import context_storage, opik_context, tracing_runtime_config
+from opik import llm_usage
 from opik.api_objects import helpers, span, trace, opik_client
 from opik.decorator import error_info_collector
 
 from .graph import build_mermaid_graph_from_module
+from .parsers import LMHistoryInfo, extract_lm_info_from_history, get_span_type
 
 LOGGER = logging.getLogger(__name__)
 
@@ -32,6 +34,8 @@ class OpikCallback(dspy_callback.BaseCallback):
     ):
         self._map_call_id_to_span_data: Dict[str, span.SpanData] = {}
         self._map_call_id_to_trace_data: Dict[str, trace.TraceData] = {}
+        # Store (lm_instance, expected_messages) for extracting usage and verifying correct history entry
+        self._map_call_id_to_lm_info: Dict[str, Tuple[Any, Optional[Any]]] = {}
 
         self._origins_metadata: Dict[str, Any] = {"created_from": "dspy"}
 
@@ -103,7 +107,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             parent_project_name=current_span_data.project_name,
             child_project_name=self._project_name,
         )
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         span_data = span.SpanData(
             trace_id=current_span_data.trace_id,
@@ -127,7 +131,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             current_trace_data.project_name,
             self._project_name,
         )
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         span_data = span.SpanData(
             trace_id=current_trace_data.id,
@@ -198,13 +202,54 @@ class OpikCallback(dspy_callback.BaseCallback):
         call_id: str,
         outputs: Optional[Any],
         exception: Optional[Exception] = None,
+        usage: Optional[llm_usage.OpikUsage] = None,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        actual_provider: Optional[str] = None,
+        actual_model: Optional[str] = None,
+        total_cost: Optional[float] = None,
     ) -> None:
         if span_data := self._map_call_id_to_span_data.pop(call_id, None):
             if exception:
                 error_info = error_info_collector.collect(exception)
                 span_data.update(error_info=error_info)
 
-            span_data.update(output={"output": outputs}).init_end_time()
+            # Prepare the update dict
+            update_kwargs: Dict[str, Any] = {
+                "output": {"output": outputs},
+                "usage": usage,
+                "total_cost": total_cost,
+            }
+
+            # Handle LLM routers like OpenRouter that return the actual serving provider/model
+            if extra_metadata is None:
+                extra_metadata = {}
+
+            # Update provider if actual provider differs (e.g., OpenRouter -> Hyperbolic)
+            if (
+                actual_provider is not None
+                and span_data.provider is not None
+                and span_data.provider.lower() != actual_provider.lower()
+            ):
+                # Store the original provider (e.g., "openrouter") in metadata
+                extra_metadata["llm_router"] = span_data.provider
+                # Update to the actual provider for accurate cost tracking
+                update_kwargs["provider"] = actual_provider.lower()
+
+            if (
+                actual_model is not None
+                and span_data.model is not None
+                and span_data.model != actual_model
+            ):
+                # Store the original model (e.g., "@preset/qwen") in metadata
+                extra_metadata["original_model"] = span_data.model
+                # Update to the actual model for accurate cost tracking
+                update_kwargs["model"] = actual_model
+
+            # Only set metadata if we have something to add
+            if extra_metadata:
+                update_kwargs["metadata"] = extra_metadata
+
+            span_data.update(**update_kwargs).init_end_time()
             if tracing_runtime_config.is_tracing_active():
                 self._opik_client.span(**span_data.as_parameters)
 
@@ -231,7 +276,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             trace_id = current_callback_context_data.id
             parent_span_id = None
 
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         return span.SpanData(
             trace_id=trace_id,
@@ -263,6 +308,13 @@ class OpikCallback(dspy_callback.BaseCallback):
             name=f"{span_data.name}: {provider} - {model}",
         )
         self._map_call_id_to_span_data[call_id] = span_data
+
+        # Store LM instance and expected messages for extracting usage
+        self._map_call_id_to_lm_info[call_id] = (
+            instance,
+            inputs.get("messages"),
+        )
+
         self._set_current_context_data(span_data)
 
     def on_lm_end(
@@ -271,10 +323,22 @@ class OpikCallback(dspy_callback.BaseCallback):
         outputs: Optional[Dict[str, Any]],
         exception: Optional[Exception] = None,
     ) -> None:
+        lm_info = self._extract_lm_info_from_history(call_id)
+
+        # Add cache_hit to span metadata only when we have a definitive value
+        extra_metadata = (
+            {"cache_hit": lm_info.cache_hit} if lm_info.cache_hit is not None else None
+        )
+
         self._end_span(
             call_id=call_id,
             exception=exception,
             outputs=outputs,
+            usage=lm_info.usage,
+            extra_metadata=extra_metadata,
+            actual_provider=lm_info.actual_provider,
+            actual_model=lm_info.actual_model,
+            total_cost=lm_info.total_cost,
         )
 
     def on_tool_start(
@@ -316,14 +380,36 @@ class OpikCallback(dspy_callback.BaseCallback):
             return span_data
         return self._context_storage.get_trace_data()
 
-    def _get_span_type(self, instance: Any) -> types.SpanType:
-        if isinstance(instance, dspy.Predict):
-            return "llm"
-        elif isinstance(instance, dspy.LM):
-            return "llm"
-        elif isinstance(instance, dspy.Tool):
-            return "tool"
-        return "general"
+    def _extract_lm_info_from_history(self, call_id: str) -> LMHistoryInfo:
+        """
+        Extract token usage, cache status, actual provider, and cost from the LM's history.
+
+        DSPy stores usage information in the LM's history after each call.
+        We verify the history entry matches our expected messages to handle
+        potential race conditions with concurrent LM calls.
+
+        For routers like OpenRouter, the response contains the actual provider
+        that served the request (e.g., "Novita", "Together"), which differs from
+        the router name used in the model string (e.g., "openrouter").
+
+        The cost field is provided by providers like OpenRouter and includes
+        accurate pricing for all token types (reasoning, cache, multimodal).
+
+        Returns:
+            LMHistoryInfo containing usage, cache_hit, actual_provider, and total_cost.
+        """
+        lm_info = self._map_call_id_to_lm_info.pop(call_id, None)
+        if lm_info is None:
+            return LMHistoryInfo(
+                usage=None,
+                cache_hit=None,
+                actual_provider=None,
+                actual_model=None,
+                total_cost=None,
+            )
+
+        lm_instance, expected_messages = lm_info
+        return extract_lm_info_from_history(lm_instance, expected_messages)
 
     def _get_opik_metadata(self, instance: Any) -> Dict[str, Any]:
         graph = None

opik/integrations/dspy/parsers.py

@@ -0,0 +1,168 @@
+"""
+Parsers and data structures for extracting information from DSPy LM responses.
+
+This module contains utilities for parsing DSPy LM history entries and
+extracting relevant information like usage, provider, and cost data.
+"""
+
+from dataclasses import dataclass
+from typing import Any, Optional
+import logging
+
+import dspy
+
+from opik import llm_usage, types
+
+LOGGER = logging.getLogger(__name__)
+
+
+@dataclass
+class LMHistoryInfo:
+    """
+    Information extracted from a DSPy LM history entry.
+
+    This dataclass holds the parsed information from an LM call's history,
+    including usage statistics, cache status, provider information, and cost.
+
+    Attributes:
+        usage: Token usage information (prompt, completion, total tokens)
+        cache_hit: Whether the response was served from cache.
+            True if cached, False if not, None if unknown.
+        actual_provider: The actual provider that served the request.
+            This is useful for LLM routers like OpenRouter that may route
+            to different underlying providers (e.g., "Novita", "Together").
+        actual_model: The actual model that served the request.
+            This is useful for LLM routers like OpenRouter when using presets
+            (e.g., "@preset/qwen" resolves to "qwen/qwen3-235b-a22b-2507").
+        total_cost: The total cost of the request from the provider.
+            This includes accurate pricing for all token types.
+    """
+
+    usage: Optional[llm_usage.OpikUsage]
+    cache_hit: Optional[bool]
+    actual_provider: Optional[str]
+    actual_model: Optional[str]
+    total_cost: Optional[float]
+
+
+def get_span_type(instance: Any) -> types.SpanType:
+    """
+    Determine the span type based on the DSPy instance type.
+
+    Args:
+        instance: A DSPy module, LM, or tool instance.
+
+    Returns:
+        The appropriate span type: "llm" for Predict/LM, "tool" for Tool,
+        or "general" for other types.
+    """
+    if isinstance(instance, dspy.Predict):
+        return "llm"
+    elif isinstance(instance, dspy.LM):
+        return "llm"
+    elif isinstance(instance, dspy.Tool):
+        return "tool"
+    return "general"
+
+
+def extract_lm_info_from_history(
+    lm_instance: Any,
+    expected_messages: Optional[Any],
+) -> LMHistoryInfo:
+    """
+    Extract token usage, cache status, actual provider, and cost from the LM's history.
+
+    DSPy stores usage information in the LM's history after each call.
+    We verify the history entry matches our expected messages to handle
+    potential race conditions with concurrent LM calls.
+
+    For routers like OpenRouter, the response contains the actual provider
+    that served the request (e.g., "Novita", "Together"), which differs from
+    the router name used in the model string (e.g., "openrouter").
+
+    The cost field is provided by providers like OpenRouter and includes
+    accurate pricing for all token types (reasoning, cache, multimodal).
+
+    Args:
+        lm_instance: The DSPy LM instance that has the history.
+        expected_messages: The expected messages to match in the history entry.
+
+    Returns:
+        LMHistoryInfo containing usage, cache_hit, actual_provider, and total_cost.
+    """
+    empty_result = LMHistoryInfo(
+        usage=None,
+        cache_hit=None,
+        actual_provider=None,
+        actual_model=None,
+        total_cost=None,
+    )
+
+    if not hasattr(lm_instance, "history") or not lm_instance.history:
+        return empty_result
+
+    try:
+        last_entry = lm_instance.history[-1]
+
+        # Verify we have the correct history entry by checking messages match
+        if last_entry.get("messages") != expected_messages:
+            LOGGER.debug(
+                "History entry messages don't match expected messages, "
+                "skipping usage extraction (possibly due to concurrent LM calls)"
+            )
+            return empty_result
+
+        response = last_entry.get("response")
+        usage_dict = last_entry.get("usage")
+
+        # Extract actual provider and model from response (useful for routers like OpenRouter)
+        # The response is a LiteLLM ModelResponse object with 'provider' and 'model' attributes
+        # when using routers like OpenRouter
+        actual_provider: Optional[str] = None
+        actual_model: Optional[str] = None
+        if response is not None:
+            if hasattr(response, "provider"):
+                actual_provider = response.provider
+            if hasattr(response, "model"):
+                actual_model = response.model
+
+        # Extract cost from history entry or usage dict
+        # OpenRouter and other providers return accurate cost including all token types
+        total_cost: Optional[float] = None
+        if (cost := last_entry.get("cost") or 0) > 0:
+            total_cost = cost
+        elif usage_dict and (cost := usage_dict.get("cost") or 0) > 0:
+            total_cost = cost
+
+        # Get explicit cache_hit if set, otherwise infer from usage (empty = cached)
+        if response is None:
+            cache_hit = not usage_dict
+        elif hasattr(response, "cache_hit") and response.cache_hit is not None:
+            cache_hit = response.cache_hit
+        else:
+            # Fallback: infer from usage (empty = cached)
+            cache_hit = not usage_dict
+
+        if usage_dict:
+            usage = llm_usage.build_opik_usage_from_unknown_provider(usage_dict)
+            return LMHistoryInfo(
+                usage=usage,
+                cache_hit=cache_hit,
+                actual_provider=actual_provider,
+                actual_model=actual_model,
+                total_cost=total_cost,
+            )
+        else:
+            return LMHistoryInfo(
+                usage=None,
+                cache_hit=cache_hit,
+                actual_provider=actual_provider,
+                actual_model=actual_model,
+                total_cost=None,
+            )
+    except Exception:
+        LOGGER.debug(
+            "Failed to extract info from DSPy LM history",
+            exc_info=True,
+        )
+        return empty_result

opik/integrations/harbor/__init__.py

@@ -0,0 +1,17 @@
+"""
+Opik integration for Harbor benchmark evaluation framework.
+
+Example:
+    >>> from opik.integrations.harbor import track_harbor
+    >>> job = Job(config)
+    >>> tracked_job = track_harbor(job)
+    >>> result = await tracked_job.run()
+
+Or enable tracking globally (for CLI usage):
+    >>> from opik.integrations.harbor import track_harbor
+    >>> track_harbor()
+"""
+
+from .opik_tracker import track_harbor, reset_harbor_tracking
+
+__all__ = ["track_harbor", "reset_harbor_tracking"]