opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/integrations/crewai/patchers/llm_client.py

@@ -0,0 +1,207 @@
+"""
+Client patchers for CrewAI LLM providers (v1.0.0+).
+
+This module handles patching of LLM clients used by CrewAI agents with Opik tracking.
+Each provider has its own patching function that handles missing dependencies gracefully.
+"""
+
+import logging
+from typing import Any, Optional, TYPE_CHECKING
+
+import crewai
+
+if TYPE_CHECKING:
+    import crewai.llms.providers.openai.completion as openai_completion
+    import crewai.llms.providers.anthropic.completion as anthropic_completion
+    import crewai.llms.providers.gemini.completion as gemini_completion
+    import crewai.llms.providers.bedrock.completion as bedrock_completion
+
+LOGGER = logging.getLogger(__name__)
+
+
+def patch_llm_client(crew: crewai.Crew, project_name: Optional[str]) -> None:
+    """
+    Patches LLM clients used by CrewAI agents with Opik tracking.
+
+    Handles missing provider libraries gracefully by logging warnings instead of failing.
+
+    Args:
+        crew: The Crew instance containing agents to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    for agent in crew.agents:
+        _patch_single_llm_client(agent.llm, project_name)
+
+
+def _patch_single_llm_client(llm: Any, project_name: Optional[str]) -> None:
+    """
+    Patches an LLM client based on its provider type.
+
+    Args:
+        llm: The CrewAI LLM instance to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    if _is_openai_llm(llm):
+        _patch_openai_client(llm, project_name)
+    elif _is_anthropic_llm(llm):
+        _patch_anthropic_client(llm, project_name)
+    elif _is_gemini_llm(llm):
+        _patch_gemini_client(llm, project_name)
+    elif _is_bedrock_llm(llm):
+        _patch_bedrock_client(llm, project_name)
+
+
+def _is_openai_llm(llm: Any) -> bool:
+    """
+    Checks if LLM is an OpenAI provider.
+
+    Args:
+        llm: The CrewAI LLM to check.
+
+    Returns:
+        True if LLM is OpenAI provider, False otherwise.
+    """
+    try:
+        import crewai.llms.providers.openai.completion
+
+        return isinstance(llm, crewai.llms.providers.openai.completion.OpenAICompletion)
+    except ImportError:
+        return False
+
+
+def _is_anthropic_llm(llm: Any) -> bool:
+    """
+    Checks if LLM is an Anthropic provider.
+
+    Args:
+        llm: The CrewAI LLM to check.
+
+    Returns:
+        True if LLM is Anthropic provider, False otherwise.
+    """
+    try:
+        import crewai.llms.providers.anthropic.completion
+
+        return isinstance(
+            llm, crewai.llms.providers.anthropic.completion.AnthropicCompletion
+        )
+    except ImportError:
+        return False
+
+
+def _is_gemini_llm(llm: Any) -> bool:
+    """
+    Checks if LLM is a Gemini provider.
+
+    Args:
+        llm: The CrewAI LLM to check.
+
+    Returns:
+        True if LLM is Gemini provider, False otherwise.
+    """
+    try:
+        import crewai.llms.providers.gemini.completion
+
+        return isinstance(llm, crewai.llms.providers.gemini.completion.GeminiCompletion)
+    except ImportError:
+        return False
+
+
+def _is_bedrock_llm(llm: Any) -> bool:
+    """
+    Checks if LLM is a Bedrock provider.
+
+    Args:
+        llm: The CrewAI LLM to check.
+
+    Returns:
+        True if LLM is Bedrock provider, False otherwise.
+    """
+    try:
+        import crewai.llms.providers.bedrock.completion
+
+        return isinstance(
+            llm, crewai.llms.providers.bedrock.completion.BedrockCompletion
+        )
+    except ImportError:
+        return False
+
+
+def _patch_openai_client(
+    llm: "openai_completion.OpenAICompletion", project_name: Optional[str]
+) -> None:
+    """
+    Patches OpenAI client for the given LLM.
+
+    Args:
+        llm: The CrewAI LLM instance with OpenAI client to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    try:
+        import opik.integrations.openai
+
+        llm.client = opik.integrations.openai.track_openai(
+            llm.client, project_name=project_name
+        )
+    except Exception:
+        LOGGER.warning("Failed to track OpenAI client for LLM", exc_info=True)
+
+
+def _patch_anthropic_client(
+    llm: "anthropic_completion.AnthropicCompletion", project_name: Optional[str]
+) -> None:
+    """
+    Patches Anthropic client for the given LLM.
+
+    Args:
+        llm: The CrewAI LLM instance with Anthropic client to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    try:
+        import opik.integrations.anthropic
+
+        llm.client = opik.integrations.anthropic.track_anthropic(
+            llm.client, project_name=project_name
+        )
+    except Exception:
+        LOGGER.warning("Failed to track Anthropic client for LLM", exc_info=True)
+
+
+def _patch_gemini_client(
+    llm: "gemini_completion.GeminiCompletion", project_name: Optional[str]
+) -> None:
+    """
+    Patches Gemini client for the given LLM.
+
+    Args:
+        llm: The CrewAI LLM instance with Gemini client to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    try:
+        import opik.integrations.genai
+
+        llm.client = opik.integrations.genai.track_genai(
+            llm.client, project_name=project_name
+        )
+    except Exception:
+        LOGGER.warning("Failed to track Gemini client for LLM", exc_info=True)
+
+
+def _patch_bedrock_client(
+    llm: "bedrock_completion.BedrockCompletion", project_name: Optional[str]
+) -> None:
+    """
+    Patches Bedrock client for the given LLM.
+
+    Args:
+        llm: The CrewAI LLM instance with Bedrock client to patch.
+        project_name: The name of the project to associate with tracking.
+    """
+    try:
+        import opik.integrations.bedrock
+
+        llm.client = opik.integrations.bedrock.track_bedrock(
+            llm.client, project_name=project_name
+        )
+    except Exception:
+        LOGGER.warning("Failed to track Bedrock client for LLM", exc_info=True)

opik/integrations/dspy/callback.py

@@ -1,17 +1,16 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Union
 import logging
 
 import dspy
 from dspy.utils import callback as dspy_callback
 
-import opik.types as types
-import opik.opik_context as opik_context
-import opik.context_storage as context_storage
+from opik import context_storage, opik_context, tracing_runtime_config
+from opik import llm_usage
 from opik.api_objects import helpers, span, trace, opik_client
-import opik.decorator.tracing_runtime_config as tracing_runtime_config
 from opik.decorator import error_info_collector
 
 from .graph import build_mermaid_graph_from_module
+from .parsers import LMHistoryInfo, extract_lm_info_from_history, get_span_type
 
 LOGGER = logging.getLogger(__name__)
 
@@ -35,6 +34,8 @@ class OpikCallback(dspy_callback.BaseCallback):
     ):
         self._map_call_id_to_span_data: Dict[str, span.SpanData] = {}
         self._map_call_id_to_trace_data: Dict[str, trace.TraceData] = {}
+        # Store (lm_instance, expected_messages) for extracting usage and verifying correct history entry
+        self._map_call_id_to_lm_info: Dict[str, Tuple[Any, Optional[Any]]] = {}
 
         self._origins_metadata: Dict[str, Any] = {"created_from": "dspy"}
 
@@ -106,7 +107,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             parent_project_name=current_span_data.project_name,
             child_project_name=self._project_name,
         )
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         span_data = span.SpanData(
             trace_id=current_span_data.trace_id,
@@ -130,7 +131,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             current_trace_data.project_name,
             self._project_name,
         )
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         span_data = span.SpanData(
             trace_id=current_trace_data.id,
@@ -201,13 +202,39 @@ class OpikCallback(dspy_callback.BaseCallback):
         call_id: str,
         outputs: Optional[Any],
         exception: Optional[Exception] = None,
+        usage: Optional[llm_usage.OpikUsage] = None,
+        extra_metadata: Optional[Dict[str, Any]] = None,
+        actual_provider: Optional[str] = None,
+        total_cost: Optional[float] = None,
     ) -> None:
         if span_data := self._map_call_id_to_span_data.pop(call_id, None):
             if exception:
                 error_info = error_info_collector.collect(exception)
                 span_data.update(error_info=error_info)
 
-            span_data.update(output={"output": outputs}).init_end_time()
+            # Prepare the update dict
+            update_kwargs: Dict[str, Any] = {
+                "output": {"output": outputs},
+                "usage": usage,
+                "total_cost": total_cost,
+            }
+
+            # Handle LLM routers like OpenRouter that return the actual serving provider
+            if (
+                actual_provider is not None
+                and span_data.provider is not None
+                and span_data.provider.lower() != actual_provider.lower()
+            ):
+                # Store the original provider (e.g., "openrouter") in metadata
+                if extra_metadata is None:
+                    extra_metadata = {}
+                extra_metadata["llm_router"] = span_data.provider
+                # Update to the actual provider for accurate cost tracking
+                update_kwargs["provider"] = actual_provider
+
+            update_kwargs["metadata"] = extra_metadata
+
+            span_data.update(**update_kwargs).init_end_time()
             if tracing_runtime_config.is_tracing_active():
                 self._opik_client.span(**span_data.as_parameters)
 
@@ -234,7 +261,7 @@ class OpikCallback(dspy_callback.BaseCallback):
             trace_id = current_callback_context_data.id
             parent_span_id = None
 
-        span_type = self._get_span_type(instance)
+        span_type = get_span_type(instance)
 
         return span.SpanData(
             trace_id=trace_id,
@@ -266,6 +293,13 @@ class OpikCallback(dspy_callback.BaseCallback):
             name=f"{span_data.name}: {provider} - {model}",
         )
         self._map_call_id_to_span_data[call_id] = span_data
+
+        # Store LM instance and expected messages for extracting usage
+        self._map_call_id_to_lm_info[call_id] = (
+            instance,
+            inputs.get("messages"),
+        )
+
         self._set_current_context_data(span_data)
 
     def on_lm_end(
@@ -274,10 +308,21 @@ class OpikCallback(dspy_callback.BaseCallback):
         outputs: Optional[Dict[str, Any]],
         exception: Optional[Exception] = None,
     ) -> None:
+        lm_info = self._extract_lm_info_from_history(call_id)
+
+        # Add cache_hit to span metadata only when we have a definitive value
+        extra_metadata = (
+            {"cache_hit": lm_info.cache_hit} if lm_info.cache_hit is not None else None
+        )
+
         self._end_span(
             call_id=call_id,
             exception=exception,
             outputs=outputs,
+            usage=lm_info.usage,
+            extra_metadata=extra_metadata,
+            actual_provider=lm_info.actual_provider,
+            total_cost=lm_info.total_cost,
         )
 
     def on_tool_start(
@@ -319,14 +364,32 @@ class OpikCallback(dspy_callback.BaseCallback):
             return span_data
         return self._context_storage.get_trace_data()
 
-    def _get_span_type(self, instance: Any) -> types.SpanType:
-        if isinstance(instance, dspy.Predict):
-            return "llm"
-        elif isinstance(instance, dspy.LM):
-            return "llm"
-        elif isinstance(instance, dspy.Tool):
-            return "tool"
-        return "general"
+    def _extract_lm_info_from_history(self, call_id: str) -> LMHistoryInfo:
+        """
+        Extract token usage, cache status, actual provider, and cost from the LM's history.
+
+        DSPy stores usage information in the LM's history after each call.
+        We verify the history entry matches our expected messages to handle
+        potential race conditions with concurrent LM calls.
+
+        For routers like OpenRouter, the response contains the actual provider
+        that served the request (e.g., "Novita", "Together"), which differs from
+        the router name used in the model string (e.g., "openrouter").
+
+        The cost field is provided by providers like OpenRouter and includes
+        accurate pricing for all token types (reasoning, cache, multimodal).
+
+        Returns:
+            LMHistoryInfo containing usage, cache_hit, actual_provider, and total_cost.
+        """
+        lm_info = self._map_call_id_to_lm_info.pop(call_id, None)
+        if lm_info is None:
+            return LMHistoryInfo(
+                usage=None, cache_hit=None, actual_provider=None, total_cost=None
+            )
+
+        lm_instance, expected_messages = lm_info
+        return extract_lm_info_from_history(lm_instance, expected_messages)
 
     def _get_opik_metadata(self, instance: Any) -> Dict[str, Any]:
         graph = None

opik/integrations/dspy/parsers.py

@@ -0,0 +1,168 @@
+"""
+Parsers and data structures for extracting information from DSPy LM responses.
+
+This module contains utilities for parsing DSPy LM history entries and
+extracting relevant information like usage, provider, and cost data.
+"""
+
+from dataclasses import dataclass
+from typing import Any, Optional, Tuple
+import logging
+
+import dspy
+
+from opik import llm_usage, types
+
+LOGGER = logging.getLogger(__name__)
+
+
+@dataclass
+class LMHistoryInfo:
+    """
+    Information extracted from a DSPy LM history entry.
+
+    This dataclass holds the parsed information from an LM call's history,
+    including usage statistics, cache status, provider information, and cost.
+
+    Attributes:
+        usage: Token usage information (prompt, completion, total tokens)
+        cache_hit: Whether the response was served from cache.
+            True if cached, False if not, None if unknown.
+        actual_provider: The actual provider that served the request.
+            This is useful for LLM routers like OpenRouter that may route
+            to different underlying providers (e.g., "Novita", "Together").
+        total_cost: The total cost of the request from the provider.
+            This includes accurate pricing for all token types.
+    """
+
+    usage: Optional[llm_usage.OpikUsage]
+    cache_hit: Optional[bool]
+    actual_provider: Optional[str]
+    total_cost: Optional[float]
+
+    def as_tuple(
+        self,
+    ) -> Tuple[
+        Optional[llm_usage.OpikUsage],
+        Optional[bool],
+        Optional[str],
+        Optional[float],
+    ]:
+        """Return the info as a tuple for backwards compatibility."""
+        return (self.usage, self.cache_hit, self.actual_provider, self.total_cost)
+
+
+def get_span_type(instance: Any) -> types.SpanType:
+    """
+    Determine the span type based on the DSPy instance type.
+
+    Args:
+        instance: A DSPy module, LM, or tool instance.
+
+    Returns:
+        The appropriate span type: "llm" for Predict/LM, "tool" for Tool,
+        or "general" for other types.
+    """
+    if isinstance(instance, dspy.Predict):
+        return "llm"
+    elif isinstance(instance, dspy.LM):
+        return "llm"
+    elif isinstance(instance, dspy.Tool):
+        return "tool"
+    return "general"
+
+
+def extract_lm_info_from_history(
+    lm_instance: Any,
+    expected_messages: Optional[Any],
+) -> LMHistoryInfo:
+    """
+    Extract token usage, cache status, actual provider, and cost from the LM's history.
+
+    DSPy stores usage information in the LM's history after each call.
+    We verify the history entry matches our expected messages to handle
+    potential race conditions with concurrent LM calls.
+
+    For routers like OpenRouter, the response contains the actual provider
+    that served the request (e.g., "Novita", "Together"), which differs from
+    the router name used in the model string (e.g., "openrouter").
+
+    The cost field is provided by providers like OpenRouter and includes
+    accurate pricing for all token types (reasoning, cache, multimodal).
+
+    Args:
+        lm_instance: The DSPy LM instance that has the history.
+        expected_messages: The expected messages to match in the history entry.
+
+    Returns:
+        LMHistoryInfo containing usage, cache_hit, actual_provider, and total_cost.
+    """
+    empty_result = LMHistoryInfo(
+        usage=None,
+        cache_hit=None,
+        actual_provider=None,
+        total_cost=None,
+    )
+
+    if not hasattr(lm_instance, "history") or not lm_instance.history:
+        return empty_result
+
+    try:
+        last_entry = lm_instance.history[-1]
+
+        # Verify we have the correct history entry by checking messages match
+        if last_entry.get("messages") != expected_messages:
+            LOGGER.debug(
+                "History entry messages don't match expected messages, "
+                "skipping usage extraction (possibly due to concurrent LM calls)"
+            )
+            return empty_result
+
+        response = last_entry.get("response")
+        usage_dict = last_entry.get("usage")
+
+        # Extract actual provider from response (useful for routers like OpenRouter)
+        # The response is a LiteLLM ModelResponse object with a 'provider' attribute
+        # when using routers like OpenRouter
+        actual_provider: Optional[str] = None
+        if response is not None and hasattr(response, "provider"):
+            actual_provider = response.provider
+
+        # Extract cost from history entry or usage dict
+        # OpenRouter and other providers return accurate cost including all token types
+        total_cost: Optional[float] = None
+        if last_entry.get("cost") is not None:
+            total_cost = last_entry.get("cost")
+        elif usage_dict and usage_dict.get("cost") is not None:
+            total_cost = usage_dict.get("cost")
+
+        # Get explicit cache_hit if set, otherwise infer from usage (empty = cached)
+        if response is None:
+            cache_hit = not usage_dict
+        elif hasattr(response, "cache_hit") and response.cache_hit is not None:
+            cache_hit = response.cache_hit
+        else:
+            # Fallback: infer from usage (empty = cached)
+            cache_hit = not usage_dict
+
+        if usage_dict:
+            usage = llm_usage.build_opik_usage_from_unknown_provider(usage_dict)
+            return LMHistoryInfo(
+                usage=usage,
+                cache_hit=cache_hit,
+                actual_provider=actual_provider,
+                total_cost=total_cost,
+            )
+        else:
+            return LMHistoryInfo(
+                usage=None,
+                cache_hit=cache_hit,
+                actual_provider=actual_provider,
+                total_cost=total_cost,
+            )
+    except Exception:
+        LOGGER.debug(
+            "Failed to extract info from DSPy LM history",
+            exc_info=True,
+        )
+        return empty_result

opik/integrations/harbor/__init__.py

@@ -0,0 +1,17 @@
+"""
+Opik integration for Harbor benchmark evaluation framework.
+
+Example:
+    >>> from opik.integrations.harbor import track_harbor
+    >>> job = Job(config)
+    >>> tracked_job = track_harbor(job)
+    >>> result = await tracked_job.run()
+
+Or enable tracking globally (for CLI usage):
+    >>> from opik.integrations.harbor import track_harbor
+    >>> track_harbor()
+"""
+
+from .opik_tracker import track_harbor, reset_harbor_tracking
+
+__all__ = ["track_harbor", "reset_harbor_tracking"]