opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/test_case.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 import dataclasses
 
 
@@ -6,5 +6,6 @@ import dataclasses
 class TestCase:
     trace_id: str
     dataset_item_id: str
-    scoring_inputs: Dict[str, Any]
     task_output: Dict[str, Any]
+    dataset_item_content: Dict[str, Any] = dataclasses.field(default_factory=dict)
+    mapped_scoring_inputs: Optional[Dict[str, Any]] = None

opik/evaluation/test_result.py

@@ -10,3 +10,4 @@ from .metrics import score_result
 class TestResult:
     test_case: test_case.TestCase
     score_results: List[score_result.ScoreResult]
+    trial_id: int

opik/evaluation/threads/evaluator.py

@@ -27,16 +27,44 @@ def evaluate_threads(
 
     Args:
         project_name: The name of the project containing the threads to evaluate.
-        filter_string: Optional filter string to select specific threads for evaluation.
+        filter_string: Optional filter string to select specific threads for evaluation using Opik Query Language (OQL).
+            The format is: "<COLUMN> <OPERATOR> <VALUE> [AND <COLUMN> <OPERATOR> <VALUE>]*"
+
+            Supported columns include:
+            - `id`, `name`, `created_by`, `thread_id`, `type`, `model`, `provider`: String fields with full operator support
+            - `status`: String field (=, contains, not_contains only)
+            - `start_time`, `end_time`: DateTime fields (use ISO 8601 format, e.g., "2024-01-01T00:00:00Z")
+            - `input`, `output`: String fields for content (=, contains, not_contains only)
+            - `metadata`: Dictionary field (use dot notation, e.g., "metadata.model")
+            - `feedback_scores`: Numeric field (use dot notation, e.g., "feedback_scores.accuracy")
+            - `tags`: List field (use "contains" operator only)
+            - `usage.total_tokens`, `usage.prompt_tokens`, `usage.completion_tokens`: Numeric usage fields
+            - `duration`, `number_of_messages`, `total_estimated_cost`: Numeric fields
+
+            Examples: 'status = "inactive"', 'id = "thread_123"', 'duration > 300'
             If None, all threads in the project will be evaluated.
         eval_project_name: Optional name for the evaluation project where evaluation traces will be stored.
             If None, the same project_name will be used.
         metrics: List of ConversationThreadMetric instances to apply to each thread.
             Must contain at least one metric.
         trace_input_transform: Function to transform trace input JSON to string representation.
-            This is used when converting traces to conversation threads.
+            This function extracts the relevant user message from your trace's input structure.
+            The function receives the raw trace input as a dictionary and should return a string.
+
+            Example: If your trace input is {"content": {"user_question": "Hello"}},
+            use: lambda x: x["content"]["user_question"]
+
+            This transformation is essential because trace inputs vary by framework, but metrics
+            expect a standardized string format representing the user's message.
         trace_output_transform: Function to transform trace output JSON to string representation.
-            This is used when converting traces to conversation threads.
+            This function extracts the relevant agent response from your trace's output structure.
+            The function receives the raw trace output as a dictionary and should return a string.
+
+            Example: If your trace output is {"response": {"text": "Hi there"}},
+            use: lambda x: x["response"]["text"]
+
+            This transformation is essential because trace outputs vary by framework, but metrics
+            expect a standardized string format representing the agent's response.
         verbose: Verbosity level for progress reporting (0=silent, 1=progress).
             Default is 1.
         num_workers: Number of concurrent workers for thread evaluation.

opik/evaluation/threads/helpers.py

@@ -4,7 +4,7 @@ from . import evaluation_result
 from ...api_objects import opik_client
 from ...api_objects.conversation import conversation_thread, conversation_factory
 from ...rest_api import TraceThread, JsonListStringPublic
-from ...types import FeedbackScoreDict
+from ...types import BatchFeedbackScoreDict
 from ...api_objects.threads import threads_client
 
 
@@ -15,7 +15,7 @@ def log_feedback_scores(
 ) -> None:
     for result in results:
         feedback_scores = [
-            FeedbackScoreDict(
+            BatchFeedbackScoreDict(
                 id=result.thread_id,
                 name=score.name,
                 value=score.value,
@@ -42,6 +42,7 @@ def load_conversation_thread(
         project_name=project_name,
         filter_string=f'thread_id = "{thread.id}"',
         max_results=max_results,
+        truncate=False,
     )
     return conversation_factory.create_conversation_from_traces(
         traces=traces,

opik/evaluation/types.py

@@ -1,5 +1,13 @@
-from typing import Any, Callable, Dict, Union
+from typing import Any, Callable, Dict, List, Union
+
+from . import test_result
+from .metrics import score_result
 
 LLMTask = Callable[[Dict[str, Any]], Dict[str, Any]]
 
 ScoringKeyMappingType = Dict[str, Union[str, Callable[[Dict[str, Any]], Any]]]
+
+ExperimentScoreFunction = Callable[
+    [List[test_result.TestResult]],
+    Union[score_result.ScoreResult, List[score_result.ScoreResult]],
+]

opik/exceptions.py

@@ -81,6 +81,23 @@ class PromptPlaceholdersDontMatchFormatArguments(OpikException):
         )
 
 
+class PromptTemplateStructureMismatch(OpikException):
+    """Exception raised when attempting to create a prompt version with a different template structure than the existing prompt."""
+
+    def __init__(
+        self, prompt_name: str, existing_structure: str, attempted_structure: str
+    ):
+        self.prompt_name = prompt_name
+        self.existing_structure = existing_structure
+        self.attempted_structure = attempted_structure
+
+    def __str__(self) -> str:
+        return (
+            f"Prompt with name '{self.prompt_name}' already exists and has immutable "
+            f"'{self.existing_structure}' template structure, not '{self.attempted_structure}'. "
+        )
+
+
 class ExperimentNotFound(OpikException):
     pass
 
@@ -130,3 +147,19 @@ class ValidationError(OpikException):
 
     def __repr__(self) -> str:
         return f"ValidationError(prefix={self._prefix}, failure_reasons={self._failure_reasons})"
+
+
+class BaseLLMError(OpikException):
+    """Base class for all LLM errors during evaluation."""
+
+    def __init__(self, message: str) -> None:
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"LLM infrastructure error: {self.message}"
+
+
+class SearchTimeoutError(OpikException):
+    """Exception raised when a search times out."""
+
+    pass

opik/file_upload/file_uploader.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from typing import Optional
 
 import httpx
@@ -28,6 +29,10 @@ def upload_attachment(
             httpx_client=upload_httpx_client,
             monitor=monitor,
         )
+
+        # delete the file after upload if requested
+        if upload_options.delete_after_upload:
+            _delete_attachment_file(upload_options.file_path)
     except Exception as e:
         LOGGER.error(
             "Failed to upload attachment: '%s' from file: [%s] with size: [%s]. Error: %s",
@@ -40,6 +45,14 @@ def upload_attachment(
         raise
 
 
+def _delete_attachment_file(file_path: str) -> None:
+    try:
+        os.unlink(file_path)
+    except OSError as e:
+        LOGGER.info(f"Failed to delete attachment file: '{file_path}'. Reason: {e}.")
+        pass
+
+
 def _do_upload_attachment(
     upload_options: file_upload_options.FileUploadOptions,
     rest_client: rest_api_client.OpikApi,

opik/file_upload/upload_options.py

@@ -16,6 +16,7 @@ class FileUploadOptions:
     entity_id: str
     project_name: str
     encoded_url_override: str
+    delete_after_upload: bool
 
 
 def file_upload_options_from_attachment(
@@ -32,4 +33,5 @@ def file_upload_options_from_attachment(
         entity_id=attachment.entity_id,
         project_name=attachment.project_name,
         encoded_url_override=attachment.encoded_url_override,
+        delete_after_upload=attachment.delete_after_upload,
     )

opik/hooks/__init__.py

@@ -0,0 +1,23 @@
+from .httpx_client_hook import (
+    HttpxClientHook,
+    add_httpx_client_hook,
+    register_httpx_client_hook,
+)
+from .anonymizer_hook import (
+    has_anonymizers,
+    add_anonymizer,
+    apply_anonymizers,
+    get_anonymizers,
+    clear_anonymizers,
+)
+
+__all__ = (
+    "HttpxClientHook",
+    "add_httpx_client_hook",
+    "register_httpx_client_hook",
+    "add_anonymizer",
+    "apply_anonymizers",
+    "clear_anonymizers",
+    "get_anonymizers",
+    "has_anonymizers",
+)

opik/hooks/anonymizer_hook.py

@@ -0,0 +1,36 @@
+from typing import List
+
+from opik.anonymizer import anonymizer
+
+
+# holder for a global list of anonymizers
+_anonymizers: List[anonymizer.Anonymizer] = []
+
+
+def add_anonymizer(anonymizer_hook: anonymizer.Anonymizer) -> None:
+    """Register a new anonymizer to be applied to all sensitive data logged by Opik."""
+    _anonymizers.append(anonymizer_hook)
+
+
+def clear_anonymizers() -> None:
+    """Clear all registered anonymizers."""
+    _anonymizers.clear()
+
+
+def has_anonymizers() -> bool:
+    """Check if any anonymizers have been registered."""
+    return len(_anonymizers) > 0
+
+
+def get_anonymizers() -> List[anonymizer.Anonymizer]:
+    """Get a list of all registered anonymizers."""
+    return _anonymizers
+
+
+def apply_anonymizers(
+    data: anonymizer.AnonymizerDataType,
+) -> anonymizer.AnonymizerDataType:
+    """Apply all registered anonymizers to the given data."""
+    for anonymizer_ in _anonymizers:
+        data = anonymizer_.anonymize(data)
+    return data

opik/hooks/httpx_client_hook.py

@@ -0,0 +1,112 @@
+import logging
+
+import httpx
+from typing import Any, Callable, List, Optional, Dict
+
+
+_deprecated_httpx_client_hooks: List[Callable[[httpx.Client], httpx.Client]] = []
+
+# holder for the global httpx client hook
+_httpx_client_hooks: List["HttpxClientHook"] = []
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class HttpxClientHook:
+    def __init__(
+        self,
+        client_modifier: Optional[Callable[[httpx.Client], None]],
+        client_init_arguments: Optional[Dict[str, Any]],
+    ) -> None:
+        """Provides a means to customize an `httpx.Client` instance used by Opik.
+
+        This class allows users to attach a callable hook to modify or interact
+        with an `httpx.Client` instance and pass initialization arguments to
+        create customized client configurations. The primary usage is to allow
+        pre-processing or setup of HTTP clients used in a broader application.
+
+        Args:
+            client_modifier: Optional callable that accepts an `httpx.Client` instance and
+                returns a modified httpx.Client instance.
+            client_init_arguments: Dictionary containing additional `httpx.Client`
+                initialization arguments to be passed to the default `httpx.Client`.
+        """
+        self._hook = client_modifier
+        self._httpx_client_arguments = client_init_arguments
+
+    def update_init_arguments(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        if self._httpx_client_arguments is not None:
+            kwargs.update(self._httpx_client_arguments)
+
+        return kwargs
+
+    def __call__(self, client: httpx.Client) -> None:
+        if self._hook is not None:
+            self._hook(client)
+
+
+def add_httpx_client_hook(hook: HttpxClientHook) -> None:
+    """
+    Adds an HttpxClientHook to the list of hooks to be used with HTTPX clients.
+
+    Injects a new hook into the global list of HTTPX client hooks, allowing for
+    custom behavior or additional functionality when making requests with
+    an HTTPX client.
+
+    Args:
+        hook (HttpxClientHook): A callable to be added as an HTTPX client hook.
+
+    """
+    global _httpx_client_hooks
+    _httpx_client_hooks.append(hook)
+
+
+def build_init_arguments(default_kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Modifies and returns initialization arguments by applying pre-defined hooks.
+
+    This function iterates through a collection of hooks and applies their logic to
+    update the initialization arguments provided.
+
+    Args:
+        default_kwargs: A dictionary containing default initialization
+            arguments.
+
+    Returns:
+        Dict[str, Any]: The modified dictionary of initialization arguments.
+    """
+    for hook in _httpx_client_hooks:
+        default_kwargs = hook.update_init_arguments(default_kwargs)
+
+    return default_kwargs
+
+
+def apply_httpx_client_hooks(client: httpx.Client) -> None:
+    """Applies registered httpx client hooks."""
+    for hook in _httpx_client_hooks:
+        hook(client)
+
+    # apply deprecated hooks if any
+    for deprecated_hook in _deprecated_httpx_client_hooks:
+        deprecated_hook(client)
+
+
+def register_httpx_client_hook(hook: Callable[[httpx.Client], httpx.Client]) -> None:
+    """
+    Deprecated: This method is deprecated and will be removed in a future release. Please use `add_httpx_client_hook` instead.
+
+    Registers a hook for the customization of `httpx.Client` instances. The provided
+    hook function will be invoked with an `httpx.Client` instance and is expected
+    to return a customized `httpx.Client`.
+
+
+    Args:
+        hook: A callable that takes an `httpx.Client` instance and returns a
+              customized `httpx.Client`.
+    """
+    _deprecated_httpx_client_hooks.append(hook)
+
+    LOGGER.warning(
+        "register_httpx_client_hook is deprecated and will be removed in a future release. Please use add_httpx_client_hook instead."
+    )

opik/httpx_client.py

@@ -40,19 +40,22 @@ def get(
         pool=POOL_TIMEOUT_SECONDS,
     )
 
-    client = OpikHttpxClient(
-        compress_json_requests=compress_json_requests,
-        limits=limits,
-        verify=verify,
-        timeout=timeout,
-        follow_redirects=True,
-        proxy=proxy,
-    )
+    # build HTTPX client arguments
+    kwargs = {
+        "limits": limits,
+        "verify": verify,
+        "timeout": timeout,
+        "follow_redirects": True,
+        "proxy": proxy,
+    }
+    kwargs = hooks.httpx_client_hook.build_init_arguments(kwargs)
+
+    client = OpikHttpxClient(compress_json_requests=compress_json_requests, **kwargs)
 
     headers = _prepare_headers(workspace=workspace, api_key=api_key)
     client.headers.update(headers)
 
-    hooks.run_httpx_client_hooks(client)
+    hooks.httpx_client_hook.apply_httpx_client_hooks(client)
 
     return client
 

opik/id_helpers.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 from typing import Optional
+import random
+import string
 import uuid
 import uuid6
 
@@ -12,6 +14,22 @@ def generate_id(timestamp: Optional[datetime] = None) -> str:
     return str(uuid6.uuid7())
 
 
+def generate_random_alphanumeric_string(length: int) -> str:
+    """Generate a random alphanumeric string of the specified length.
+
+    Args:
+        length: The length of the string to generate.
+
+    Returns:
+        A random string containing only alphanumeric characters (a-z, A-Z, 0-9).
+    """
+    if length < 0:
+        raise ValueError("Length must be non-negative")
+
+    characters = string.ascii_letters + string.digits
+    return "".join(random.choice(characters) for _ in range(length))
+
+
 def uuid4_to_uuid7(user_datetime: datetime, user_uuid: str) -> uuid.UUID:
     """Convert a UUID v4 into a UUID v7 following RFC draft specification."""
     # Get Unix timestamp in milliseconds

opik/integrations/adk/graph/subgraph_edges_builders.py

@@ -1,6 +1,5 @@
 from typing import List, Optional
 from . import nodes
-import itertools
 
 
 def build_edge_definitions_for_parallel_subagents(
@@ -16,7 +15,7 @@ def build_edge_definitions_for_sequential_subagents(
         return [f"{children[0].name}"]
 
     result: List[str] = []
-    for current, next in itertools.pairwise(children):
+    for current, next in zip(children, children[1:]):
         edge_definition = f"{current.name} ==> {next.name}"
         result.append(edge_definition)
 

opik/integrations/adk/helpers.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from typing import (
     Any,
@@ -8,6 +9,8 @@ from google.adk.models import LlmResponse
 import opik.types as opik_types
 import pydantic
 
+LOGGER = logging.getLogger(__name__)
+
 
 def convert_adk_base_model_to_dict(value: pydantic.BaseModel) -> Dict[str, Any]:
     """Most ADK objects are Pydantic Base Models"""
@@ -27,13 +30,19 @@ def get_adk_provider() -> opik_types.LLMProvider:
 
 
 def has_empty_text_part_content(llm_response: LlmResponse) -> bool:
-    if llm_response.content is None or len(llm_response.content.parts) == 0:
-        return True
+    try:
+        if llm_response.content is None:
+            return True
 
-    # to filter out something like this: {"candidates":[{"content":{"parts":[{"text":""}],"role":"model"}}],...}}
-    if len(llm_response.content.parts) == 1:
-        part = llm_response.content.parts[0]
-        if part.text is not None and len(part.text) == 0:
+        if not llm_response.content.parts:
             return True
 
-    return False
+        # to filter out something like this: {"candidates":[{"content":{"parts":[{"text":""}],"role":"model"}}],...}}
+        if len(llm_response.content.parts) == 1:
+            part = llm_response.content.parts[0]
+            if part.text is not None and len(part.text) == 0:
+                return True
+        return False
+    except Exception as e:
+        LOGGER.warning(f"Exception in has_empty_text_part_content {e}", exc_info=True)
+        return True

opik/integrations/adk/legacy_opik_tracer.py

@@ -8,6 +8,7 @@ from google.adk import models
 from google.adk.tools import base_tool
 from google.adk.tools import tool_context
 
+import opik
 from opik import context_storage
 from opik.decorator import arguments_helpers, span_creation_handler
 from opik.api_objects import opik_client, span, trace
@@ -77,7 +78,8 @@ class LegacyOpikTracer:
         trace_data = self._context_storage.pop_trace_data()
         assert trace_data is not None
         trace_data.init_end_time()
-        self._opik_client.trace(**trace_data.as_parameters)
+        if opik.is_tracing_active():
+            self._opik_client.trace(**trace_data.as_parameters)
 
     def _end_current_span(
         self,
@@ -85,20 +87,21 @@ class LegacyOpikTracer:
         span_data = self._context_storage.pop_span_data()
         assert span_data is not None
         span_data.init_end_time()
-        self._opik_client.span(**span_data.as_parameters)
+        if opik.is_tracing_active():
+            self._opik_client.span(**span_data.as_parameters)
 
     def _start_span(self, span_data: span.SpanData) -> None:
         self._context_storage.add_span_data(span_data)
         self._opik_created_spans.add(span_data.id)
 
-        if self._opik_client.config.log_start_trace_span:
+        if self._opik_client.config.log_start_trace_span and opik.is_tracing_active():
             self._opik_client.span(**span_data.as_start_parameters)
 
     def _start_trace(self, trace_data: trace.TraceData) -> None:
         self._context_storage.set_trace_data(trace_data)
         self._current_trace_created_by_opik_tracer.set(trace_data.id)
 
-        if self._opik_client.config.log_start_trace_span:
+        if self._opik_client.config.log_start_trace_span and opik.is_tracing_active():
             self._opik_client.trace(**trace_data.as_start_parameters)
 
     def _set_current_context_data(self, value: SpanOrTraceData) -> None:

opik/integrations/adk/opik_tracer.py

@@ -7,6 +7,7 @@ from google.adk import models
 from google.adk.tools import base_tool
 from google.adk.tools import tool_context
 
+import opik
 from opik import context_storage
 from opik.api_objects import opik_client, span, trace
 from opik.types import DistributedTraceHeadersDict
@@ -253,7 +254,8 @@ class OpikTracer:
             current_span.init_end_time()
             # We close this span manually because otherwise ADK will close it too late,
             # and it will also add tool spans inside of it, which we want to avoid.
-            self._opik_client.span(**current_span.as_parameters)
+            if opik.is_tracing_active():
+                self._opik_client.span(**current_span.as_parameters)
             self._last_model_output = output
 
         except Exception as e:
@@ -288,6 +290,7 @@ class OpikTracer:
                 LOGGER.warning(
                     f"No current span found in context for tool: {tool.name}"
                 )
+                _log_tool_context_warning(context=tool_context)
 
         except Exception as e:
             LOGGER.error(f"Failed during before_tool_callback(): {e}", exc_info=True)
@@ -321,6 +324,7 @@ class OpikTracer:
                 LOGGER.warning(
                     f"No current span found in context for tool output update: {tool.name}"
                 )
+                _log_tool_context_warning(context=tool_context)
         except Exception as e:
             LOGGER.error(f"Failed during after_tool_callback(): {e}", exc_info=True)
 
@@ -355,3 +359,12 @@ def _try_add_agent_graph_to_metadata(
         }
     except Exception:
         LOGGER.error("Failed to build mermaid graph for agent.", exc_info=True)
+
+
+def _log_tool_context_warning(context: tool_context.ToolContext) -> None:
+    if context is not None:
+        warning = f"Function call id: {context.function_call_id}, agent name: {context.agent_name}"
+        if context.actions is not None:
+            warning += f", is escalate: {context.actions.escalate}, transfer to: {context.actions.transfer_to_agent}"
+
+        LOGGER.warning(warning)

opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py

@@ -2,6 +2,7 @@ import logging
 from typing import Iterator, Optional, Tuple
 
 import opentelemetry.trace
+import opik
 import opik.context_storage
 from opik.api_objects import trace, span
 from opik.decorator import (
@@ -100,7 +101,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
             # so we manually finalize it here to avoid incorrect span nesting.
             opik.context_storage.pop_span_data(ensure_id=current_span_data.id)
             current_span_data.init_end_time()
-            self.opik_client.span(**current_span_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.span(**current_span_data.as_parameters)
             current_span_data = opik.context_storage.top_span_data()
 
         try:
@@ -145,7 +147,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
         trace_data = opik.context_storage.pop_trace_data(ensure_id=trace_id)
         if trace_data is not None:
             trace_data.init_end_time()
-            self.opik_client.trace(**trace_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.trace(**trace_data.as_parameters)
 
     def _ensure_span_is_finalized(self, span_id: str) -> None:
         opik.context_storage.trim_span_data_stack_to_certain_span(span_id)
@@ -153,7 +156,8 @@ class OpikADKOtelTracer(opentelemetry.trace.NoOpTracer):
         span_data = opik.context_storage.pop_span_data(ensure_id=span_id)
         if span_data is not None:
             span_data.init_end_time()
-            self.opik_client.span(**span_data.as_parameters)
+            if opik.is_tracing_active():
+                self.opik_client.span(**span_data.as_parameters)
 
 
 def _prepare_trace_and_span_to_be_finalized(

opik/integrations/adk/recursive_callback_injector.py

@@ -2,7 +2,6 @@ import types
 from typing import TypeVar, List, Any, Set
 from . import opik_tracer
 import logging
-from opik import _logging
 
 from google.adk.tools import agent_tool
 from google.adk import agents
@@ -46,7 +45,9 @@ class RecursiveCallbackInjector:
                 callbacks=current_callback_value
             ):
                 current_callback_value.append(callback_func)
-            elif not self._is_opik_callback_function(current_callback_value):
+            elif not self._is_opik_callback_function(
+                current_callback_value
+            ) and callable(current_callback_value):
                 setattr(
                     agent, callback_field_name, [current_callback_value, callback_func]
                 )
@@ -118,11 +119,7 @@ def track_adk_agent_recursive(
     Returns:
         The modified root agent with tracking enabled
     """
-    _logging.log_once_at_level(
-        logging.INFO,
-        "`track_adk_agent_recursive` is experimental feature. Please let us know if something is not working as expected: https://github.com/comet-ml/opik/issues",
-        logger=LOGGER,
-    )
+
     recursive_callback_injector = RecursiveCallbackInjector(tracer)
     recursive_callback_injector.inject(root_agent)