opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/rest_client_configurator/api.py

@@ -8,6 +8,7 @@ def configure(rest_client: rest_api.OpikApi) -> None:
 
 def _configure_retries(rest_client: rest_api.OpikApi) -> None:
     domain_client_names = [
+        "attachments",
         "datasets",
         "experiments",
         "traces",

opik/rest_client_configurator/retry_decorator.py

@@ -29,4 +29,5 @@ opik_rest_retry = tenacity.retry(
     stop=tenacity.stop_after_attempt(3),
     wait=tenacity.wait_exponential(multiplier=3, min=5, max=15),  # 5, 6, 12
     retry=tenacity.retry_if_exception(_allowed_to_retry),
+    reraise=True,
 )

opik/s3_httpx_client.py

@@ -0,0 +1,67 @@
+import functools
+
+import httpx
+import tenacity
+
+from opik import hooks
+
+KEEPALIVE_EXPIRY_SECONDS = 10
+CONNECT_TIMEOUT_SECONDS = 20
+READ_TIMEOUT_SECONDS = 100
+WRITE_TIMEOUT_SECONDS = 100
+POOL_TIMEOUT_SECONDS = 20
+
+RETRYABLE_STATUS_CODES = [500]
+
+
+@functools.lru_cache
+def get_cached() -> httpx.Client:
+    return get()
+
+
+def get() -> httpx.Client:
+    limits = httpx.Limits(keepalive_expiry=KEEPALIVE_EXPIRY_SECONDS)
+
+    timeout = httpx.Timeout(
+        connect=CONNECT_TIMEOUT_SECONDS,
+        read=READ_TIMEOUT_SECONDS,
+        write=WRITE_TIMEOUT_SECONDS,
+        pool=POOL_TIMEOUT_SECONDS,
+    )
+
+    # build HTTPX client arguments
+    kwargs = {
+        "limits": limits,
+        "timeout": timeout,
+    }
+    kwargs = hooks.httpx_client_hook.build_init_arguments(kwargs)
+
+    client = httpx.Client(**kwargs)
+
+    hooks.httpx_client_hook.apply_httpx_client_hooks(client)
+
+    return client
+
+
+def _allowed_to_retry(exception: Exception) -> bool:
+    if isinstance(
+        exception,
+        (
+            httpx.ConnectError,
+            httpx.TimeoutException,
+        ),
+    ):
+        return True
+
+    if isinstance(exception, httpx.HTTPStatusError):
+        if exception.response.status_code in RETRYABLE_STATUS_CODES:
+            return True
+
+    return False
+
+
+s3_retry = tenacity.retry(
+    stop=tenacity.stop_after_attempt(3),
+    wait=tenacity.wait_exponential(multiplier=5, min=10, max=45),
+    retry=tenacity.retry_if_exception(_allowed_to_retry),
+)

opik/simulation/__init__.py

@@ -0,0 +1,6 @@
+"""Multi-turn simulation module for Opik."""
+
+from .simulated_user import SimulatedUser
+from .simulator import run_simulation
+
+__all__ = ["SimulatedUser", "run_simulation"]

opik/simulation/simulated_user.py

@@ -0,0 +1,99 @@
+"""SimulatedUser class for multi-turn conversation simulation."""
+
+from typing import List, Dict, Optional
+from opik.evaluation.models.models_factory import get as get_model
+
+
+class SimulatedUser:
+    """
+    A simulated user that generates responses using LLMs or fixed responses.
+
+    The user simulator generates string responses that are then incorporated
+    into the conversation by the application logic.
+    """
+
+    def __init__(
+        self,
+        persona: str,
+        model: str = "gpt-4o-mini",
+        fixed_responses: Optional[List[str]] = None,
+    ):
+        """
+        Initialize a simulated user.
+
+        Args:
+            persona: Description of the user's personality and behavior
+            model: LLM model to use for generating responses (default: gpt-4o-mini)
+            fixed_responses: Optional list of predefined responses to cycle through
+        """
+        self.persona = persona
+        self.model = model
+        self.fixed_responses = fixed_responses or []
+        self._response_index = 0
+
+        # Initialize LLM backend using models_factory for consistency
+        self._llm = get_model(model_name=model)
+
+    def generate_response(self, conversation_history: List[Dict[str, str]]) -> str:
+        """
+        Generate a response based on the conversation history.
+
+        Args:
+            conversation_history: List of message dicts with 'role' and 'content' keys
+
+        Returns:
+            String response from the simulated user
+        """
+        # Use fixed responses first if available
+        if self.fixed_responses:
+            response = self.fixed_responses[
+                self._response_index % len(self.fixed_responses)
+            ]
+            self._response_index += 1
+            return response
+
+        # Generate response using LLM
+        return self._generate_llm_response(conversation_history)
+
+    def _generate_llm_response(self, conversation_history: List[Dict[str, str]]) -> str:
+        """Generate response using the LLM backend."""
+        # Build system prompt with persona and clear instructions
+        system_prompt = f"""You are a simulated user with the following persona: {self.persona}
+
+Your task is to generate realistic user messages that this persona would send in a conversation.
+Respond as if you are the user, not as an assistant describing the user.
+Generate a single user message that fits your persona and the conversation context."""
+
+        # Convert conversation history to messages format expected by LLM
+        messages = [{"role": "system", "content": system_prompt}]
+
+        # Add all conversation history
+        messages.extend(conversation_history)
+
+        # Convert messages to string format for generate_string
+        conversation_text = self._format_messages_as_text(messages)
+
+        # Generate response
+        try:
+            response = self._llm.generate_string(input=conversation_text)
+            return response
+        except Exception as e:
+            # Fallback response if LLM fails
+            return f"I'm having trouble responding right now. ({str(e)})"
+
+    def _format_messages_as_text(self, messages: List[Dict[str, str]]) -> str:
+        """Convert message list to text format for LLM input."""
+        formatted_messages = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if role == "system":
+                formatted_messages.append(f"System: {content}")
+            elif role == "user":
+                formatted_messages.append(f"User: {content}")
+            elif role == "assistant":
+                formatted_messages.append(f"Assistant: {content}")
+            else:
+                formatted_messages.append(f"{role.title()}: {content}")
+
+        return "\n".join(formatted_messages)

opik/simulation/simulator.py

@@ -0,0 +1,108 @@
+"""Multi-turn simulation functionality."""
+
+from typing import Callable, Optional, Dict, Any, List
+from opik import id_helpers, track
+from .simulated_user import SimulatedUser
+
+
+def run_simulation(
+    app: Callable,
+    user_simulator: SimulatedUser,
+    initial_message: Optional[str] = None,
+    max_turns: int = 5,
+    thread_id: Optional[str] = None,
+    project_name: Optional[str] = None,
+    **app_kwargs: Any,
+) -> Dict[str, Any]:
+    """
+    Run a multi-turn conversation simulation between a simulated user and an app.
+
+    1. The simulator passes single message strings to the app
+    2. The app manages full conversation history internally using thread_id
+    3. The app logs traces with thread_id for evaluation
+
+    Args:
+        app: Callable that processes messages and manages conversation history internally.
+            Signature: app(message: str, *, thread_id: str, **kwargs) -> Dict[str, str]
+            The app is automatically decorated with @track and thread_id is injected via opik_args.
+        user_simulator: SimulatedUser instance that generates user responses
+        initial_message: Optional initial message from the user. If None, generated by simulator
+        max_turns: Maximum number of conversation turns (default: 5)
+        thread_id: Optional thread ID for grouping traces. Generated if not provided
+        project_name: Optional project name for trace logging
+        **app_kwargs: Additional keyword arguments passed to the app
+
+    Returns:
+        Dict containing:
+        - thread_id: The thread ID used for this simulation
+        - conversation_history: List of message dicts from the simulation
+        - project_name: Project name if provided
+    """
+    # Generate thread_id if not provided
+    if thread_id is None:
+        thread_id = id_helpers.generate_id()
+
+    # Automatically decorate app if not already decorated
+    if not hasattr(app, "opik_tracked"):
+        app_name = app.__name__ if hasattr(app, "__name__") else "simulation_app"
+        app = track(name=app_name)(app)
+
+    # Track conversation for simulator (app manages its own history internally)
+    conversation_history: List[Dict[str, str]] = []
+
+    # Generate initial message if needed
+    if initial_message is None:
+        initial_message = user_simulator.generate_response(conversation_history)
+
+    # Simulation loop
+    for turn in range(max_turns):
+        # Get user message
+        if turn == 0:
+            user_message_text = initial_message
+        else:
+            user_message_text = user_simulator.generate_response(conversation_history)
+
+        # Create message dict for tracking
+        user_message = {"role": "user", "content": user_message_text}
+        conversation_history.append(user_message)
+
+        # Call app with SINGLE message string, thread_id parameter, and opik_args for tracing
+        try:
+            assistant_message = app(
+                user_message_text,
+                thread_id=thread_id,
+                **app_kwargs,
+                opik_args={
+                    "trace": {
+                        "thread_id": thread_id,
+                        "metadata": {"turn": turn + 1, "project_name": project_name},
+                    }
+                },
+            )
+        except Exception as e:
+            # Handle app errors gracefully
+            assistant_message = {
+                "role": "assistant",
+                "content": f"Error processing message: {str(e)}",
+            }
+
+        # Validate assistant message format
+        if (
+            not isinstance(assistant_message, dict)
+            or "role" not in assistant_message
+            or "content" not in assistant_message
+        ):
+            assistant_message = {
+                "role": "assistant",
+                "content": str(assistant_message)
+                if assistant_message
+                else "No response",
+            }
+
+        conversation_history.append(assistant_message)
+
+    return {
+        "thread_id": thread_id,
+        "conversation_history": conversation_history,
+        "project_name": project_name,
+    }

opik/synchronization.py

@@ -1,6 +1,6 @@
 import time
 import logging
-from typing import Callable, Optional, Any
+from typing import Callable, Optional
 
 
 LOGGER = logging.getLogger(__name__)
@@ -12,7 +12,9 @@ def wait_for_done(
     progress_callback: Optional[Callable] = None,
     sleep_time: float = 1,
 ) -> None:
-    """Wait up to TIMEOUT seconds for the check function to return True"""
+    """
+    Wait up to TIMEOUT seconds for the check function to return True
+    """
     end_time = time.time() + timeout if timeout else float("inf")
     while check_function() is False and time.time() < end_time:
         if progress_callback is not None:
@@ -31,36 +33,21 @@ def until(
     max_try_seconds: float = 10,
     allow_errors: bool = False,
 ) -> bool:
+    """
+    Call a function until it returns True or max_try_seconds is reached.
+    """
     start_time = time.time()
     while True:
         try:
             if function():
-                break
+                return True
         except Exception:
             LOGGER.debug(
                 f"{function.__name__} raised error in 'until' function.", exc_info=True
             )
             if not allow_errors:
                 raise
-        finally:
-            if (time.time() - start_time) > max_try_seconds:
-                return False
-            time.sleep(sleep)
-    return True
-
 
-def try_get_until(
-    function: Callable[[], Any], sleep: float = 0.5, max_try_seconds: float = 10
-) -> Any:
-    """
-    As soon
-    """
-    start_time = time.time()
-    while True:
-        try:
-            return function()
-        except Exception:
-            if (time.time() - start_time) > max_try_seconds:
-                raise
-            time.sleep(sleep)
-    return True
+        if (time.time() - start_time) > max_try_seconds:
+            return False
+        time.sleep(sleep)

opik/tracing_runtime_config.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import threading
+from typing import Optional
+from . import config
+
+
+class TracingRuntimeConfig:
+    def __init__(self) -> None:
+        self._lock = threading.RLock()
+        self._tracing_active: Optional[bool] = None
+
+    def set_tracing_active(self, active: bool) -> None:
+        with self._lock:
+            self._tracing_active = active
+
+    def reset_to_config_default(self) -> None:
+        with self._lock:
+            self._tracing_active = None
+
+    def is_tracing_active(self) -> bool:
+        with self._lock:
+            if self._tracing_active is not None:
+                return self._tracing_active
+
+        try:
+            enabled = not config.OpikConfig().track_disable
+            self._tracing_active = enabled
+
+        except Exception:
+            return True
+
+        return enabled
+
+
+runtime_config = TracingRuntimeConfig()
+
+
+def set_tracing_active(active: bool) -> None:
+    runtime_config.set_tracing_active(active)
+
+
+def is_tracing_active() -> bool:
+    return runtime_config.is_tracing_active()
+
+
+def reset_tracing_to_config_default() -> None:
+    runtime_config.reset_to_config_default()

opik/types.py

@@ -2,6 +2,7 @@ import enum
 import sys
 from typing import Literal, Optional
 
+from pydantic import StrictStr
 from typing_extensions import TypedDict
 
 if sys.version_info < (3, 11):
@@ -9,9 +10,10 @@ if sys.version_info < (3, 11):
 else:
     from typing import NotRequired, Required
 
-SpanType = Literal["general", "tool", "llm"]
+SpanType = Literal["general", "tool", "llm", "guardrail"]
 FeedbackType = Literal["numerical", "categorical"]
 CreatedByType = Literal["evaluation"]
+AttachmentEntityType = Literal["trace", "span"]
 
 
 class LLMProvider(str, enum.Enum):
@@ -27,6 +29,15 @@ class LLMProvider(str, enum.Enum):
     ANTHROPIC = "anthropic"
     """Used for models hosted by Anthropic. https://www.anthropic.com"""
 
+    ANTHROPIC_VERTEXAI = "anthropic_vertexai"
+    """Used for Anthropic models hosted by VertexAI. https://cloud.google.com/vertex-ai"""
+
+    GROQ = "groq"
+    """Used for models hosted by Groq. https://groq.com"""
+
+    BEDROCK = "bedrock"
+    """Used for models hosted by AWS Bedrock. https://aws.amazon.com/bedrock"""
+
     @classmethod
     def has_value(cls, value: str) -> bool:
         return value in [enum_item.value for enum_item in cls]
@@ -53,7 +64,7 @@ class FeedbackScoreDict(TypedDict):
     id: NotRequired[str]
     """
     A unique identifier for the object this score should be assigned to.
-    Refers to either the trace_id or span_id depending on how the score is logged.
+    Refers to either the trace_id, span_id or thread_id depending on how the score is logged.
     """
 
     name: Required[str]
@@ -69,6 +80,41 @@ class FeedbackScoreDict(TypedDict):
     """An optional explanation or justification for the given score."""
 
 
+class BatchFeedbackScoreDict(TypedDict):
+    """
+    A TypedDict representing a feedback score for batch operations.
+
+    This class defines the structure for feedback scores used in batch logging
+    operations, with a required id field and optional per-score project_name.
+    """
+
+    id: Required[str]
+    """
+    A unique identifier for the object this score should be assigned to.
+    Refers to either the trace_id, span_id or thread_id depending on how the score is logged.
+    Required for batch operations.
+    """
+
+    name: Required[str]
+    """The name of the feedback metric or criterion."""
+
+    value: Required[float]
+    """The numerical value of the feedback score."""
+
+    project_name: NotRequired[Optional[StrictStr]]
+    """
+    The name of the project for this specific score.
+    If not provided, falls back to the project_name parameter in the method call,
+    or the default project name configured in the Opik instance.
+    """
+
+    category_name: NotRequired[Optional[str]]
+    """An optional category name for the given score."""
+
+    reason: NotRequired[Optional[str]]
+    """An optional explanation or justification for the given score."""
+
+
 class ErrorInfoDict(TypedDict):
     """
     A TypedDict representing the information about the error occurred.

opik/url_helpers.py

@@ -16,6 +16,10 @@ HEALTH_CHECK_URL_POSTFIX: Final[str] = "/is-alive/ping"
 ALLOWED_URL_CHARACTERS: Final[str] = ":/&?="
 
 
+def ensure_ending_slash(url: str) -> str:
+    return url.rstrip("/") + "/"
+
+
 def get_ui_url() -> str:
     config = opik.config.OpikConfig()
     opik_url_override = config.url_override
@@ -32,7 +36,7 @@ def get_experiment_url_by_id(
         f"v1/session/redirect/experiments/?experiment_id={experiment_id}&dataset_id={dataset_id}&path={encoded_opik_url}",
         safe=ALLOWED_URL_CHARACTERS,
     )
-    return urllib.parse.urljoin(url_override, project_path)
+    return urllib.parse.urljoin(ensure_ending_slash(url_override), project_path)
 
 
 def get_project_url_by_workspace(
@@ -53,7 +57,7 @@ def get_project_url_by_trace_id(trace_id: str, url_override: str) -> str:
         f"v1/session/redirect/projects/?trace_id={trace_id}&path={encoded_opik_url}",
         safe=ALLOWED_URL_CHARACTERS,
     )
-    return urllib.parse.urljoin(url_override, project_path)
+    return urllib.parse.urljoin(ensure_ending_slash(url_override), project_path)
 
 
 def get_dataset_url_by_id(dataset_id: str, url_override: str) -> str:
@@ -63,7 +67,7 @@ def get_dataset_url_by_id(dataset_id: str, url_override: str) -> str:
         f"v1/session/redirect/datasets/?dataset_id={dataset_id}&path={encoded_opik_url}",
         safe=ALLOWED_URL_CHARACTERS,
     )
-    return urllib.parse.urljoin(url_override, project_path)
+    return urllib.parse.urljoin(ensure_ending_slash(url_override), project_path)
 
 
 def get_base_url(url: str) -> str:
@@ -83,3 +87,9 @@ def get_workspace_list_url(base_url: str) -> str:
 
 def get_is_alive_ping_url(base_url: str) -> str:
     return urllib.parse.urljoin(base_url, HEALTH_CHECK_URL_POSTFIX)
+
+
+def is_aws_presigned_url(url: str) -> bool:
+    return "X-Amz-Signature" in url or (
+        "Signature=" in url and "AWSAccessKeyId=" in url
+    )