opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/simulation/__init__.py

@@ -0,0 +1,6 @@
+"""Multi-turn simulation module for Opik."""
+
+from .simulated_user import SimulatedUser
+from .simulator import run_simulation
+
+__all__ = ["SimulatedUser", "run_simulation"]

opik/simulation/simulated_user.py

@@ -0,0 +1,99 @@
+"""SimulatedUser class for multi-turn conversation simulation."""
+
+from typing import List, Dict, Optional
+from opik.evaluation.models.models_factory import get as get_model
+
+
+class SimulatedUser:
+    """
+    A simulated user that generates responses using LLMs or fixed responses.
+
+    The user simulator generates string responses that are then incorporated
+    into the conversation by the application logic.
+    """
+
+    def __init__(
+        self,
+        persona: str,
+        model: str = "gpt-4o-mini",
+        fixed_responses: Optional[List[str]] = None,
+    ):
+        """
+        Initialize a simulated user.
+
+        Args:
+            persona: Description of the user's personality and behavior
+            model: LLM model to use for generating responses (default: gpt-4o-mini)
+            fixed_responses: Optional list of predefined responses to cycle through
+        """
+        self.persona = persona
+        self.model = model
+        self.fixed_responses = fixed_responses or []
+        self._response_index = 0
+
+        # Initialize LLM backend using models_factory for consistency
+        self._llm = get_model(model_name=model)
+
+    def generate_response(self, conversation_history: List[Dict[str, str]]) -> str:
+        """
+        Generate a response based on the conversation history.
+
+        Args:
+            conversation_history: List of message dicts with 'role' and 'content' keys
+
+        Returns:
+            String response from the simulated user
+        """
+        # Use fixed responses first if available
+        if self.fixed_responses:
+            response = self.fixed_responses[
+                self._response_index % len(self.fixed_responses)
+            ]
+            self._response_index += 1
+            return response
+
+        # Generate response using LLM
+        return self._generate_llm_response(conversation_history)
+
+    def _generate_llm_response(self, conversation_history: List[Dict[str, str]]) -> str:
+        """Generate response using the LLM backend."""
+        # Build system prompt with persona and clear instructions
+        system_prompt = f"""You are a simulated user with the following persona: {self.persona}
+
+Your task is to generate realistic user messages that this persona would send in a conversation.
+Respond as if you are the user, not as an assistant describing the user.
+Generate a single user message that fits your persona and the conversation context."""
+
+        # Convert conversation history to messages format expected by LLM
+        messages = [{"role": "system", "content": system_prompt}]
+
+        # Add all conversation history
+        messages.extend(conversation_history)
+
+        # Convert messages to string format for generate_string
+        conversation_text = self._format_messages_as_text(messages)
+
+        # Generate response
+        try:
+            response = self._llm.generate_string(input=conversation_text)
+            return response
+        except Exception as e:
+            # Fallback response if LLM fails
+            return f"I'm having trouble responding right now. ({str(e)})"
+
+    def _format_messages_as_text(self, messages: List[Dict[str, str]]) -> str:
+        """Convert message list to text format for LLM input."""
+        formatted_messages = []
+        for message in messages:
+            role = message["role"]
+            content = message["content"]
+            if role == "system":
+                formatted_messages.append(f"System: {content}")
+            elif role == "user":
+                formatted_messages.append(f"User: {content}")
+            elif role == "assistant":
+                formatted_messages.append(f"Assistant: {content}")
+            else:
+                formatted_messages.append(f"{role.title()}: {content}")
+
+        return "\n".join(formatted_messages)

opik/simulation/simulator.py

@@ -0,0 +1,108 @@
+"""Multi-turn simulation functionality."""
+
+from typing import Callable, Optional, Dict, Any, List
+from opik import id_helpers, track
+from .simulated_user import SimulatedUser
+
+
+def run_simulation(
+    app: Callable,
+    user_simulator: SimulatedUser,
+    initial_message: Optional[str] = None,
+    max_turns: int = 5,
+    thread_id: Optional[str] = None,
+    project_name: Optional[str] = None,
+    **app_kwargs: Any,
+) -> Dict[str, Any]:
+    """
+    Run a multi-turn conversation simulation between a simulated user and an app.
+
+    1. The simulator passes single message strings to the app
+    2. The app manages full conversation history internally using thread_id
+    3. The app logs traces with thread_id for evaluation
+
+    Args:
+        app: Callable that processes messages and manages conversation history internally.
+            Signature: app(message: str, *, thread_id: str, **kwargs) -> Dict[str, str]
+            The app is automatically decorated with @track and thread_id is injected via opik_args.
+        user_simulator: SimulatedUser instance that generates user responses
+        initial_message: Optional initial message from the user. If None, generated by simulator
+        max_turns: Maximum number of conversation turns (default: 5)
+        thread_id: Optional thread ID for grouping traces. Generated if not provided
+        project_name: Optional project name for trace logging
+        **app_kwargs: Additional keyword arguments passed to the app
+
+    Returns:
+        Dict containing:
+        - thread_id: The thread ID used for this simulation
+        - conversation_history: List of message dicts from the simulation
+        - project_name: Project name if provided
+    """
+    # Generate thread_id if not provided
+    if thread_id is None:
+        thread_id = id_helpers.generate_id()
+
+    # Automatically decorate app if not already decorated
+    if not hasattr(app, "opik_tracked"):
+        app_name = app.__name__ if hasattr(app, "__name__") else "simulation_app"
+        app = track(name=app_name)(app)
+
+    # Track conversation for simulator (app manages its own history internally)
+    conversation_history: List[Dict[str, str]] = []
+
+    # Generate initial message if needed
+    if initial_message is None:
+        initial_message = user_simulator.generate_response(conversation_history)
+
+    # Simulation loop
+    for turn in range(max_turns):
+        # Get user message
+        if turn == 0:
+            user_message_text = initial_message
+        else:
+            user_message_text = user_simulator.generate_response(conversation_history)
+
+        # Create message dict for tracking
+        user_message = {"role": "user", "content": user_message_text}
+        conversation_history.append(user_message)
+
+        # Call app with SINGLE message string, thread_id parameter, and opik_args for tracing
+        try:
+            assistant_message = app(
+                user_message_text,
+                thread_id=thread_id,
+                **app_kwargs,
+                opik_args={
+                    "trace": {
+                        "thread_id": thread_id,
+                        "metadata": {"turn": turn + 1, "project_name": project_name},
+                    }
+                },
+            )
+        except Exception as e:
+            # Handle app errors gracefully
+            assistant_message = {
+                "role": "assistant",
+                "content": f"Error processing message: {str(e)}",
+            }
+
+        # Validate assistant message format
+        if (
+            not isinstance(assistant_message, dict)
+            or "role" not in assistant_message
+            or "content" not in assistant_message
+        ):
+            assistant_message = {
+                "role": "assistant",
+                "content": str(assistant_message)
+                if assistant_message
+                else "No response",
+            }
+
+        conversation_history.append(assistant_message)
+
+    return {
+        "thread_id": thread_id,
+        "conversation_history": conversation_history,
+        "project_name": project_name,
+    }

opik/synchronization.py

@@ -40,15 +40,14 @@ def until(
     while True:
         try:
             if function():
-                break
+                return True
         except Exception:
             LOGGER.debug(
                 f"{function.__name__} raised error in 'until' function.", exc_info=True
             )
             if not allow_errors:
                 raise
-        finally:
-            if (time.time() - start_time) > max_try_seconds:
-                return False
-            time.sleep(sleep)
-    return True
+
+        if (time.time() - start_time) > max_try_seconds:
+            return False
+        time.sleep(sleep)

opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py}

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import threading
 from typing import Optional
+from . import config
 
 
 class TracingRuntimeConfig:
@@ -23,9 +24,7 @@ class TracingRuntimeConfig:
                 return self._tracing_active
 
         try:
-            from .. import config as _config_module
-
-            enabled = not _config_module.OpikConfig().track_disable
+            enabled = not config.OpikConfig().track_disable
             self._tracing_active = enabled
 
         except Exception:
@@ -34,16 +33,16 @@ class TracingRuntimeConfig:
         return enabled
 
 
-runtime_cfg = TracingRuntimeConfig()
+runtime_config = TracingRuntimeConfig()
 
 
 def set_tracing_active(active: bool) -> None:
-    runtime_cfg.set_tracing_active(active)
+    runtime_config.set_tracing_active(active)
 
 
 def is_tracing_active() -> bool:
-    return runtime_cfg.is_tracing_active()
+    return runtime_config.is_tracing_active()
 
 
 def reset_tracing_to_config_default() -> None:
-    runtime_cfg.reset_to_config_default()
+    runtime_config.reset_to_config_default()

opik/types.py

@@ -2,6 +2,7 @@ import enum
 import sys
 from typing import Literal, Optional
 
+from pydantic import StrictStr
 from typing_extensions import TypedDict
 
 if sys.version_info < (3, 11):
@@ -79,6 +80,41 @@ class FeedbackScoreDict(TypedDict):
     """An optional explanation or justification for the given score."""
 
 
+class BatchFeedbackScoreDict(TypedDict):
+    """
+    A TypedDict representing a feedback score for batch operations.
+
+    This class defines the structure for feedback scores used in batch logging
+    operations, with a required id field and optional per-score project_name.
+    """
+
+    id: Required[str]
+    """
+    A unique identifier for the object this score should be assigned to.
+    Refers to either the trace_id, span_id or thread_id depending on how the score is logged.
+    Required for batch operations.
+    """
+
+    name: Required[str]
+    """The name of the feedback metric or criterion."""
+
+    value: Required[float]
+    """The numerical value of the feedback score."""
+
+    project_name: NotRequired[Optional[StrictStr]]
+    """
+    The name of the project for this specific score.
+    If not provided, falls back to the project_name parameter in the method call,
+    or the default project name configured in the Opik instance.
+    """
+
+    category_name: NotRequired[Optional[str]]
+    """An optional category name for the given score."""
+
+    reason: NotRequired[Optional[str]]
+    """An optional explanation or justification for the given score."""
+
+
 class ErrorInfoDict(TypedDict):
     """
     A TypedDict representing the information about the error occurred.

opik/validation/chat_prompt_messages.py

@@ -0,0 +1,241 @@
+from typing import Any, List, Optional
+
+import opik.exceptions as exceptions
+from . import validator, result
+
+
+class ChatPromptMessagesValidator(validator.RaisableValidator):
+    """
+    Validator for ChatPrompt messages list.
+
+    Validates that messages is a list of dicts with:
+    - "role" key with value "system", "user", or "assistant"
+    - "content" key with value either string or list of dicts
+    - If content is list of dicts, each dict must have "type" key
+    """
+
+    VALID_ROLES = {"system", "user", "assistant"}
+    URL_BASED_CONTENT_TYPES = {"image_url", "video_url", "audio_url"}
+
+    def __init__(self, messages: Any):
+        self.messages = messages
+        self.validation_result: Optional[result.ValidationResult] = None
+
+    def validate(self) -> result.ValidationResult:
+        failure_reasons: List[str] = []
+
+        # Validate messages is a list
+        if not self._validate_messages_is_list(failure_reasons):
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+            return self.validation_result
+
+        # Validate each message in the list
+        for idx, message in enumerate(self.messages):
+            prefix = f"messages[{idx}]"
+            self._validate_message(prefix, message, failure_reasons)
+
+        # Create validation result
+        if len(failure_reasons) > 0:
+            self.validation_result = result.ValidationResult(
+                failed=True, failure_reasons=failure_reasons
+            )
+        else:
+            self.validation_result = result.ValidationResult(failed=False)
+
+        return self.validation_result
+
+    def _validate_messages_is_list(self, failure_reasons: List[str]) -> bool:
+        """Validate that messages is a list. Returns False if validation fails."""
+        if not isinstance(self.messages, list):
+            msg = (
+                f"messages must be a list but {type(self.messages).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return False
+        return True
+
+    def _validate_message(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single message structure, role, and content."""
+        if not self._validate_message_structure(prefix, message, failure_reasons):
+            return
+
+        self._validate_role(prefix, message, failure_reasons)
+        self._validate_content(prefix, message, failure_reasons)
+
+    def _validate_message_structure(
+        self, prefix: str, message: Any, failure_reasons: List[str]
+    ) -> bool:
+        """Validate that message is a dict with exactly 'role' and 'content' keys. Returns False if validation fails."""
+        # Validate message is a dict
+        if not isinstance(message, dict):
+            msg = f"{prefix}: must be a dict but {type(message).__name__} was given"
+            failure_reasons.append(msg)
+            return False
+
+        # Validate message has exactly "role" and "content" keys
+        message_keys = set(message.keys())
+        expected_keys = {"role", "content"}
+
+        if message_keys != expected_keys:
+            if not message_keys.issubset(expected_keys):
+                missing_keys = expected_keys - message_keys
+                msg = f"{prefix}: missing required keys: {sorted(missing_keys)}"
+                failure_reasons.append(msg)
+            if not expected_keys.issubset(message_keys):
+                extra_keys = message_keys - expected_keys
+                msg = (
+                    f"{prefix}: unexpected keys: {sorted(extra_keys)}. "
+                    f"Expected only: {sorted(expected_keys)}"
+                )
+                failure_reasons.append(msg)
+            return False
+
+        return True
+
+    def _validate_role(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the role field of a message."""
+        role = message.get("role")
+        if role not in self.VALID_ROLES:
+            valid_roles_str = ", ".join([f"'{r}'" for r in sorted(self.VALID_ROLES)])
+            msg = (
+                f"{prefix}.role: must be one of [{valid_roles_str}] "
+                f"but {repr(role)} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_content(
+        self, prefix: str, message: dict, failure_reasons: List[str]
+    ) -> None:
+        """Validate the content field of a message."""
+        content = message.get("content")
+        if content is None:
+            msg = f"{prefix}.content: must not be None"
+            failure_reasons.append(msg)
+        elif not isinstance(content, (str, list)):
+            msg = (
+                f"{prefix}.content: must be either str or list of dicts "
+                f"but {type(content).__name__} was given"
+            )
+            failure_reasons.append(msg)
+        elif isinstance(content, list):
+            self._validate_content_list(prefix, content, failure_reasons)
+
+    def _validate_content_list(
+        self, prefix: str, content: list, failure_reasons: List[str]
+    ) -> None:
+        """Validate content when it is a list of content parts."""
+        for content_idx, content_part in enumerate(content):
+            content_prefix = f"{prefix}.content[{content_idx}]"
+            self._validate_content_part(content_prefix, content_part, failure_reasons)
+
+    def _validate_content_part(
+        self, content_prefix: str, content_part: Any, failure_reasons: List[str]
+    ) -> None:
+        """Validate a single content part in the content list."""
+        if not isinstance(content_part, dict):
+            msg = (
+                f"{content_prefix}: must be a dict "
+                f"but {type(content_part).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "type" not in content_part:
+            msg = f"{content_prefix}: must have 'type' key"
+            failure_reasons.append(msg)
+            return
+
+        # Validate type-specific requirements
+        content_type = content_part.get("type")
+        self._validate_content_type_specific(
+            content_prefix, content_type, content_part, failure_reasons
+        )
+
+    def _validate_content_type_specific(
+        self,
+        content_prefix: str,
+        content_type: Any,
+        content_part: dict,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate type-specific requirements for content parts."""
+        if content_type in self.URL_BASED_CONTENT_TYPES:
+            self._validate_required_url_object(
+                content_prefix,
+                content_part,
+                content_type,
+                content_type,
+                failure_reasons,
+            )
+        elif content_type == "text":
+            self._validate_required_string_key(
+                content_prefix, content_part, "text", "text", failure_reasons
+            )
+
+    def _validate_required_string_key(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+        elif not isinstance(content_part.get(key_name), str):
+            msg = (
+                f"{prefix}.{key_name}: must be a string "
+                f"but {type(content_part.get(key_name)).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def _validate_required_url_object(
+        self,
+        prefix: str,
+        content_part: dict,
+        key_name: str,
+        type_name: str,
+        failure_reasons: List[str],
+    ) -> None:
+        """Validate that a required key exists and is a dict with a 'url' key that is a string."""
+        if key_name not in content_part:
+            msg = f"{prefix}: must have '{key_name}' key when type is '{type_name}'"
+            failure_reasons.append(msg)
+            return
+
+        url_object = content_part.get(key_name)
+        if not isinstance(url_object, dict):
+            msg = (
+                f"{prefix}.{key_name}: must be a dict "
+                f"but {type(url_object).__name__} was given"
+            )
+            failure_reasons.append(msg)
+            return
+
+        if "url" not in url_object:
+            msg = f"{prefix}.{key_name}: must have 'url' key"
+            failure_reasons.append(msg)
+        elif not isinstance(url_object.get("url"), str):
+            msg = (
+                f"{prefix}.{key_name}.url: must be a string "
+                f"but {type(url_object.get('url')).__name__} was given"
+            )
+            failure_reasons.append(msg)
+
+    def raise_if_validation_failed(self) -> None:
+        if (
+            self.validation_result is not None
+            and len(self.validation_result.failure_reasons) > 0
+        ):
+            raise exceptions.ValidationError(
+                prefix="ChatPrompt.__init__",
+                failure_reasons=self.validation_result.failure_reasons,
+            )

opik/validation/feedback_score.py

@@ -1,16 +1,16 @@
 import pydantic
 
 from typing import Any
-from ..types import FeedbackScoreDict
+from ..types import BatchFeedbackScoreDict
 from . import validator, result
 
 
 class PydanticWrapper(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(extra="forbid")
-    feedback_score: FeedbackScoreDict
+    feedback_score: BatchFeedbackScoreDict
 
 
-EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str]}"
+EXPECTED_TYPES = "{'id': str, 'name': str, 'value': float, 'reason': NotRequired[str], 'category_name': NotRequired[str], 'project_name': NotRequired[str]}"
 
 
 class FeedbackScoreValidator(validator.Validator):

opik/validation/validator.py

@@ -1,4 +1,5 @@
 import abc
+from typing import Any
 
 from . import result
 
@@ -7,3 +8,30 @@ class Validator(abc.ABC):
     @abc.abstractmethod
     def validate(self) -> result.ValidationResult:
         pass
+
+
+class RaisableValidator(Validator):
+    """
+    Abstract validator class that extends Validator and adds raise_if_validation_failed method.
+
+    This is used for validators that need to raise ValidationError exceptions
+    when validation fails, typically used in class initialization.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Initialize the validator.
+
+        Subclasses can override this method with their own initialization signature.
+        """
+        pass
+
+    @abc.abstractmethod
+    def raise_if_validation_failed(self) -> None:
+        """
+        Raise a ValidationError if validation failed.
+
+        This method should check the validation result and raise an appropriate
+        ValidationError exception if validation failed.
+        """
+        pass