langwatch-scenario 0.4.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

scenario/_events/event_bus.py

@@ -0,0 +1,185 @@
+from rx.core.observable.observable import Observable
+from typing import Optional, Any
+from .events import ScenarioEvent
+from .event_reporter import EventReporter
+
+import asyncio
+import queue
+import threading
+import logging
+
+class ScenarioEventBus:
+    """
+    Subscribes to scenario event streams and handles HTTP posting using a dedicated worker thread.
+
+    The EventBus acts as an observer of scenario events, automatically
+    posting them to external APIs. It uses a queue-based threading model
+    where events are processed by a dedicated worker thread.
+
+    Key design principles:
+    - Single worker thread handles all HTTP posting (simplifies concurrency)
+    - Thread created lazily when first event arrives
+    - Thread terminates when queue empty and stream completed
+    - Non-daemon thread ensures all events posted before program exit
+
+    Attributes:
+        _event_reporter: EventReporter instance for HTTP posting of events
+        _max_retries: Maximum number of retry attempts for failed event processing
+        _event_queue: Thread-safe queue for passing events to worker thread
+        _completed: Whether the event stream has completed
+        _subscription: RxPY subscription to the event stream
+        _worker_thread: Dedicated thread for processing events
+    """
+
+    def __init__(
+        self, event_reporter: Optional[EventReporter] = None, max_retries: int = 3
+    ):
+        """
+        Initialize the event bus with optional event reporter and retry configuration.
+
+        Args:
+            event_reporter: Optional EventReporter for HTTP posting of events.
+                          If not provided, a default EventReporter will be created.
+            max_retries: Maximum number of retry attempts for failed event processing.
+                       Defaults to 3 attempts with exponential backoff.
+        """
+        self._event_reporter: EventReporter = event_reporter or EventReporter()
+        self._max_retries = max_retries
+        
+        # Custom logger for this class
+        self.logger = logging.getLogger(__name__)
+        
+        # Threading infrastructure
+        self._event_queue: queue.Queue[ScenarioEvent] = queue.Queue()
+        self._completed = False
+        self._subscription: Optional[Any] = None
+        self._worker_thread: Optional[threading.Thread] = None
+        self._shutdown_event = threading.Event()  # Signal worker to shutdown
+
+    def _get_or_create_worker(self) -> None:
+        """Lazily create worker thread when first event arrives"""
+        if self._worker_thread is None or not self._worker_thread.is_alive():
+            self.logger.debug("Creating new worker thread")
+            self._worker_thread = threading.Thread(
+                target=self._worker_loop,
+                daemon=False,
+                name="ScenarioEventBus-Worker"
+            )
+            self._worker_thread.start()
+            self.logger.debug("Worker thread started")
+
+    def _worker_loop(self) -> None:
+        """Main worker thread loop - processes events from queue until shutdown"""
+        self.logger.debug("Worker thread loop started")
+        while True:
+            try:
+                if self._shutdown_event.wait(timeout=0.1):
+                    self.logger.debug("Worker thread received shutdown signal")
+                    break
+                
+                try:
+                    event = self._event_queue.get(timeout=0.1)
+                    self.logger.debug(f"Worker picked up event: {event.type_} ({event.scenario_run_id})")
+                    self._process_event_sync(event)
+                    self._event_queue.task_done()
+                except queue.Empty:
+                    # Exit if stream completed and no more events
+                    if self._completed:
+                        self.logger.debug("Stream completed and no more events, worker thread exiting")
+                        break
+                    continue
+                    
+            except Exception as e:
+                self.logger.error(f"Worker thread error: {e}")
+        
+        self.logger.debug("Worker thread loop ended")
+
+    def _process_event_sync(self, event: ScenarioEvent) -> None:
+        """
+        Process event synchronously in worker thread with retry logic.
+        """
+        self.logger.debug(f"Processing HTTP post for {event.type_} ({event.scenario_run_id})")
+        
+        try:
+            # Convert async to sync using asyncio.run - this blocks until HTTP completes
+            success = asyncio.run(self._process_event_with_retry(event))
+            if not success:
+                self.logger.warning(f"Failed to process event {event.type_} after {self._max_retries} attempts")
+            else:
+                self.logger.debug(f"Successfully posted {event.type_} ({event.scenario_run_id})")
+        except Exception as e:
+            self.logger.error(f"Error processing event {event.type_}: {e}")
+
+    async def _process_event_with_retry(self, event: ScenarioEvent, attempt: int = 1) -> bool:
+        """
+        Process a single event with retry logic (now runs in worker thread context).
+        """
+        try:
+            if self._event_reporter:
+                await self._event_reporter.post_event(event)
+            return True
+        except Exception as e:
+            if attempt >= self._max_retries:
+                return False
+            print(f"Error processing event (attempt {attempt}/{self._max_retries}): {e}")
+            await asyncio.sleep(0.1 * (2 ** (attempt - 1)))  # Exponential backoff
+            return await self._process_event_with_retry(event, attempt + 1)
+
+    def subscribe_to_events(self, event_stream: Observable) -> None:
+        """
+        Subscribe to any observable stream of scenario events.
+        Events are queued for processing by the dedicated worker thread.
+        """
+        if self._subscription is not None:
+            self.logger.debug("Already subscribed to event stream")
+            return
+
+        def handle_event(event: ScenarioEvent) -> None:
+            self.logger.debug(f"Event received, queuing: {event.type_} ({event.scenario_run_id})")
+            self._get_or_create_worker()
+            self._event_queue.put(event)
+            self.logger.debug(f"Event queued: {event.type_} ({event.scenario_run_id})")
+
+        self.logger.info("Subscribing to event stream")
+        self._subscription = event_stream.subscribe(
+            handle_event,
+            lambda e: self.logger.error(f"Error in event stream: {e}"),
+            lambda: self._set_completed()
+        )
+
+    def _set_completed(self):
+        """Helper to set completed state with logging"""
+        self.logger.debug("Event stream completed")
+        self._completed = True
+
+    def drain(self) -> None:
+        """
+        Waits for all queued events to complete processing.
+        
+        This method blocks until all events in the queue have been processed.
+        Since _process_event_sync() uses asyncio.run(), HTTP requests complete
+        before task_done() is called, so join() ensures everything is finished.
+        """
+        self.logger.debug("Drain started - waiting for queue to empty")
+        
+        # Wait for all events to be processed - this is sufficient!
+        self._event_queue.join()
+        self.logger.debug("Event queue drained")
+        
+        # Signal worker to shutdown and wait for it
+        self._shutdown_event.set()
+        if self._worker_thread and self._worker_thread.is_alive():
+            self.logger.debug("Waiting for worker thread to shutdown...")
+            self._worker_thread.join(timeout=5.0)
+            if self._worker_thread.is_alive():
+                self.logger.warning("Worker thread did not shutdown within timeout")
+            else:
+                self.logger.debug("Worker thread shutdown complete")
+        
+        self.logger.info("Drain completed")
+
+    def is_completed(self) -> bool:
+        """
+        Returns whether all events have been processed.
+        """
+        return self._completed and self._event_queue.empty()

scenario/_events/event_reporter.py

@@ -0,0 +1,83 @@
+import logging
+import os
+import httpx
+from typing import Optional
+from .events import ScenarioEvent
+
+
+class EventReporter:
+    """
+    Handles HTTP posting of scenario events to external endpoints.
+
+    Single responsibility: Send events via HTTP to configured endpoints
+    with proper authentication and error handling.
+
+    Args:
+        endpoint (str, optional): The base URL to post events to. Defaults to LANGWATCH_ENDPOINT env var.
+        api_key (str, optional): The API key for authentication. Defaults to LANGWATCH_API_KEY env var.
+
+    Example:
+        event = {
+            "type": "SCENARIO_RUN_STARTED",
+            "batch_run_id": "batch-1",
+            "scenario_id": "scenario-1",
+            "scenario_run_id": "run-1",
+            "metadata": {
+                "name": "test",
+                "description": "test scenario"
+            }
+        }
+
+        reporter = EventReporter(endpoint="https://api.langwatch.ai", api_key="test-api-key")
+        await reporter.post_event(event)
+    """
+
+    def __init__(self, endpoint: Optional[str] = None, api_key: Optional[str] = None):
+        self.endpoint = endpoint or os.getenv("LANGWATCH_ENDPOINT")
+        self.api_key = api_key or os.getenv("LANGWATCH_API_KEY", "")
+        self.logger = logging.getLogger(__name__)
+
+    async def post_event(self, event: ScenarioEvent):
+        """
+        Posts an event to the configured endpoint.
+
+        Args:
+            event: A dictionary containing the event data
+
+        Returns:
+            None - logs success/failure internally
+        """
+        event_type = event.type_
+        self.logger.info(f"[{event_type}] Publishing event ({event.scenario_run_id})")
+
+        if not self.endpoint:
+            self.logger.warning(
+                "No LANGWATCH_ENDPOINT configured, skipping event posting"
+            )
+            return
+
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(
+                    f"{self.endpoint}/api/scenario-events",
+                    json=event.to_dict(),
+                    headers={
+                        "Content-Type": "application/json",
+                        "X-Auth-Token": self.api_key,
+                    },
+                )
+                self.logger.info(f"[{event_type}] POST response status: {response.status_code} ({event.scenario_run_id})")
+                
+                if response.is_success:
+                    data = response.json()
+                    self.logger.info(f"[{event_type}] POST response: {data} ({event.scenario_run_id})")
+                else:
+                    error_text = response.text
+                    self.logger.error(
+                        f"[{event_type}] Event POST failed: status={response.status_code}, "
+                        f"reason={response.reason_phrase}, error={error_text}, "
+                        f"event={event}"
+                    )
+        except Exception as error:
+            self.logger.error(
+                f"[{event_type}] Event POST error: {error}, event={event}, endpoint={self.endpoint}") 

scenario/_events/events.py

@@ -0,0 +1,162 @@
+"""
+Exports scenario event models from the generated LangWatch API client,
+renaming the auto-generated types to clean, meaningful names.
+
+This ensures all event types are always in sync with the OpenAPI spec and
+the backend, and provides a single import location for event models.
+
+If you need to add custom logic or helpers, you can extend or wrap these models here.
+"""
+
+from typing import Union, Any, Optional, TypeAlias
+from scenario._generated.langwatch_api_client.lang_watch_api_client.models import (
+    PostApiScenarioEventsBodyType0,
+    PostApiScenarioEventsBodyType0Metadata,
+    PostApiScenarioEventsBodyType1,
+    PostApiScenarioEventsBodyType1ResultsType0,
+    PostApiScenarioEventsBodyType1ResultsType0Verdict,
+    PostApiScenarioEventsBodyType1Status,
+    PostApiScenarioEventsBodyType2,
+)
+from .messages import MessageType
+
+# Create alias for cleaner naming
+ScenarioRunStartedEventMetadata: TypeAlias = PostApiScenarioEventsBodyType0Metadata
+ScenarioRunFinishedEventResults: TypeAlias = PostApiScenarioEventsBodyType1ResultsType0
+ScenarioRunFinishedEventVerdict: TypeAlias = PostApiScenarioEventsBodyType1ResultsType0Verdict
+ScenarioRunFinishedEventStatus: TypeAlias = PostApiScenarioEventsBodyType1Status
+
+
+class ScenarioRunStartedEvent(PostApiScenarioEventsBodyType0):
+    """
+    Event published when a scenario run begins execution.
+
+    Automatically sets type_ to "SCENARIO_RUN_STARTED" and includes metadata
+    about the scenario (name, description, etc.).
+
+    Args:
+        batch_run_id (str): Unique identifier for the batch of scenario runs
+        scenario_id (str): Unique identifier for the scenario definition
+        scenario_run_id (str): Unique identifier for this specific run
+        metadata (ScenarioRunStartedEventMetadata): Scenario details like name and description
+        timestamp (Optional[int], optional): Unix timestamp in milliseconds, auto-generated if not provided
+        raw_event (Optional[Any], optional): Raw event data
+        scenario_set_id (Optional[str], optional): Set identifier, defaults to "default"
+    """
+    def __init__(
+        self,
+        batch_run_id: str,
+        scenario_id: str,
+        scenario_run_id: str,
+        metadata: ScenarioRunStartedEventMetadata,
+        timestamp: int,
+        raw_event: Optional[Any] = None,
+        scenario_set_id: Optional[str] = "default"
+    ):
+        super().__init__(
+            type_="SCENARIO_RUN_STARTED",
+            batch_run_id=batch_run_id,
+            scenario_id=scenario_id,
+            scenario_run_id=scenario_run_id,
+            metadata=metadata,
+            timestamp=timestamp,
+            raw_event=raw_event,
+            scenario_set_id=scenario_set_id or "default"
+        )
+
+class ScenarioRunFinishedEvent(PostApiScenarioEventsBodyType1):
+    """
+    Event published when a scenario run completes execution.
+
+    Automatically sets type_ to "SCENARIO_RUN_FINISHED" and includes results
+    with verdict (PASS/FAIL/SUCCESS) and reasoning.
+
+    Args:
+        batch_run_id (str): Unique identifier for the batch of scenario runs
+        scenario_id (str): Unique identifier for the scenario definition
+        scenario_run_id (str): Unique identifier for this specific run
+        status (ScenarioRunFinishedEventStatus): Overall execution status
+        timestamp (Optional[int], optional): Unix timestamp in milliseconds, auto-generated if not provided
+        raw_event (Optional[Any], optional): Raw event data
+        scenario_set_id (Optional[str], optional): Set identifier, defaults to "default"
+        results (Optional[ScenarioRunFinishedEventResults], optional): Verdict and reasoning for the outcome
+    """
+    def __init__(
+        self,
+        batch_run_id: str,
+        scenario_id: str,
+        scenario_run_id: str,
+        status: ScenarioRunFinishedEventStatus,
+        timestamp: int,
+        results: Optional[ScenarioRunFinishedEventResults] = None,
+        raw_event: Optional[Any] = None,
+        scenario_set_id: Optional[str] = "default",
+    ):
+        super().__init__(
+            type_="SCENARIO_RUN_FINISHED",
+            batch_run_id=batch_run_id,
+            scenario_id=scenario_id,
+            scenario_run_id=scenario_run_id,
+            status=status,
+            timestamp=timestamp,
+            raw_event=raw_event,
+            scenario_set_id=scenario_set_id or "default",
+            results=results
+        )
+
+class ScenarioMessageSnapshotEvent(PostApiScenarioEventsBodyType2):
+    """
+    Event published to capture intermediate state during scenario execution.
+
+    Automatically sets type_ to "SCENARIO_MESSAGE_SNAPSHOT" and allows tracking
+    of messages, context, or other runtime data during scenario processing.
+
+    Args:
+        batch_run_id (str): Unique identifier for the batch of scenario runs
+        scenario_id (str): Unique identifier for the scenario definition
+        scenario_run_id (str): Unique identifier for this specific run
+        messages (list[MessageType]): List of message objects in the conversation
+        timestamp (Optional[int], optional): Unix timestamp in milliseconds, auto-generated if not provided
+        raw_event (Optional[Any], optional): Raw event data
+        scenario_set_id (Optional[str], optional): Set identifier, defaults to "default"
+    """
+    def __init__(
+        self,
+        batch_run_id: str,
+        scenario_id: str,
+        scenario_run_id: str,
+        messages: list[MessageType],
+        timestamp: int,
+        raw_event: Optional[Any] = None,
+        scenario_set_id: Optional[str] = "default"
+    ):
+        super().__init__(
+            type_="SCENARIO_MESSAGE_SNAPSHOT",
+            batch_run_id=batch_run_id,
+            scenario_id=scenario_id,
+            scenario_run_id=scenario_run_id,
+            messages=messages,
+            timestamp=timestamp,
+            raw_event=raw_event,
+            scenario_set_id=scenario_set_id or "default"
+        )
+
+# Union type for all supported event types
+ScenarioEvent = Union[
+    ScenarioRunStartedEvent,
+    ScenarioRunFinishedEvent,
+    ScenarioMessageSnapshotEvent
+]
+
+
+__all__ = [
+    "ScenarioEvent",
+    "ScenarioRunStartedEvent",
+    "ScenarioRunStartedEventMetadata",
+    "ScenarioRunFinishedEvent",
+    "ScenarioRunFinishedEventResults",
+    "ScenarioRunFinishedEventVerdict",
+    "ScenarioRunFinishedEventStatus",
+    "ScenarioMessageSnapshotEvent",
+    "MessageType",
+]

scenario/_events/messages.py

@@ -0,0 +1,58 @@
+"""
+Exports message models from the generated LangWatch API client,
+renaming the auto-generated types to clean, meaningful names.
+
+This ensures all message types are always in sync with the OpenAPI spec and
+the backend, and provides a single import location for message models.
+
+If you need to add custom logic or helpers, you can extend or wrap these models here.
+"""
+
+from typing import Union, TypeAlias
+from scenario._generated.langwatch_api_client.lang_watch_api_client.models import (
+    PostApiScenarioEventsBodyType2MessagesItemType0,
+    PostApiScenarioEventsBodyType2MessagesItemType1,
+    PostApiScenarioEventsBodyType2MessagesItemType2,
+    PostApiScenarioEventsBodyType2MessagesItemType3,
+    PostApiScenarioEventsBodyType2MessagesItemType4,
+    PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItem,
+    PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItemFunction,
+)
+
+# Create aliases for cleaner naming
+DeveloperMessage: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType0
+SystemMessage: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType1
+AssistantMessage: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType2
+UserMessage: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType3
+ToolMessage: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType4
+ToolCall: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItem
+FunctionCall: TypeAlias = PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItemFunction
+
+# Union type for all supported message types
+MessageType = Union[
+    DeveloperMessage,
+    SystemMessage,
+    AssistantMessage,
+    UserMessage,
+    ToolMessage,
+]
+
+__all__ = [
+    "MessageType",
+    "DeveloperMessage",
+    "SystemMessage",
+    "AssistantMessage",
+    "UserMessage",
+    "ToolMessage",
+    "ToolCall",
+    "FunctionCall",
+
+    # API client models -- Required for PDocs
+    "PostApiScenarioEventsBodyType2MessagesItemType0",
+    "PostApiScenarioEventsBodyType2MessagesItemType1",
+    "PostApiScenarioEventsBodyType2MessagesItemType2",
+    "PostApiScenarioEventsBodyType2MessagesItemType3",
+    "PostApiScenarioEventsBodyType2MessagesItemType4",
+    "PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItem",
+    "PostApiScenarioEventsBodyType2MessagesItemType2ToolCallsItemFunction",
+]

scenario/_events/utils.py

@@ -0,0 +1,97 @@
+import warnings
+from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
+from .events import MessageType
+from .messages import (
+    SystemMessage,
+    AssistantMessage,
+    UserMessage,
+    ToolMessage,
+    ToolCall,
+    FunctionCall,
+)
+from typing import List
+import uuid
+
+def convert_messages_to_api_client_messages(messages: list[ChatCompletionMessageParam]) -> list[MessageType]:
+    """
+    Converts OpenAI ChatCompletionMessageParam messages to API client Message format.
+
+    This function transforms messages from OpenAI's format to the API client format
+    that matches the expected schema for ScenarioMessageSnapshotEvent.
+
+    Args:
+        messages: List of OpenAI ChatCompletionMessageParam messages
+
+    Returns:
+        List of API client Message objects
+
+    Raises:
+        ValueError: If message role is not supported or message format is invalid
+    """
+
+    converted_messages: list[MessageType] = []
+
+    for i, message in enumerate(messages):
+        # Generate unique ID for each message
+        message_id = message.get("id") or str(uuid.uuid4())
+
+        role = message.get("role")
+        content = message.get("content")
+
+        if role == "user":
+            if not content:
+                raise ValueError(f"User message at index {i} missing required content")
+            converted_messages.append(UserMessage(
+                id=message_id,
+                role="user",
+                content=str(content)
+            ))
+        elif role == "assistant":
+            # Handle tool calls if present
+            tool_calls = message.get("tool_calls")
+            api_tool_calls: List[ToolCall] = []
+
+            if tool_calls:
+                for tool_call in tool_calls:
+                    api_tool_calls.append(ToolCall(
+                        id=tool_call.get("id", str(uuid.uuid4())),
+                        type_="function",
+                        function=FunctionCall(
+                            name=tool_call["function"].get("name", "unknown"),
+                            arguments=tool_call["function"].get("arguments", "{}")
+                        )
+                    ))
+
+            converted_messages.append(AssistantMessage(
+                id=message_id,
+                role="assistant",
+                content=str(content),
+                tool_calls=api_tool_calls
+            ))
+        elif role == "system":
+            if not content:
+                raise ValueError(f"System message at index {i} missing required content")
+            converted_messages.append(SystemMessage(
+                id=message_id,
+                role="system",
+                content=str(content)
+            ))
+        elif role == "tool":
+            tool_call_id = message.get("tool_call_id")
+            if not tool_call_id:
+                warnings.warn(f"Tool message at index {i} missing required tool_call_id, skipping tool message")
+                continue
+            if not content:
+                warnings.warn(f"Tool message at index {i} missing required content, skipping tool message")
+                continue
+
+            converted_messages.append(ToolMessage(
+                id=message_id,
+                role="tool",
+                content=str(content),
+                tool_call_id=tool_call_id
+            ))
+        else:
+            raise ValueError(f"Unsupported message role '{role}' at index {i}")
+
+    return converted_messages