langwatch-scenario 0.6.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

scenario/scenario_executor.py

@@ -19,14 +19,15 @@ from typing import (
     TypedDict,
 )
 import time
+import warnings
 import termcolor
 import asyncio
 import concurrent.futures
 
 from scenario.config import ScenarioConfig
 from scenario._utils import (
-    check_valid_return_type,
     convert_agent_return_types_to_openai_messages,
+    check_valid_return_type,
     print_openai_messages,
     show_spinner,
     await_if_awaitable,
@@ -46,17 +47,20 @@ from .agent_adapter import AgentAdapter
 from .script import proceed
 from pksuid import PKSUID
 from .scenario_state import ScenarioState
-from .events import (
-    ScenarioEventBus, 
-    ScenarioRunStartedEvent, 
-    ScenarioMessageSnapshotEvent, 
-    ScenarioRunFinishedEvent, 
-    ScenarioRunStartedEventMetadata, 
-    ScenarioRunFinishedEventResults, 
-    ScenarioRunFinishedEventVerdict, 
-    ScenarioRunFinishedEventStatus, 
-    convert_messages_to_ag_ui_messages,
+from ._events import (
+    ScenarioEventBus,
+    ScenarioEvent,
+    ScenarioRunStartedEvent,
+    ScenarioMessageSnapshotEvent,
+    ScenarioRunFinishedEvent,
+    ScenarioRunStartedEventMetadata,
+    ScenarioRunFinishedEventResults,
+    ScenarioRunFinishedEventVerdict,
+    ScenarioRunFinishedEventStatus,
+    convert_messages_to_api_client_messages,
 )
+from rx.subject.subject import Subject
+from rx.core.observable.observable import Observable
 
 
 class ScenarioExecutor:
@@ -80,40 +84,6 @@ class ScenarioExecutor:
         agents: List of agent adapters participating in the scenario
         script: Optional list of script steps to control scenario flow
         config: Configuration settings for execution behavior
-
-    Example:
-        ```
-        # Direct instantiation (less common)
-        executor = ScenarioExecutor(
-           name="weather query test",
-           description="User asks about weather, agent should provide helpful response",
-           agents=[
-               weather_agent,
-               scenario.UserSimulatorAgent(),
-               scenario.JudgeAgent(criteria=["Agent provides helpful weather info"])
-           ],
-           max_turns=10,
-           verbose=True
-        )
-        result = await executor._run()
-
-        # Preferred high-level API
-        result = await scenario.run(
-           name="weather query test",
-           description="User asks about weather, agent should provide helpful response",
-           agents=[
-               weather_agent,
-               scenario.UserSimulatorAgent(),
-               scenario.JudgeAgent(criteria=["Agent provides helpful weather info"])
-           ]
-        )
-        ```
-
-    Note:
-        - Scenarios run in isolated thread pools to support parallel execution
-        - All agent interactions are cached when cache_key is configured
-        - Debug mode allows step-by-step execution with user intervention
-        - Results include detailed timing information and conversation history
     """
 
     name: str
@@ -130,6 +100,7 @@ class ScenarioExecutor:
     _pending_roles_on_turn: List[AgentRole] = []
     _pending_agents_on_turn: Set[AgentAdapter] = set()
     _agent_times: Dict[int, float] = {}
+    _events: Subject
 
     event_bus: ScenarioEventBus
 
@@ -167,27 +138,7 @@ class ScenarioExecutor:
                       Overrides global configuration for this scenario.
             debug: Whether to enable debug mode with step-by-step execution.
                   Overrides global configuration for this scenario.
-            event_reporter: Optional event reporter for the scenario
-
-        Example:
-            ```python
-            executor = ScenarioExecutor(
-                name="customer service test",
-                description="Customer has a billing question and needs help",
-                agents=[
-                    customer_service_agent,
-                    scenario.UserSimulatorAgent(),
-                    scenario.JudgeAgent(criteria=[
-                        "Agent is polite and professional",
-                        "Agent addresses the billing question",
-                        "Agent provides clear next steps"
-                    ])
-                ],
-                max_turns=15,
-                verbose=True,
-                debug=False
-            )
-            ```
+            event_bus: Optional event bus that will subscribe to this executor's events
         """
         self.name = name
         self.description = description
@@ -204,120 +155,33 @@ class ScenarioExecutor:
 
         self.reset()
 
+        # Create executor's own event stream
+        self._events = Subject()
+
+        # Create and configure event bus to subscribe to our events
         self.event_bus = event_bus or ScenarioEventBus()
+        self.event_bus.subscribe_to_events(self._events)
 
         self.batch_run_id = get_or_create_batch_run_id()
 
-    @classmethod
-    async def run(
-        cls,
-        name: str,
-        description: str,
-        agents: List[AgentAdapter] = [],
-        max_turns: Optional[int] = None,
-        verbose: Optional[Union[bool, int]] = None,
-        cache_key: Optional[str] = None,
-        debug: Optional[bool] = None,
-        script: Optional[List[ScriptStep]] = None,
-    ) -> ScenarioResult:
+    @property
+    def events(self) -> Observable:
+        """Expose event stream for subscribers like the event bus."""
+        return self._events
+
+    def _emit_event(self, event: ScenarioEvent) -> None:
         """
-        High-level interface for running a scenario test.
+        Emit a domain event to all subscribers.
 
-        This is the main entry point for executing scenario tests. It creates a
-        ScenarioExecutor instance and runs it in an isolated thread pool to support
-        parallel execution and prevent blocking.
+        This method publishes scenario events to the internal event stream,
+        which subscribers (like the event bus) can observe and react to.
+        The timestamp is automatically set to the current time.
 
         Args:
-            name: Human-readable name for the scenario
-            description: Detailed description of what the scenario tests
-            agents: List of agent adapters (agent under test, user simulator, judge)
-            max_turns: Maximum conversation turns before timeout (default: 10)
-            verbose: Show detailed output during execution
-            cache_key: Cache key for deterministic behavior
-            debug: Enable debug mode for step-by-step execution
-            script: Optional script steps to control scenario flow
-
-        Returns:
-            ScenarioResult containing the test outcome, conversation history,
-            success/failure status, and detailed reasoning
-
-        Example:
-            ```
-            import scenario
-
-            # Simple scenario with automatic flow
-            result = await scenario.run(
-               name="help request",
-               description="User asks for help with a technical problem",
-               agents=[
-                   my_agent,
-                   scenario.UserSimulatorAgent(),
-                   scenario.JudgeAgent(criteria=["Agent provides helpful response"])
-               ]
-            )
-
-            # Scripted scenario with custom evaluations
-            result = await scenario.run(
-               name="custom interaction",
-               description="Test specific conversation flow",
-               agents=[
-                   my_agent,
-                   scenario.UserSimulatorAgent(),
-                   scenario.JudgeAgent(criteria=["Agent provides helpful response"])
-               ],
-               script=[
-                   scenario.user("Hello"),
-                   scenario.agent(),
-                   custom_eval,
-                   scenario.succeed()
-               ]
-            )
-
-            # Results analysis
-            print(f"Test {'PASSED' if result.success else 'FAILED'}")
-            print(f"Reasoning: {result.reasoning}")
-            print(f"Conversation had {len(result.messages)} messages")
-            ```
-
-        Note:
-            - Runs in isolated thread pool to support parallel execution
-            - Blocks until scenario completes or times out
-            - All agent calls are automatically cached when cache_key is set
-            - Exception handling ensures clean resource cleanup
+            event: The scenario event to emit
         """
-        scenario = cls(
-            name=name,
-            description=description,
-            agents=agents,
-            max_turns=max_turns,
-            verbose=verbose,
-            cache_key=cache_key,
-            debug=debug,
-            script=script,
-        )
-
-        # We'll use a thread pool to run the execution logic, we
-        # require a separate thread because even though asyncio is
-        # being used throughout, any user code on the callback can
-        # be blocking, preventing them from running scenarios in parallel
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-
-            def run_in_thread():
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
-
-                try:
-                    return loop.run_until_complete(scenario._run())
-                finally:
-                    loop.run_until_complete(scenario.event_bus.drain())
-                    loop.close()
-
-            # Run the function in the thread pool and await its result
-            # This converts the thread's execution into a Future that the current
-            # event loop can await without blocking
-            loop = asyncio.get_event_loop()
-            result = await loop.run_in_executor(executor, run_in_thread)
-            return result
+        event.timestamp = int(time.time() * 1000)
+        self._events.on_next(event)
 
     def reset(self):
         """
@@ -394,7 +258,6 @@ class ScenarioExecutor:
                 self._pending_messages[idx] = []
             self._pending_messages[idx].append(message)
 
-
     def add_messages(
         self,
         messages: List[ChatCompletionMessageParam],
@@ -522,7 +385,7 @@ class ScenarioExecutor:
             agent_time=agent_time,
         )
 
-    async def _run(self) -> ScenarioResult:
+    async def run(self) -> ScenarioResult:
         """
         Run a scenario against the agent under test.
 
@@ -535,7 +398,6 @@ class ScenarioExecutor:
         scenario_run_id = generate_scenario_run_id()
 
         try:
-            await self.event_bus.listen()
             self._emit_run_started_event(scenario_run_id)
 
             if self.config.verbose:
@@ -552,7 +414,11 @@ class ScenarioExecutor:
                 self._emit_message_snapshot_event(scenario_run_id)
 
                 if isinstance(result, ScenarioResult):
-                    status = ScenarioRunFinishedEventStatus.SUCCESS if result.success else ScenarioRunFinishedEventStatus.FAILED
+                    status = (
+                        ScenarioRunFinishedEventStatus.SUCCESS
+                        if result.success
+                        else ScenarioRunFinishedEventStatus.FAILED
+                    )
                     self._emit_run_finished_event(scenario_run_id, result, status)
                     return result
 
@@ -565,7 +431,11 @@ class ScenarioExecutor:
                 """
             )
 
-            status = ScenarioRunFinishedEventStatus.SUCCESS if result.success else ScenarioRunFinishedEventStatus.FAILED
+            status = (
+                ScenarioRunFinishedEventStatus.SUCCESS
+                if result.success
+                else ScenarioRunFinishedEventStatus.FAILED
+            )
             self._emit_run_finished_event(scenario_run_id, result, status)
             return result
 
@@ -578,7 +448,9 @@ class ScenarioExecutor:
                 total_time=time.time() - self._total_start_time,
                 agent_time=0,
             )
-            self._emit_run_finished_event(scenario_run_id, error_result, ScenarioRunFinishedEventStatus.ERROR)
+            self._emit_run_finished_event(
+                scenario_run_id, error_result, ScenarioRunFinishedEventStatus.ERROR
+            )
             raise  # Re-raise the exception after cleanup
 
     async def _call_agent(
@@ -620,16 +492,19 @@ class ScenarioExecutor:
         ):
             start_time = time.time()
 
-            agent_response = agent.call(
-                AgentInput(
-                    # TODO: test thread_id
-                    thread_id=self._state.thread_id,
-                    messages=self._state.messages,
-                    new_messages=self._pending_messages.get(idx, []),
-                    judgment_request=request_judgment,
-                    scenario_state=self._state,
+            # Prevent pydantic validation warnings which should already be disabled
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                agent_response = agent.call(
+                    AgentInput(
+                        # TODO: test thread_id
+                        thread_id=self._state.thread_id,
+                        messages=self._state.messages,
+                        new_messages=self._pending_messages.get(idx, []),
+                        judgment_request=request_judgment,
+                        scenario_state=self._state,
+                    )
                 )
-            )
             if not isinstance(agent_response, Awaitable):
                 raise Exception(
                     agent_response_not_awaitable(agent.__class__.__name__),
@@ -819,16 +694,17 @@ class ScenarioExecutor:
     class _CommonEventFields(TypedDict):
         """
         Common fields shared across all scenario events.
-        
+
         These fields provide consistent identification and timing information
         for all events emitted during scenario execution.
-        
+
         Attributes:
             batch_run_id: Unique identifier for the batch of scenario runs
             scenario_run_id: Unique identifier for this specific scenario run
             scenario_id: Human-readable name/identifier for the scenario
             timestamp: Unix timestamp in milliseconds when the event occurred
         """
+
         batch_run_id: str
         scenario_run_id: str
         scenario_id: str
@@ -837,13 +713,13 @@ class ScenarioExecutor:
     def _create_common_event_fields(self, scenario_run_id: str) -> _CommonEventFields:
         """
         Create common fields used across all scenario events.
-        
+
         This method generates the standard fields that every scenario event
         must include for proper identification and timing.
-        
+
         Args:
             scenario_run_id: Unique identifier for the current scenario run
-            
+
         Returns:
             Dictionary containing common event fields with current timestamp
         """
@@ -857,86 +733,183 @@ class ScenarioExecutor:
     def _emit_run_started_event(self, scenario_run_id: str) -> None:
         """
         Emit a scenario run started event.
-        
+
         This event is published when a scenario begins execution. It includes
         metadata about the scenario such as name and description, and is used
         to track the start of scenario runs in monitoring systems.
-        
+
         Args:
             scenario_run_id: Unique identifier for the current scenario run
-            
-        Note:
-            This event is automatically published at the beginning of `_run()`
-            and signals the start of scenario execution to any event listeners.
         """
         common_fields = self._create_common_event_fields(scenario_run_id)
         metadata = ScenarioRunStartedEventMetadata(
             name=self.name,
             description=self.description,
         )
-        
+
         event = ScenarioRunStartedEvent(
             **common_fields,
             metadata=metadata,
         )
-        self.event_bus.publish(event)
+        self._emit_event(event)
 
     def _emit_message_snapshot_event(self, scenario_run_id: str) -> None:
         """
         Emit a message snapshot event.
-        
+
         This event captures the current state of the conversation during
         scenario execution. It's published whenever messages are added to
         the conversation, allowing real-time tracking of scenario progress.
-        
-        Note:
-            This event is automatically published by `add_message()` and
-            `add_messages()` to provide continuous visibility into scenario
-            execution state.
         """
         common_fields = self._create_common_event_fields(scenario_run_id)
-        
+
         event = ScenarioMessageSnapshotEvent(
             **common_fields,
-            messages=convert_messages_to_ag_ui_messages(self._state.messages),
+            messages=convert_messages_to_api_client_messages(self._state.messages),
         )
-        self.event_bus.publish(event)
+        self._emit_event(event)
 
     def _emit_run_finished_event(
-        self, 
-        scenario_run_id: str, 
-        result: ScenarioResult, 
-        status: ScenarioRunFinishedEventStatus
+        self,
+        scenario_run_id: str,
+        result: ScenarioResult,
+        status: ScenarioRunFinishedEventStatus,
     ) -> None:
         """
         Emit a scenario run finished event.
-        
+
         This event is published when a scenario completes execution, whether
         successfully or with an error. It includes the final results, verdict,
         and reasoning for the scenario outcome.
-        
+
         Args:
             scenario_run_id: Unique identifier for the current scenario run
             result: The final scenario result containing success/failure status
             status: The execution status (SUCCESS, FAILED, or ERROR)
-            
-        Note:
-            This event is automatically published at the end of `_run()` and
-            signals the completion of scenario execution to any event listeners.
-            It includes detailed results for monitoring and analysis purposes.
         """
         common_fields = self._create_common_event_fields(scenario_run_id)
-        
+
         results = ScenarioRunFinishedEventResults(
-            verdict=ScenarioRunFinishedEventVerdict.SUCCESS if result.success else ScenarioRunFinishedEventVerdict.FAILURE,
+            verdict=(
+                ScenarioRunFinishedEventVerdict.SUCCESS
+                if result.success
+                else ScenarioRunFinishedEventVerdict.FAILURE
+            ),
             reasoning=result.reasoning or "",
             met_criteria=result.passed_criteria,
             unmet_criteria=result.failed_criteria,
         )
-        
+
         event = ScenarioRunFinishedEvent(
             **common_fields,
             status=status,
             results=results,
         )
-        self.event_bus.publish(event)
+        self._emit_event(event)
+
+        # Signal end of event stream
+        self._events.on_completed()
+
+
+async def run(
+    name: str,
+    description: str,
+    agents: List[AgentAdapter] = [],
+    max_turns: Optional[int] = None,
+    verbose: Optional[Union[bool, int]] = None,
+    cache_key: Optional[str] = None,
+    debug: Optional[bool] = None,
+    script: Optional[List[ScriptStep]] = None,
+) -> ScenarioResult:
+    """
+    High-level interface for running a scenario test.
+
+    This is the main entry point for executing scenario tests. It creates a
+    ScenarioExecutor instance and runs it in an isolated thread pool to support
+    parallel execution and prevent blocking.
+
+    Args:
+        name: Human-readable name for the scenario
+        description: Detailed description of what the scenario tests
+        agents: List of agent adapters (agent under test, user simulator, judge)
+        max_turns: Maximum conversation turns before timeout (default: 10)
+        verbose: Show detailed output during execution
+        cache_key: Cache key for deterministic behavior
+        debug: Enable debug mode for step-by-step execution
+        script: Optional script steps to control scenario flow
+
+    Returns:
+        ScenarioResult containing the test outcome, conversation history,
+        success/failure status, and detailed reasoning
+
+    Example:
+        ```
+        import scenario
+
+        # Simple scenario with automatic flow
+        result = await scenario.run(
+           name="help request",
+           description="User asks for help with a technical problem",
+           agents=[
+               my_agent,
+               scenario.UserSimulatorAgent(),
+               scenario.JudgeAgent(criteria=["Agent provides helpful response"])
+           ]
+        )
+
+        # Scripted scenario with custom evaluations
+        result = await scenario.run(
+           name="custom interaction",
+           description="Test specific conversation flow",
+           agents=[
+               my_agent,
+               scenario.UserSimulatorAgent(),
+               scenario.JudgeAgent(criteria=["Agent provides helpful response"])
+           ],
+           script=[
+               scenario.user("Hello"),
+               scenario.agent(),
+               custom_eval,
+               scenario.succeed()
+           ]
+        )
+
+        # Results analysis
+        print(f"Test {'PASSED' if result.success else 'FAILED'}")
+        print(f"Reasoning: {result.reasoning}")
+        print(f"Conversation had {len(result.messages)} messages")
+        ```
+    """
+    scenario = ScenarioExecutor(
+        name=name,
+        description=description,
+        agents=agents,
+        max_turns=max_turns,
+        verbose=verbose,
+        cache_key=cache_key,
+        debug=debug,
+        script=script,
+    )
+
+    # We'll use a thread pool to run the execution logic, we
+    # require a separate thread because even though asyncio is
+    # being used throughout, any user code on the callback can
+    # be blocking, preventing them from running scenarios in parallel
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+
+        def run_in_thread():
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+            try:
+                return loop.run_until_complete(scenario.run())
+            finally:
+                scenario.event_bus.drain()
+                loop.close()
+
+        # Run the function in the thread pool and await its result
+        # This converts the thread's execution into a Future that the current
+        # event loop can await without blocking
+        loop = asyncio.get_event_loop()
+        result = await loop.run_in_executor(executor, run_in_thread)
+        return result

scenario/types.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from pydantic import BaseModel, Field, SkipValidation
+from pydantic import BaseModel, SkipValidation
 from typing import (
     TYPE_CHECKING,
     Annotated,
@@ -35,6 +35,7 @@ class AgentRole(Enum):
         AGENT: Represents the agent under test that responds to user inputs
         JUDGE: Represents a judge agent that evaluates the conversation and determines success/failure
     """
+
     USER = "User"
     AGENT = "Agent"
     JUDGE = "Judge"
@@ -71,6 +72,7 @@ class AgentInput(BaseModel):
                 return response
         ```
     """
+
     thread_id: str
     # Prevent pydantic from validating/parsing the messages and causing issues: https://github.com/pydantic/pydantic/issues/9541
     messages: Annotated[List[ChatCompletionMessageParam], SkipValidation]
@@ -168,7 +170,8 @@ class ScenarioResult(BaseModel):
     """
 
     success: bool
-    messages: List[ChatCompletionMessageParam]
+    # Prevent issues with slightly inconsistent message types for example when comming from Gemini right at the result level
+    messages: Annotated[List[ChatCompletionMessageParam], SkipValidation]
     reasoning: Optional[str] = None
     passed_criteria: List[str] = []
     failed_criteria: List[str] = []

langwatch_scenario-0.6.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-scenario/__init__.py,sha256=UJ5l-sG4TMG0wR8Ba-dxdDW36m3apTvawP-lNvk7Jm0,4293
-scenario/_error_messages.py,sha256=6lEx3jBGMbPx0kG0eX5zoZE-ENVM3O_ZkIbVMlnidYs,3892
-scenario/agent_adapter.py,sha256=PoY2KQqYuqzIIb3-nhIU-MPXwHJc1vmwdweMy7ut-hk,4255
-scenario/cache.py,sha256=J6s6Sia_Ce6TrnsInlhfxm6SF8tygo3sH-_cQCRX1WA,6213
-scenario/config.py,sha256=xhUuXH-sThwPTmJNSuajKxX-WC_tcFwJ1jZc119DswA,6093
-scenario/judge_agent.py,sha256=9CCO699qoWqXvWdQ73Yc3dqPOwaJdJ-zqxVaLaKi_cA,16161
-scenario/pytest_plugin.py,sha256=f2ETBpATz80k7K87M6046ZIFiQpHEvDN7dxakd3y2wk,11321
-scenario/scenario_executor.py,sha256=nkSIuIlwPHfr6pueSBbARrgiqPtW0SxajV3PFypAnJ4,34508
-scenario/scenario_state.py,sha256=dQDjazem-dn1c5mw6TwngEu6Tv_cHwEzemepsPBy2f0,7039
-scenario/script.py,sha256=A0N5pP0l4FFn1xdKc78U_wkwWhEWH3EFeU_LRDtNyEI,12241
-scenario/types.py,sha256=BhXcTEMGyGg_1QysN-GXVjm8DP2VH3UEzj_qvoglp2k,9466
-scenario/user_simulator_agent.py,sha256=fhwi8W44s343BGrjJXSJw960wcK7MgwTg-epxR1bqHo,9088
-scenario/_utils/__init__.py,sha256=wNX9hU8vzYlyLDwjkt7JUW3IPo2DhME6UIt_zvLM3B0,1000
-scenario/_utils/ids.py,sha256=K1iPuJgPh3gX9HCrDZGqK5lDgdwZXfOBF1YXVOWNHRg,1843
-scenario/_utils/message_conversion.py,sha256=AM9DLyWpy97CrAH8RmId9Mv2rmLquQhFoUpRyp-jVeY,3622
-scenario/_utils/utils.py,sha256=msQgUWaLh3U9jIIHmxkEbOaklga63AF0KJzsaKa_mZc,14008
-scenario/events/__init__.py,sha256=_autF1cMZYpNXE-kJNvvRb-H_hYqy4gOSSp2fT3Wi9k,1533
-scenario/events/event_bus.py,sha256=MThIMIaI2nj2CoegZazTNxeHbtl4_M7bW3vEAHz6R8g,7102
-scenario/events/event_reporter.py,sha256=cMh_5jA5hG3Q9IsoAgPJhxnIVs_M1Q0e2lgLTEK4oPc,3100
-scenario/events/events.py,sha256=jPXylwiADb0Bdk7u1YkAaU_jLebH7NW8J7SZI9JDTxw,6750
-scenario/events/messages.py,sha256=1QAkwDExdF6AHgXdEFhHwmCv3Mxu3j0AXIptMekc_bg,3299
-scenario/events/utils.py,sha256=yrTUTByeb0eAAQniQH7EyKs-usgGti8f17IemUyBZBw,3357
-langwatch_scenario-0.6.0.dist-info/METADATA,sha256=IvD9on4tP57ldmizFzfGQBtiCT6Z7yoz0trlCSPSW9M,14227
-langwatch_scenario-0.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-langwatch_scenario-0.6.0.dist-info/entry_points.txt,sha256=WlEnJ_gku0i18bIa3DSuGqXRX-QDQLe_s0YmRzK45TI,45
-langwatch_scenario-0.6.0.dist-info/top_level.txt,sha256=45Mn28aedJsetnBMB5xSmrJ-yo701QLH89Zlz4r1clE,9
-langwatch_scenario-0.6.0.dist-info/RECORD,,