langwatch-scenario 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

scenario/agent_adapter.py

@@ -0,0 +1,111 @@
+"""
+Agent adapter module for integrating custom agents with the Scenario framework.
+
+This module provides the abstract base class that users must implement to integrate
+their existing agents with the Scenario testing framework. The adapter pattern allows
+any agent implementation to work with the framework regardless of its underlying
+architecture or API.
+"""
+
+from abc import ABC, abstractmethod
+from typing import ClassVar
+
+from .types import AgentInput, AgentReturnTypes, AgentRole
+
+
+class AgentAdapter(ABC):
+    """
+    Abstract base class for integrating custom agents with the Scenario framework.
+
+    This adapter pattern allows you to wrap any existing agent implementation
+    (LLM calls, agent frameworks, or complex multi-step systems) to work with
+    the Scenario testing framework. The adapter receives structured input about
+    the conversation state and returns responses in a standardized format.
+
+    Attributes:
+        role: The role this agent plays in scenarios (USER, AGENT, or JUDGE)
+
+    Example:
+        ```python
+        import scenario
+        from my_agent_library import MyCustomAgent
+
+        class MyAgentAdapter(scenario.AgentAdapter):
+            def __init__(self):
+                self.agent = MyCustomAgent()
+
+            async def call(self, input: scenario.AgentInput) -> scenario.AgentReturnTypes:
+                # Get the latest user message
+                user_message = input.last_new_user_message_str()
+
+                # Call your existing agent
+                response = await self.agent.process(
+                    message=user_message,
+                    history=input.messages,
+                    thread_id=input.thread_id
+                )
+
+                # Return the response (can be string, message dict, or list of messages)
+                return response
+
+        # Use in a scenario
+        result = await scenario.run(
+            name="test my agent",
+            description="User asks for help with a coding problem",
+            agents=[
+                MyAgentAdapter(),
+                scenario.UserSimulatorAgent(),
+                scenario.JudgeAgent(criteria=["Provides helpful coding advice"])
+            ]
+        )
+        ```
+
+    Note:
+        - The call method must be async
+        - Return types can be: str, ChatCompletionMessageParam, List[ChatCompletionMessageParam], or ScenarioResult
+        - For stateful agents, use input.thread_id to maintain conversation context
+        - For stateless agents, use input.messages for the full conversation history
+    """
+    role: ClassVar[AgentRole] = AgentRole.AGENT
+
+    @abstractmethod
+    async def call(self, input: AgentInput) -> AgentReturnTypes:
+        """
+        Process the input and generate a response.
+
+        This is the main method that your agent implementation must provide.
+        It receives structured information about the current conversation state
+        and must return a response in one of the supported formats.
+
+        Args:
+            input: AgentInput containing conversation history, thread context, and scenario state
+
+        Returns:
+            AgentReturnTypes: The agent's response, which can be:
+                - str: Simple text response
+                - ChatCompletionMessageParam: Single OpenAI-format message
+                - List[ChatCompletionMessageParam]: Multiple messages for complex responses
+                - ScenarioResult: Direct test result (typically only used by judge agents)
+
+        Example:
+            ```python
+            async def call(self, input: AgentInput) -> AgentReturnTypes:
+                # Simple string response
+                user_msg = input.last_new_user_message_str()
+                return f"I understand you said: {user_msg}"
+
+                # Or structured message response
+                return {
+                    "role": "assistant",
+                    "content": "Let me help you with that...",
+                    "tool_calls": [...]  # If your agent uses tools
+                }
+
+                # Or multiple messages for complex interactions
+                return [
+                    {"role": "assistant", "content": "Let me search for that information..."},
+                    {"role": "assistant", "content": "Here's what I found: ..."}
+                ]
+            ```
+        """
+        pass

scenario/cache.py

@@ -1,3 +1,12 @@
+"""
+Caching module for deterministic scenario testing.
+
+This module provides caching functionality to make scenario tests deterministic
+and repeatable. It caches LLM calls and other non-deterministic operations based
+on scenario configuration and function arguments, enabling consistent test results
+across multiple runs.
+"""
+
 from contextvars import ContextVar
 import inspect
 import os
@@ -8,29 +17,99 @@ from joblib import Memory
 import json
 
 import wrapt
+from scenario.types import AgentInput
 from scenario.utils import SerializableWithStringFallback
 
 if TYPE_CHECKING:
-    from scenario.scenario import Scenario
+    from scenario.scenario_executor import ScenarioExecutor
 
 
 context_scenario = ContextVar("scenario")
 
+
 def get_cache() -> Memory:
-    """Get a cross-platform cache directory for scenario."""
+    """
+    Get a cross-platform cache directory for scenario execution.
+
+    Creates and returns a joblib Memory instance configured to use a
+    cross-platform cache directory. The cache location can be customized
+    via the SCENARIO_CACHE_DIR environment variable.
+
+    Returns:
+        Memory instance configured with the appropriate cache directory
+
+    Example:
+        ```python
+        # Default cache location: ~/.scenario/cache
+        cache = get_cache()
+
+        # Custom cache location via environment variable
+        os.environ["SCENARIO_CACHE_DIR"] = "/tmp/my_scenario_cache"
+        cache = get_cache()
+        ```
+    """
     home_dir = str(Path.home())
     cache_dir = os.path.join(home_dir, ".scenario", "cache")
 
     return Memory(location=os.environ.get("SCENARIO_CACHE_DIR", cache_dir), verbose=0)
 
+
 memory = get_cache()
 
+
 def scenario_cache(ignore=[]):
+    """
+    Decorator for caching function calls during scenario execution.
+
+    This decorator caches function calls based on the scenario's cache_key,
+    scenario configuration, and function arguments. It enables deterministic
+    testing by ensuring the same inputs always produce the same outputs,
+    making tests repeatable and faster on subsequent runs.
+
+    Args:
+        ignore: List of argument names to exclude from the cache key computation.
+                Commonly used to ignore 'self' for instance methods or other
+                non-deterministic arguments.
+
+    Returns:
+        Decorator function that can be applied to any function or method
+
+    Example:
+        ```python
+        import scenario
+
+        class MyAgent:
+            @scenario.cache(ignore=["self"])
+            def invoke(self, message: str, context: dict) -> str:
+                # This LLM call will be cached
+                response = llm_client.complete(
+                    model="gpt-4",
+                    messages=[{"role": "user", "content": message}]
+                )
+                return response.choices[0].message.content
+
+        # Usage in tests
+        scenario.configure(cache_key="my-test-suite-v1")
+
+        # First run: makes actual LLM calls and caches results
+        result1 = await scenario.run(...)
+
+        # Second run: uses cached results, much faster
+        result2 = await scenario.run(...)
+        # result1 and result2 will be identical
+        ```
+
+    Note:
+        - Caching only occurs when a cache_key is set in the scenario configuration
+        - The cache key is computed from scenario config, function arguments, and cache_key
+        - AgentInput objects are specially handled to exclude thread_id from caching
+        - Both sync and async functions are supported
+    """
     @wrapt.decorator
     def wrapper(wrapped: Callable, instance=None, args=[], kwargs={}):
-        scenario: "Scenario" = context_scenario.get()
+        scenario: "ScenarioExecutor" = context_scenario.get()
 
-        if not scenario.cache_key:
+        if not scenario.config.cache_key:
             return wrapped(*args, **kwargs)
 
         sig = inspect.signature(wrapped)
@@ -43,20 +122,65 @@ def scenario_cache(ignore=[]):
             if arg in all_args:
                 del all_args[arg]
 
+        for key, value in all_args.items():
+            if isinstance(value, AgentInput):
+                scenario_state = value.scenario_state.model_dump(exclude={"thread_id"})
+                all_args[key] = value.model_dump(exclude={"thread_id"})
+                all_args[key]["scenario_state"] = scenario_state
+
         cache_key = json.dumps(
             {
-                "cache_key": scenario.cache_key,
-                "scenario": scenario.model_dump(exclude={"agent"}),
+                "cache_key": scenario.config.cache_key,
+                "scenario": scenario.config.model_dump(exclude={"agents"}),
                 "all_args": all_args,
             },
             cls=SerializableWithStringFallback,
         )
 
-        return _cached_call(wrapped, args, kwargs, cache_key=cache_key)
+        # if is an async function, we need to wrap it in a sync function
+        if inspect.iscoroutinefunction(wrapped):
+            return _async_cached_call(wrapped, args, kwargs, cache_key=cache_key)
+        else:
+            return _cached_call(wrapped, args, kwargs, cache_key=cache_key)
 
     return wrapper
 
 
 @memory.cache(ignore=["func", "args", "kwargs"])
 def _cached_call(func: Callable, args, kwargs, cache_key):
-    return func(*args, **kwargs)
+    """
+    Internal function for caching synchronous function calls.
+
+    This function is used internally by the scenario_cache decorator
+    to cache synchronous function calls using joblib.Memory.
+
+    Args:
+        func: The function to call and cache
+        args: Positional arguments for the function
+        kwargs: Keyword arguments for the function
+        cache_key: Cache key for deterministic caching
+
+    Returns:
+        The result of calling func(*args, **kwargs)
+    """
+    return func(*args, **kwargs)
+
+
+@memory.cache(ignore=["func", "args", "kwargs"])
+async def _async_cached_call(func: Callable, args, kwargs, cache_key):
+    """
+    Internal function for caching asynchronous function calls.
+
+    This function is used internally by the scenario_cache decorator
+    to cache asynchronous function calls using joblib.Memory.
+
+    Args:
+        func: The async function to call and cache
+        args: Positional arguments for the function
+        kwargs: Keyword arguments for the function
+        cache_key: Cache key for deterministic caching
+
+    Returns:
+        The result of calling await func(*args, **kwargs)
+    """
+    return await func(*args, **kwargs)

scenario/config.py

@@ -1,28 +1,183 @@
 """
 Configuration module for Scenario.
+
+This module provides configuration classes for customizing the behavior of the
+Scenario testing framework, including LLM model settings, execution parameters,
+and debugging options.
 """
 
-from typing import Optional, Union
+from typing import Optional, Union, ClassVar
 from pydantic import BaseModel
 
-from scenario.testing_agent import TestingAgent
+class ModelConfig(BaseModel):
+    """
+    Configuration for LLM model settings.
+
+    This class encapsulates all the parameters needed to configure an LLM model
+    for use with user simulator and judge agents in the Scenario framework.
+
+    Attributes:
+        model: The model identifier (e.g., "openai/gpt-4.1-mini", "anthropic/claude-3-sonnet")
+        api_key: Optional API key for the model provider
+        temperature: Sampling temperature for response generation (0.0 = deterministic, 1.0 = creative)
+        max_tokens: Maximum number of tokens to generate in responses
+
+    Example:
+        ```python
+        model_config = ModelConfig(
+            model="openai/gpt-4.1-mini",
+            api_key="your-api-key",
+            temperature=0.1,
+            max_tokens=1000
+        )
+        ```
+    """
+    model: str
+    api_key: Optional[str] = None
+    temperature: float = 0.0
+    max_tokens: Optional[int] = None
+
 
 class ScenarioConfig(BaseModel):
     """
-    Configuration class for the Scenario library.
+    Global configuration class for the Scenario testing framework.
 
-    This allows users to set global configuration settings for the library,
-    such as the LLM provider and model to use for the testing agent.
+    This class allows users to set default behavior and parameters that apply
+    to all scenario executions, including the LLM model to use for simulator
+    and judge agents, execution limits, and debugging options.
+
+    Attributes:
+        default_model: Default LLM model configuration for agents (can be string or ModelConfig)
+        max_turns: Maximum number of conversation turns before scenario times out
+        verbose: Whether to show detailed output during execution (True/False or verbosity level)
+        cache_key: Key for caching scenario results to ensure deterministic behavior
+        debug: Whether to enable debug mode with step-by-step interaction
+
+    Example:
+        ```python
+        # Configure globally for all scenarios
+        scenario.configure(
+            default_model="openai/gpt-4.1-mini",
+            max_turns=15,
+            verbose=True,
+            cache_key="my-test-suite-v1",
+            debug=False
+        )
+
+        # Or create a specific config instance
+        config = ScenarioConfig(
+            default_model=ModelConfig(
+                model="openai/gpt-4.1-mini",
+                temperature=0.2
+            ),
+            max_turns=20
+        )
+        ```
     """
 
-    testing_agent: Optional[TestingAgent] = None
+    default_model: Optional[Union[str, ModelConfig]] = None
     max_turns: Optional[int] = 10
     verbose: Optional[Union[bool, int]] = True
     cache_key: Optional[str] = None
     debug: Optional[bool] = False
 
+    default_config: ClassVar[Optional["ScenarioConfig"]] = None
+
+    @classmethod
+    def configure(
+        cls,
+        default_model: Optional[str] = None,
+        max_turns: Optional[int] = None,
+        verbose: Optional[Union[bool, int]] = None,
+        cache_key: Optional[str] = None,
+        debug: Optional[bool] = None,
+    ) -> None:
+        """
+        Set global configuration settings for all scenario executions.
+
+        This method allows you to configure default behavior that will be applied
+        to all scenarios unless explicitly overridden in individual scenario runs.
+
+        Args:
+            default_model: Default LLM model identifier for user simulator and judge agents
+            max_turns: Maximum number of conversation turns before timeout (default: 10)
+            verbose: Enable verbose output during scenario execution
+            cache_key: Cache key for deterministic scenario behavior across runs
+            debug: Enable debug mode for step-by-step execution with user intervention
+
+        Example:
+            ```python
+            import scenario
+
+            # Set up default configuration
+            scenario.configure(
+                default_model="openai/gpt-4.1-mini",
+                max_turns=15,
+                verbose=True,
+                debug=False
+            )
+
+            # All subsequent scenario runs will use these defaults
+            result = await scenario.run(
+                name="my test",
+                description="Test scenario",
+                agents=[my_agent, scenario.UserSimulatorAgent(), scenario.JudgeAgent()]
+            )
+            ```
+        """
+        existing_config = cls.default_config or ScenarioConfig()
+
+        cls.default_config = existing_config.merge(
+            ScenarioConfig(
+                default_model=default_model,
+                max_turns=max_turns,
+                verbose=verbose,
+                cache_key=cache_key,
+                debug=debug,
+            )
+        )
+
     def merge(self, other: "ScenarioConfig") -> "ScenarioConfig":
-        return ScenarioConfig(**{
-            **self.model_dump(),
-            **other.model_dump(exclude_none=True),
-        })
+        """
+        Merge this configuration with another configuration.
+
+        Values from the other configuration will override values in this
+        configuration where they are not None.
+
+        Args:
+            other: Another ScenarioConfig instance to merge with
+
+        Returns:
+            A new ScenarioConfig instance with merged values
+
+        Example:
+            ```python
+            base_config = ScenarioConfig(max_turns=10, verbose=True)
+            override_config = ScenarioConfig(max_turns=20)
+
+            merged = base_config.merge(override_config)
+            # Result: max_turns=20, verbose=True
+            ```
+        """
+        return ScenarioConfig(
+            **{
+                **self.items(),
+                **other.items(),
+            }
+        )
+
+    def items(self):
+        """
+        Get configuration items as a dictionary.
+
+        Returns:
+            Dictionary of configuration key-value pairs, excluding None values
+
+        Example:
+            ```python
+            config = ScenarioConfig(max_turns=15, verbose=True)
+            items = config.items()
+            # Result: {"max_turns": 15, "verbose": True}
+            ```
+        """
+        return {k: getattr(self, k) for k in self.model_dump(exclude_none=True).keys()}

scenario/error_messages.py

@@ -3,74 +3,102 @@ from typing import Any
 import termcolor
 
 
-default_config_error_message = f"""
+def agent_not_configured_error_message(class_name: str):
+    return f"""
 
- {termcolor.colored("->", "cyan")} Please set a default config with at least a testing_agent model for running your scenarios at the top of your test file, for example:
+ {termcolor.colored("->", "cyan")} {class_name} was initialized without a model, please set the model when defining the testing agent, for example:
 
-    from scenario import Scenario, TestingAgent
+    {class_name}(model="openai/gpt-4.1-mini")
+    {termcolor.colored("^" * (29 + len(class_name)), "green")}
 
-    Scenario.configure(testing_agent=TestingAgent(model="openai/gpt-4o-mini"))
-    {termcolor.colored("^" * 74, "green")}
+ {termcolor.colored("->", "cyan")} Alternatively, you can set the default model globally, for example:
 
-    @pytest.mark.agent_test
-    def test_vegetarian_recipe_agent():
-        scenario = Scenario(
-            # ...
-        )
-        result = scenario.run()
+    scenario.configure(default_model="openai/gpt-4.1-mini")
+    {termcolor.colored("^" * 55, "green")}
+"""
 
-        assert result.success
 
+def message_return_error_message(got: Any, class_name: str):
+    got_ = repr(got)
+    if len(got_) > 100:
+        got_ = got_[:100] + "..."
+
+    return f"""
+ {termcolor.colored("->", "cyan")} On the {termcolor.colored("call", "green")} method of the {class_name} agent adapter, you returned:
 
- {termcolor.colored("->", "cyan")} Alternatively, you can set the config specifically for this scenario:
+{indent(got_, ' ' * 4)}
 
-    from scenario import Scenario, TestingAgent
+ {termcolor.colored("->", "cyan")} But the adapter should return either a string, a dict on the OpenAI messages format, or a list of messages in the OpenAI messages format so the testing agent can understand what happened. For example:
 
-    @pytest.mark.agent_test
-    def test_vegetarian_recipe_agent():
-        scenario = Scenario(
-            # ...
-            testing_agent=TestingAgent(model="openai/gpt-4o-mini")
-            {termcolor.colored("^" * 54, "green")}
-        )
-        result = scenario.run()
+    class MyAgentAdapter(ScenarioAgentAdapter):
+        async def call(self, input: AgentInput) -> AgentReturnTypes:
+            response = call_my_agent(message)
 
-        assert result.success
-                          """
+            return response.output_text
+            {termcolor.colored("^" * 27, "green")}
 
+ {termcolor.colored("->", "cyan")} Alternatively, you can return a list of messages in OpenAI messages format, this is useful for capturing tool calls and other before the final response:
 
-def message_return_error_message(got: Any):
-    got_ = got.__repr__()
+    class MyAgentAdapter(ScenarioAgentAdapter):
+        async def call(self, input: AgentInput) -> AgentReturnTypes:
+            response = call_my_agent(message)
+
+            return [
+                {{"role": "assistant", "content": response.output_text}},
+                {termcolor.colored("^" * 55, "green")}
+            ]
+"""
+
+
+def message_invalid_agent_type(got: Any):
+    got_ = repr(got)
     if len(got_) > 100:
         got_ = got_[:100] + "..."
 
     return f"""
- {termcolor.colored("->", "cyan")} Your agent returned:
+ {termcolor.colored("->", "cyan")} The {termcolor.colored("agent", "green")} argument of Scenario needs to receive a class that inherits from {termcolor.colored("ScenarioAgentAdapter", "green")}, but you passed:
 
 {indent(got_, ' ' * 4)}
 
- {termcolor.colored("->", "cyan")} But your agent should return a dict with either a "message" string key or a "messages" key in OpenAI messages format so the testing agent can understand what happened. For example:
-
-    def my_agent_under_test(message, context):
-        response = call_my_agent(message)
+ {termcolor.colored("->", "cyan")} Instead, wrap your agent in a ScenarioAgentAdapter subclass. For example:
 
-        return {{
-            "message": response.output_text
-            {termcolor.colored("^" * 31, "green")}
-        }}
+    class MyAgentAdapter(ScenarioAgentAdapter):
+    {termcolor.colored("^" * 43, "green")}
+        async def call(self, input: AgentInput) -> AgentReturnTypes:
+            response = call_my_agent(message)
 
- {termcolor.colored("->", "cyan")} Alternatively, you can return a list of messages in OpenAI messages format, you can also optionally provide extra artifacts:
+            return response.output_text
 
-    def my_agent_under_test(message, context):
-        response = call_my_agent(message)
+ {termcolor.colored("->", "cyan")} And then you can use that on your scenario definition:
 
-        return {{
-            "messages": [
-                {{"role": "assistant", "content": response}}
-                {termcolor.colored("^" * 42, "green")}
+    @pytest.mark.agent_test
+    def test_my_agent():
+        scenario = Scenario(
+            name="first scenario",
+            description=\"\"\"
+                Example scenario description to test your agent.
+            \"\"\",
+            agent=MyAgentAdapter,
+            {termcolor.colored("^" * 20, "green")}
+            criteria=[
+                "Requirement One",
+                "Requirement Two",
             ],
-            "extra": {{
-                # ... optional extra artifacts
-            }}
-        }}
-                          """
+        )
+        result = scenario.run()
+
+        assert result.success
+"""
+
+
+def agent_response_not_awaitable(class_name: str):
+    return f"""
+ {termcolor.colored("->", "cyan")} The {termcolor.colored("call", "green")} method of the {class_name} agent adapter returned a non-awaitable response, you probably forgot to add the {termcolor.colored("async", "green")} keyword to the method definition, make sure your code looks like this:
+
+    class {class_name}(ScenarioAgentAdapter):
+        async def call(self, input: AgentInput) -> AgentReturnTypes:
+        {termcolor.colored("^" * 5, "green")}
+            response = call_my_agent(message)
+
+            return response.output_text
+"""