langwatch-scenario 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

langwatch_scenario-0.4.0.dist-info/METADATA

@@ -0,0 +1,363 @@
+Metadata-Version: 2.4
+Name: langwatch-scenario
+Version: 0.4.0
+Summary: The end-to-end agent testing library
+Author-email: LangWatch Team <support@langwatch.ai>
+License: MIT
+Project-URL: Homepage, https://github.com/langwatch/scenario
+Project-URL: Bug Tracker, https://github.com/langwatch/scenario/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pytest>=8.1.1
+Requires-Dist: litellm>=1.49.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: termcolor>=2.4.0
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: joblib>=1.4.2
+Requires-Dist: wrapt>=1.17.2
+Requires-Dist: pytest-asyncio>=0.26.0
+Requires-Dist: rich<15.0.0,>=13.3.3
+Requires-Dist: pksuid>=1.1.2
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: commitizen; extra == "dev"
+Requires-Dist: pyright; extra == "dev"
+Requires-Dist: pydantic-ai; extra == "dev"
+Requires-Dist: function-schema; extra == "dev"
+
+![scenario](https://github.com/langwatch/scenario/raw/main/assets/scenario-wide.webp)
+
+<div align="center">
+<!-- Discord, PyPI, Docs, etc links -->
+</div>
+
+# Scenario
+
+Scenario is an Agent Testing Framework for testing AI agents through Simulation Testing.
+
+You define the conversation scenario and let it play out, it will keep chatting back and forth with _your_ agent until it reaches the desired goal or detects an unexpected behavior based on the criteria you defined.
+
+- Test your agents end-to-end conversations with specified scenarios to capture both happy paths and edge cases
+- Full flexibility of how much you want to guide the conversation, from fully scripted scenarios to completely automated simulations
+- Run evaluations at any point of the conversation, designed for multi-turn
+- Works in combination with any testing and LLM evaluation frameworks, completely agnostic
+- Works with any LLM and Agent Framework, easy integration
+
+[📺 Video Tutorial](https://www.youtube.com/watch?v=f8NLpkY0Av4)
+
+### See also
+
+- [Scenario TypeScript](https://github.com/langwatch/scenario-ts/)
+- [Scenario Go](https://github.com/langwatch/scenario-go/)
+
+## Example
+
+```python
+@pytest.mark.agent_test
+@pytest.mark.asyncio
+async def test_weather_agent():
+    # Integrate with your agent
+    class WeatherAgent(scenario.AgentAdapter):
+        async def call(self, input: scenario.AgentInput) -> scenario.AgentReturnTypes:
+            return weather_agent(input.messages)
+
+    # Define any custom assertions
+    def check_for_weather_tool_call(state: scenario.ScenarioState):
+        assert state.has_tool_call("get_current_weather")
+
+    # Run the scenario
+    result = await scenario.run(
+        name="checking the weather",
+        description="""
+            The user is planning a boat trip from Barcelona to Rome,
+            and is wondering what the weather will be like.
+        """,
+        agents=[
+            WeatherAgent(),
+            scenario.UserSimulatorAgent(model="openai/gpt-4.1-mini"),
+        ],
+        script=[
+            scenario.user(),
+            scenario.agent(),
+            check_for_weather_tool_call, # check for tool call after the first agent response
+            scenario.succeed(),
+        ],
+    )
+
+    # Assert the simulation was successful
+    assert result.success
+```
+
+> [!NOTE]
+> This is a very basic example, keep reading to see how to run a simulation completely script-free, using a Judge Agent to evaluate in real-time.
+
+Check out more examples in the [examples folder](./examples/).
+
+## Getting Started
+
+Install pytest and scenario:
+
+```bash
+pip install pytest langwatch-scenario
+```
+
+Now create your first scenario and save it as `tests/test_vegetarian_recipe_agent.py`, copy the full working example below:
+
+```python
+import pytest
+import scenario
+import litellm
+
+scenario.configure(default_model="openai/gpt-4.1-mini")
+
+
+@pytest.mark.agent_test
+@pytest.mark.asyncio
+async def test_vegetarian_recipe_agent():
+    class Agent(scenario.AgentAdapter):
+        async def call(self, input: scenario.AgentInput) -> scenario.AgentReturnTypes:
+            return vegetarian_recipe_agent(input.messages)
+
+    # Run a simulation scenario
+    result = await scenario.run(
+        name="dinner idea",
+        description="""
+            It's saturday evening, the user is very hungry and tired,
+            but have no money to order out, so they are looking for a recipe.
+        """,
+        agents=[
+            Agent(),
+            scenario.UserSimulatorAgent(),
+            scenario.JudgeAgent(
+                criteria=[
+                    "Agent should not ask more than two follow-up questions",
+                    "Agent should generate a recipe",
+                    "Recipe should include a list of ingredients",
+                    "Recipe should include step-by-step cooking instructions",
+                    "Recipe should be vegetarian and not include any sort of meat",
+                ]
+            ),
+        ],
+    )
+
+    # Assert for pytest to know whether the test passed
+    assert result.success
+
+
+# Example agent implementation
+import litellm
+
+
+@scenario.cache()
+def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
+    response = litellm.completion(
+        model="openai/gpt-4.1-mini",
+        messages=[
+            {
+                "role": "system",
+                "content": """
+                    You are a vegetarian recipe agent.
+                    Given the user request, ask AT MOST ONE follow-up question,
+                    then provide a complete recipe. Keep your responses concise and focused.
+                """,
+            },
+            *messages,
+        ],
+    )
+
+    return response.choices[0].message  # type: ignore
+```
+
+Create a `.env` file and put your OpenAI API key in it:
+
+```bash
+OPENAI_API_KEY=<your-api-key>
+```
+
+Now run it with pytest:
+
+```bash
+pytest -s tests/test_vegetarian_recipe_agent.py
+```
+
+This is how it will look like:
+
+[![asciicast](https://asciinema.org/a/nvO5GWGzqKTTCd8gtNSezQw11.svg)](https://asciinema.org/a/nvO5GWGzqKTTCd8gtNSezQw11)
+
+You can find the same code example in [examples/test_vegetarian_recipe_agent.py](examples/test_vegetarian_recipe_agent.py).
+
+## Script-free Simulation
+
+By providing a User Simulator Agent and a description of the Scenario, the simulated user will automatically generate messages to the agent until the scenario is successful or the maximum number of turns is reached.
+
+You can then use a Judge Agent to evaluate the scenario in real-time given certain criteria, at every turn, the Judge Agent will decide if it should let the simulation proceed or end it with a verdict.
+
+You can combine it with a script, to control for example the beginning of the conversation, or simply let it run scriptless, this is very useful to test an open case like a vibe coding assistant:
+
+```python
+result = await scenario.run(
+    name="dog walking startup landing page",
+    description="""
+        the user wants to create a new landing page for their dog walking startup
+
+        send the first message to generate the landing page, then a single follow up request to extend it, then give your final verdict
+    """,
+    agents=[
+        LovableAgentAdapter(template_path=template_path),
+        scenario.UserSimulatorAgent(),
+        scenario.JudgeAgent(
+            criteria=[
+                "agent reads the files before go and making changes",
+                "agent modified the index.css file, not only the Index.tsx file",
+                "agent created a comprehensive landing page",
+                "agent extended the landing page with a new section",
+                "agent should NOT say it can't read the file",
+                "agent should NOT produce incomplete code or be too lazy to finish",
+            ],
+        ),
+    ],
+    max_turns=5, # optional
+)
+```
+
+Check out the fully working Lovable Clone example in [examples/test_lovable_clone.py](examples/test_lovable_clone.py).
+
+## Full Control of the Conversation
+
+You can specify a script for guiding the scenario by passing a list of steps to the `script` field, those steps are simply arbitrary functions that take the current state of the scenario as an argument, so you can do things like:
+
+- Control what the user says, or let it be generated automatically
+- Control what the agent says, or let it be generated automatically
+- Add custom assertions, for example making sure a tool was called
+- Add a custom evaluation, from an external library
+- Let the simulation proceed for a certain number of turns, and evaluate at each new turn
+- Trigger the judge agent to decide on a verdict
+- Add arbitrary messages like mock tool calls in the middle of the conversation
+
+Everything is possible, using the same simple structure:
+
+```python
+@pytest.mark.agent_test
+@pytest.mark.asyncio
+async def test_ai_assistant_agent():
+    scenario = Scenario(
+        name="false assumptions",
+        description="""
+            The agent makes false assumption that the user is talking about an ATM bank, and user corrects it that they actually mean river banks
+        """,
+        agent=AiAssistantAgentAdapter,
+        criteria=[
+            "user should get good recommendations on river crossing",
+            "agent should NOT follow up about ATM recommendation after user has corrected them they are just hiking",
+        ],
+        max_turns=5,
+    )
+
+    def check_if_tool_was_called(state: ScenarioExecutor) -> None:
+        assert state.has_tool_call("web_search")
+
+    result = await scenario.script(
+        [
+            # Define existing history of messages
+            scenario.user("how do I safely approach a bank?"),
+
+            # Or let it be generate automatically
+            scenario.agent(),
+
+            # Add custom assertions, for example making sure a tool was called
+            check_if_tool_was_called,
+
+            # Another user message
+            scenario.user(),
+
+            # Let the simulation proceed for 2 more turns, print at every turn
+            scenario.proceed(
+                turns=2,
+                on_turn=lambda state: print(f"Turn {state.current_turn}: {state.messages}"),
+            ),
+
+            # Time to make a judgment call
+            scenario.judge(),
+        ]
+    ).run()
+
+    assert result.success
+```
+
+## Debug mode
+
+You can enable debug mode by setting the `debug` field to `True` in the `Scenario.configure` method or in the specific scenario you are running, or by passing the `--debug` flag to pytest.
+
+Debug mode allows you to see the messages in slow motion step by step, and intervene with your own inputs to debug your agent from the middle of the conversation.
+
+```python
+Scenario.configure(testing_agent=TestingAgent(model="openai/gpt-4o-mini"), debug=True)
+```
+
+or
+
+```bash
+pytest -s tests/test_vegetarian_recipe_agent.py --debug
+```
+
+## Cache
+
+Each time the scenario runs, the testing agent might chose a different input to start, this is good to make sure it covers the variance of real users as well, however we understand that the non-deterministic nature of it might make it less repeatable, costly and harder to debug. To solve for it, you can use the `cache_key` field in the `Scenario.configure` method or in the specific scenario you are running, this will make the testing agent give the same input for given the same scenario:
+
+```python
+Scenario.configure(testing_agent=TestingAgent(model="openai/gpt-4o-mini"), cache_key="42")
+```
+
+To bust the cache, you can simply pass a different `cache_key`, disable it, or delete the cache files located at `~/.scenario/cache`.
+
+To go a step further and fully cache the test end-to-end, you can also wrap the LLM calls or any other non-deterministic functions in your application side with the `@scenario_cache` decorator:
+
+```python
+class MyAgent:
+    @scenario_cache(ignore=["self"])
+    def invoke(self, message, context):
+        return client.chat.completions.create(
+            # ...
+        )
+```
+
+This will cache any function call you decorate when running the tests and make them repeatable, hashed by the function arguments, the scenario being executed, and the `cache_key` you provided. You can exclude arguments that should not be hashed for the cache key by naming them in the `ignore` argument.
+
+## Disable Output
+
+You can remove the `-s` flag from pytest to hide the output during test, which will only show up if the test fails. Alternatively, you can set `verbose=False` in the `Scenario.configure` method or in the specific scenario you are running.
+
+## Running in parallel
+
+As the number of your scenarios grows, you might want to run them in parallel to speed up your whole test suite. We suggest you to use the [pytest-asyncio-concurrent](https://pypi.org/project/pytest-asyncio-concurrent/) plugin to do so.
+
+Simply install the plugin from the link above, then replace the `@pytest.mark.asyncio` annotation in the tests with `@pytest.mark.asyncio_concurrent`, adding a group name to it to mark the group of scenarions that should be run in parallel together, e.g.:
+
+```python
+@pytest.mark.agent_test
+@pytest.mark.asyncio_concurrent(group="vegetarian_recipe_agent")
+async def test_vegetarian_recipe_agent():
+    # ...
+
+@pytest.mark.agent_test
+@pytest.mark.asyncio_concurrent(group="vegetarian_recipe_agent")
+async def test_user_is_very_hungry():
+    # ...
+```
+
+Those two scenarios should now run in parallel.
+
+## License
+
+MIT License

langwatch_scenario-0.4.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+scenario/__init__.py,sha256=oMh5le4c4sIN2K1Ylv2xnkyKHpcOzBeqvW58fTWAFlU,7794
+scenario/agent_adapter.py,sha256=pd3BdNUWna8h_9hykn1FvcyareMzUofQKKvXaAfQluY,4338
+scenario/cache.py,sha256=iPpMmjKruLnnxCeLnRiQjiH89LhcVIfQQXKH5etU_m4,6217
+scenario/config.py,sha256=AeDbKE-_Rrxkan64tDDDynaSNyijoIKHxWaRMqGd4oY,6121
+scenario/error_messages.py,sha256=6lEx3jBGMbPx0kG0eX5zoZE-ENVM3O_ZkIbVMlnidYs,3892
+scenario/judge_agent.py,sha256=7fKK_oevXzWKXDioBjHzgGSDpS0aby3oRcrc6oaip68,16973
+scenario/pytest_plugin.py,sha256=s2M2mll9JSCSWB5SKDQIWT5DOCvzZOo_8JCCfJzyy8k,12849
+scenario/scenario_executor.py,sha256=oz7Odv41HNLcNd_7sKUW-AKKdY-on_PyVLaxpvKjrGE,27211
+scenario/scenario_state.py,sha256=I_fWoY_LvNuKCBL-b62z5bQOAI25dx55FuZNWwtIeVs,7075
+scenario/script.py,sha256=7wsHZxdSgFaYLflkV6sysDxefkkag79mySR7yp7N3ug,12278
+scenario/types.py,sha256=CsexCupg2WUi4dToYF5RqFdNIHx1JhaRaRRBs78YVd0,9498
+scenario/user_simulator_agent.py,sha256=o8sZLMWOcTf7BKgPO_a5rPnC6GgdZQe3HujqwjPzjV8,9346
+scenario/utils.py,sha256=ryJYcMoSAjVzA_f5V6Mcga5GkipYbCzaYNNpBjAQI_g,16992
+langwatch_scenario-0.4.0.dist-info/METADATA,sha256=d9tNTNioHH5_1q8oIvIABaTgC6J9XmEJR4Tjim3sFks,13827
+langwatch_scenario-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+langwatch_scenario-0.4.0.dist-info/entry_points.txt,sha256=WlEnJ_gku0i18bIa3DSuGqXRX-QDQLe_s0YmRzK45TI,45
+langwatch_scenario-0.4.0.dist-info/top_level.txt,sha256=45Mn28aedJsetnBMB5xSmrJ-yo701QLH89Zlz4r1clE,9
+langwatch_scenario-0.4.0.dist-info/RECORD,,

scenario/__init__.py

@@ -1,24 +1,248 @@
 """
-Scenario: A testing library for conversational agents.
+Scenario: Agent Testing Framework through Simulation Testing
+
+Scenario is a comprehensive testing framework for AI agents that uses simulation testing
+to validate agent behavior through realistic conversations. It enables testing of both
+happy paths and edge cases by simulating user interactions and evaluating agent responses
+against configurable success criteria.
+
+Key Features:
+- End-to-end conversation testing with specified scenarios
+- Flexible control from fully scripted to completely automated simulations
+- Multi-turn evaluation designed for complex conversational agents
+- Works with any testing framework (pytest, unittest, etc.)
+- Framework-agnostic integration with any LLM or agent architecture
+- Built-in caching for deterministic and faster test execution
+
+Basic Usage:
+    ```python
+    import scenario
+
+    # Configure global settings
+    scenario.configure(default_model="openai/gpt-4.1-mini")
+
+    # Create your agent adapter
+    class MyAgent(scenario.AgentAdapter):
+        async def call(self, input: scenario.AgentInput) -> scenario.AgentReturnTypes:
+            return my_agent_function(input.last_new_user_message_str())
+
+    # Run a scenario test
+    result = await scenario.run(
+        name="customer service test",
+        description="Customer asks about billing, agent should help politely",
+        agents=[
+            MyAgent(),
+            scenario.UserSimulatorAgent(),
+            scenario.JudgeAgent(criteria=[
+                "Agent is polite and professional",
+                "Agent addresses the billing question",
+                "Agent provides clear next steps"
+            ])
+        ]
+    )
+
+    assert result.success
+    ```
+
+Advanced Usage:
+    ```python
+    # Script-controlled scenario with custom evaluations
+    def check_tool_usage(state: scenario.ScenarioState) -> None:
+        assert state.has_tool_call("get_customer_info")
+
+    result = await scenario.run(
+        name="scripted interaction",
+        description="Test specific conversation flow",
+        agents=[
+            MyAgent(),
+            scenario.UserSimulatorAgent(),
+            scenario.JudgeAgent(criteria=["Agent provides helpful response"])
+        ],
+        script=[
+            scenario.user("I have a billing question"),
+            scenario.agent(),
+            check_tool_usage,  # Custom assertion
+            scenario.proceed(turns=2),  # Let it continue automatically
+            scenario.succeed("All requirements met")
+        ]
+    )
+    ```
+
+Integration with Testing Frameworks:
+    ```python
+    import pytest
+
+    @pytest.mark.agent_test
+    @pytest.mark.asyncio
+    async def test_weather_agent():
+        result = await scenario.run(
+            name="weather query",
+            description="User asks about weather in a specific city",
+            agents=[
+                WeatherAgent(),
+                scenario.UserSimulatorAgent(),
+                scenario.JudgeAgent(criteria=["Provides accurate weather information"])
+            ]
+        )
+        assert result.success
+    ```
+
+For more examples and detailed documentation, visit: https://github.com/langwatch/scenario
 """
 
 # First import non-dependent modules
-from .result import ScenarioResult
+from .types import ScenarioResult, AgentInput, AgentRole, AgentReturnTypes
 from .config import ScenarioConfig
 
 # Then import modules with dependencies
-from .testing_agent import TestingAgent
-from .scenario import Scenario
+from .scenario_executor import ScenarioExecutor
+from .scenario_state import ScenarioState
+from .agent_adapter import AgentAdapter
+from .judge_agent import JudgeAgent
+from .user_simulator_agent import UserSimulatorAgent
 from .cache import scenario_cache
+from .script import message, user, agent, judge, proceed, succeed, fail
 
 # Import pytest plugin components
 from .pytest_plugin import pytest_configure, scenario_reporter
 
+run = ScenarioExecutor.run
+"""
+High-level interface for running scenario tests.
+
+This is the main entry point for executing scenario-based agent tests. It creates
+and runs a complete scenario simulation including user interactions, agent responses,
+and success evaluation.
+
+Args:
+    name: Human-readable name for the scenario
+    description: Detailed description that guides the simulation behavior
+    agents: List of agent adapters (agent under test, user simulator, judge)
+    max_turns: Maximum conversation turns before timeout (default: 10)
+    verbose: Show detailed output during execution
+    cache_key: Cache key for deterministic behavior across runs
+    debug: Enable debug mode for step-by-step execution
+    script: Optional script steps to control scenario flow
+
+Returns:
+    ScenarioResult containing test outcome, conversation history, and detailed analysis
+
+Example:
+    ```python
+    result = await scenario.run(
+        name="help request",
+        description="User needs help with a technical problem",
+        agents=[
+            MyAgentAdapter(),
+            scenario.UserSimulatorAgent(),
+            scenario.JudgeAgent(criteria=["Provides helpful response"])
+        ]
+    )
+
+    print(f"Test {'PASSED' if result.success else 'FAILED'}")
+    print(f"Reasoning: {result.reasoning}")
+    ```
+"""
+
+configure = ScenarioConfig.configure
+"""
+Set global configuration settings for all scenario executions.
+
+This function allows you to configure default behavior that will be applied
+to all scenarios unless explicitly overridden in individual scenario runs.
+
+Args:
+    default_model: Default LLM model identifier for user simulator and judge agents
+    max_turns: Maximum number of conversation turns before timeout (default: 10)
+    verbose: Enable verbose output during scenario execution
+    cache_key: Cache key for deterministic scenario behavior across runs
+    debug: Enable debug mode for step-by-step execution with user intervention
+
+Example:
+    ```python
+    # Set up global defaults
+    scenario.configure(
+        default_model="openai/gpt-4.1-mini",
+        max_turns=15,
+        verbose=True,
+        cache_key="my-test-suite-v1"
+    )
+
+    # All subsequent scenarios will use these defaults
+    result = await scenario.run(...)
+    ```
+"""
+
+default_config = ScenarioConfig.default_config
+"""
+Access to the current global configuration settings.
+
+This provides read-only access to the default configuration that has been
+set via scenario.configure(). Useful for debugging or conditional logic
+based on current settings.
+
+Example:
+    ```python
+    if scenario.default_config and scenario.default_config.debug:
+        print("Debug mode is enabled")
+    ```
+"""
+
+cache = scenario_cache
+"""
+Decorator for caching function calls during scenario execution.
+
+This decorator enables deterministic testing by caching LLM calls and other
+non-deterministic operations based on scenario configuration and function arguments.
+Results are cached when a cache_key is configured, making tests repeatable and faster.
+
+Args:
+    ignore: List of argument names to exclude from cache key computation
+
+Example:
+    ```python
+    class MyAgent:
+        @scenario.cache(ignore=["self"])
+        def invoke(self, message: str) -> str:
+            # This LLM call will be cached when cache_key is set
+            return llm_client.complete(model="gpt-4", prompt=message)
+
+    # Enable caching for deterministic tests
+    scenario.configure(cache_key="test-suite-v1")
+    ```
+"""
+
 __all__ = [
-    "Scenario",
-    "TestingAgent",
+    # Functions
+    "run",
+    "configure",
+    "default_config",
+    "cache",
+
+    # Script
+    "message",
+    "proceed",
+    "succeed",
+    "fail",
+    "judge",
+    "agent",
+    "user",
+
+    # Types
     "ScenarioResult",
+    "AgentInput",
+    "AgentRole",
     "ScenarioConfig",
+    "AgentReturnTypes",
+
+    # Classes
+    "ScenarioExecutor",
+    "ScenarioState",
+    "AgentAdapter",
+    "UserSimulatorAgent",
+    "JudgeAgent",
+
+    # Plugins
     "pytest_configure",
     "scenario_reporter",
     "scenario_cache",