langwatch-scenario 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

{langwatch_scenario-0.1.2.dist-info → langwatch_scenario-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.1.2
+Version: 0.1.3
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT
@@ -175,7 +175,7 @@ You can find a fully working Lovable Clone example in [examples/test_lovable_clo
 
 ## Debug mode
 
-You can enable debug mode by setting the `debug` field to `True` in the `Scenario.configure` method or in the specific scenario you are running.
+You can enable debug mode by setting the `debug` field to `True` in the `Scenario.configure` method or in the specific scenario you are running, or by passing the `--debug` flag to pytest.
 
 Debug mode allows you to see the messages in slow motion step by step, and intervene with your own inputs to debug your agent from the middle of the conversation.
 
@@ -183,6 +183,12 @@ Debug mode allows you to see the messages in slow motion step by step, and inter
 Scenario.configure(testing_agent=TestingAgent(model="openai/gpt-4o-mini"), debug=True)
 ```
 
+or
+
+```bash
+pytest -s tests/test_vegetarian_recipe_agent.py --debug
+```
+
 ## Cache
 
 Each time the scenario runs, the testing agent might chose a different input to start, this is good to make sure it covers the variance of real users as well, however we understand that the non-deterministic nature of it might make it less repeatable, costly and harder to debug. To solve for it, you can use the `cache_key` field in the `Scenario.configure` method or in the specific scenario you are running, this will make the testing agent give the same input for given the same scenario:

{langwatch_scenario-0.1.2.dist-info → langwatch_scenario-0.1.3.dist-info}/RECORD

@@ -2,14 +2,14 @@ scenario/__init__.py,sha256=LfCjOpbn55jYBBZHyMSZtRAWeCDFn4z4OhAyFnu8aMg,602
 scenario/cache.py,sha256=sYu16SAf-BnVYkWSlEDzpyynJGIQyNYsgMXPgCqEnmk,1719
 scenario/config.py,sha256=5UVBmuQDtni0Yu00bMh5p0xMGsrymYVRftXBGTsi2fI,802
 scenario/error_messages.py,sha256=ZMcAOKJmKaLIinMZ0yBIOgDhPfeJH0uZxIEmolRArtc,2344
-scenario/pytest_plugin.py,sha256=ydtQxaN09qzoo12nNT8BQY_UPPHAt-AH92HWnPEN6bI,5212
+scenario/pytest_plugin.py,sha256=BuBbyKLa-t9AFVn9EETl7OvGSt__dFO7KnbZynfS1UM,5789
 scenario/result.py,sha256=SGF8uYNtkP7cJy4KsshUozZRevmdiyX2TFzr6VreTv8,2717
-scenario/scenario.py,sha256=MqsyiNue1KC4mtvTHnJqJ6Fj3u0TTAdAYann8P8WBBQ,4010
+scenario/scenario.py,sha256=tYn3Y1sK6_7pg7hFb_5w0TW6nun-za_4F8kqcnrXXU4,4077
 scenario/scenario_executor.py,sha256=c8xV6GoJgO2JoZBWpYPQN5YwwQ3G9iJUtXV9UGSf1q8,7919
 scenario/testing_agent.py,sha256=eS-c_io5cHgzJ88wwRvU_vve-pmB2HsGWN6qwlq0sPg,10865
 scenario/utils.py,sha256=tMESosrxesA1B5zZB3IJ-sNSXDmnpNNib-DHobveVLA,3918
-langwatch_scenario-0.1.2.dist-info/METADATA,sha256=La0j89kCoJpCriv3R8Sx5aqKgZy_iC-WNF-NqZzptfk,8684
-langwatch_scenario-0.1.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-langwatch_scenario-0.1.2.dist-info/entry_points.txt,sha256=WlEnJ_gku0i18bIa3DSuGqXRX-QDQLe_s0YmRzK45TI,45
-langwatch_scenario-0.1.2.dist-info/top_level.txt,sha256=45Mn28aedJsetnBMB5xSmrJ-yo701QLH89Zlz4r1clE,9
-langwatch_scenario-0.1.2.dist-info/RECORD,,
+langwatch_scenario-0.1.3.dist-info/METADATA,sha256=7OIolGcZ3fkCXFmE6JHkckVCeJb1r3yYSYveJ6iE9zw,8801
+langwatch_scenario-0.1.3.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+langwatch_scenario-0.1.3.dist-info/entry_points.txt,sha256=WlEnJ_gku0i18bIa3DSuGqXRX-QDQLe_s0YmRzK45TI,45
+langwatch_scenario-0.1.3.dist-info/top_level.txt,sha256=45Mn28aedJsetnBMB5xSmrJ-yo701QLH89Zlz4r1clE,9
+langwatch_scenario-0.1.3.dist-info/RECORD,,

scenario/pytest_plugin.py

@@ -11,14 +11,16 @@ from scenario.result import ScenarioResult
 
 from .scenario import Scenario
 
+
 class ScenarioReporterResults(TypedDict):
     scenario: Scenario
     result: ScenarioResult
 
+
 # ScenarioReporter class definition moved outside the fixture for global use
 class ScenarioReporter:
     def __init__(self):
-        self.results : list[ScenarioReporterResults] = []
+        self.results: list[ScenarioReporterResults] = []
 
     def add_result(self, scenario, result):
         """Add a test result to the reporter."""
@@ -83,7 +85,12 @@ class ScenarioReporter:
                 f"\n{idx}. {scenario.description} - {colored(status, status_color, attrs=['bold'])}{time}"
             )
 
-            print(colored(f"   Reasoning: {result.reasoning}", "green" if result.success else "red"))
+            print(
+                colored(
+                    f"   Reasoning: {result.reasoning}",
+                    "green" if result.success else "red",
+                )
+            )
 
             if hasattr(result, "met_criteria") and result.met_criteria:
                 criteria_count = len(result.met_criteria)
@@ -119,6 +126,10 @@ def pytest_configure(config):
         "markers", "agent_test: mark test as an agent scenario test"
     )
 
+    if config.getoption("--debug"):
+        print(colored("\nScenario debug mode enabled (--debug).", "yellow"))
+        Scenario.configure(verbose=True, debug=True)
+
     # Create a global reporter instance
     config._scenario_reporter = ScenarioReporter()
 
@@ -128,7 +139,12 @@ def pytest_configure(config):
         result = await original_run(self, *args, **kwargs)
 
         # Always report to the global reporter
-        config._scenario_reporter.add_result(self, result)
+        # Ensure the reporter exists before adding result
+        if hasattr(config, "_scenario_reporter"):
+            config._scenario_reporter.add_result(self, result)
+        else:
+            # Handle case where reporter might not be initialized (should not happen with current setup)
+            print(colored("Warning: Scenario reporter not found during run.", "yellow"))
 
         return result
 

scenario/scenario.py

@@ -105,6 +105,7 @@ class Scenario(ScenarioConfig):
         max_turns: Optional[int] = None,
         verbose: Optional[Union[bool, int]] = None,
         cache_key: Optional[str] = None,
+        debug: Optional[bool] = None,
     ) -> None:
         existing_config = getattr(cls, "default_config", ScenarioConfig())
 
@@ -114,5 +115,6 @@ class Scenario(ScenarioConfig):
                 max_turns=max_turns,
                 verbose=verbose,
                 cache_key=cache_key,
+                debug=debug,
             )
         )