langwatch-scenario 0.7.3__tar.gz → 0.7.8__tar.gz

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.3
+Version: 0.7.8
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT
@@ -30,12 +30,12 @@ Requires-Dist: pksuid>=1.1.2
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: rx>=3.2.0
 Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: pydantic-settings>=2.9.1
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
-Requires-Dist: commitizen; extra == "dev"
 Requires-Dist: pyright; extra == "dev"
 Requires-Dist: pydantic-ai; extra == "dev"
 Requires-Dist: function-schema; extra == "dev"
@@ -88,7 +88,7 @@ result = await scenario.run(
     # Define the agents that will play this simulation
     agents=[
         WeatherAgent(),
-        scenario.UserSimulatorAgent(model="openai/gpt-4.1-mini"),
+        scenario.UserSimulatorAgent(model="openai/gpt-4.1"),
     ],
 
     # (Optional) Control the simulation
@@ -159,7 +159,7 @@ import pytest
 import scenario
 import litellm
 
-scenario.configure(default_model="openai/gpt-4.1-mini")
+scenario.configure(default_model="openai/gpt-4.1")
 
 
 @pytest.mark.agent_test
@@ -189,6 +189,7 @@ async def test_vegetarian_recipe_agent():
                 ]
             ),
         ],
+        set_id="python-examples",
     )
 
     # Assert for pytest to know whether the test passed
@@ -202,7 +203,7 @@ import litellm
 @scenario.cache()
 def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
     response = litellm.completion(
-        model="openai/gpt-4.1-mini",
+        model="openai/gpt-4.1",
         messages=[
             {
                 "role": "system",
@@ -227,17 +228,17 @@ def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
 Save it as `tests/vegetarian-recipe-agent.test.ts`:
 
 ```typescript
+import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario";
 import { openai } from "@ai-sdk/openai";
-import * as scenario from "@langwatch/scenario";
 import { generateText } from "ai";
 import { describe, it, expect } from "vitest";
 
 describe("Vegetarian Recipe Agent", () => {
-  const agent: scenario.AgentAdapter = {
-    role: scenario.AgentRole.AGENT,
+  const agent: AgentAdapter = {
+    role: AgentRole.AGENT,
     call: async (input) => {
       const response = await generateText({
-        model: openai("gpt-4.1-mini"),
+        model: openai("gpt-4.1"),
         messages: [
           {
             role: "system",
@@ -258,7 +259,7 @@ describe("Vegetarian Recipe Agent", () => {
         agent,
         scenario.userSimulatorAgent(),
         scenario.judgeAgent({
-          model: openai("gpt-4.1-mini"),
+          model: openai("gpt-4.1"),
           criteria: [
             "Agent should not ask more than two follow-up questions",
             "Agent should generate a recipe",
@@ -268,6 +269,7 @@ describe("Vegetarian Recipe Agent", () => {
           ],
         }),
       ],
+      setId: "javascript-examples",
     });
     expect(result.success).toBe(true);
   });
@@ -417,7 +419,7 @@ You can enable debug mode by setting the `debug` field to `True` in the `Scenari
 Debug mode allows you to see the messages in slow motion step by step, and intervene with your own inputs to debug your agent from the middle of the conversation.
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", debug=True)
+scenario.configure(default_model="openai/gpt-4.1", debug=True)
 ```
 
 or
@@ -431,7 +433,7 @@ pytest -s tests/test_vegetarian_recipe_agent.py --debug
 Each time the scenario runs, the testing agent might chose a different input to start, this is good to make sure it covers the variance of real users as well, however we understand that the non-deterministic nature of it might make it less repeatable, costly and harder to debug. To solve for it, you can use the `cache_key` field in the `Scenario.configure` method or in the specific scenario you are running, this will make the testing agent give the same input for given the same scenario:
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", cache_key="42")
+scenario.configure(default_model="openai/gpt-4.1", cache_key="42")
 ```
 
 To bust the cache, you can simply pass a different `cache_key`, disable it, or delete the cache files located at `~/.scenario/cache`.

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/README.md

@@ -44,7 +44,7 @@ result = await scenario.run(
     # Define the agents that will play this simulation
     agents=[
         WeatherAgent(),
-        scenario.UserSimulatorAgent(model="openai/gpt-4.1-mini"),
+        scenario.UserSimulatorAgent(model="openai/gpt-4.1"),
     ],
 
     # (Optional) Control the simulation
@@ -115,7 +115,7 @@ import pytest
 import scenario
 import litellm
 
-scenario.configure(default_model="openai/gpt-4.1-mini")
+scenario.configure(default_model="openai/gpt-4.1")
 
 
 @pytest.mark.agent_test
@@ -145,6 +145,7 @@ async def test_vegetarian_recipe_agent():
                 ]
             ),
         ],
+        set_id="python-examples",
     )
 
     # Assert for pytest to know whether the test passed
@@ -158,7 +159,7 @@ import litellm
 @scenario.cache()
 def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
     response = litellm.completion(
-        model="openai/gpt-4.1-mini",
+        model="openai/gpt-4.1",
         messages=[
             {
                 "role": "system",
@@ -183,17 +184,17 @@ def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
 Save it as `tests/vegetarian-recipe-agent.test.ts`:
 
 ```typescript
+import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario";
 import { openai } from "@ai-sdk/openai";
-import * as scenario from "@langwatch/scenario";
 import { generateText } from "ai";
 import { describe, it, expect } from "vitest";
 
 describe("Vegetarian Recipe Agent", () => {
-  const agent: scenario.AgentAdapter = {
-    role: scenario.AgentRole.AGENT,
+  const agent: AgentAdapter = {
+    role: AgentRole.AGENT,
     call: async (input) => {
       const response = await generateText({
-        model: openai("gpt-4.1-mini"),
+        model: openai("gpt-4.1"),
         messages: [
           {
             role: "system",
@@ -214,7 +215,7 @@ describe("Vegetarian Recipe Agent", () => {
         agent,
         scenario.userSimulatorAgent(),
         scenario.judgeAgent({
-          model: openai("gpt-4.1-mini"),
+          model: openai("gpt-4.1"),
           criteria: [
             "Agent should not ask more than two follow-up questions",
             "Agent should generate a recipe",
@@ -224,6 +225,7 @@ describe("Vegetarian Recipe Agent", () => {
           ],
         }),
       ],
+      setId: "javascript-examples",
     });
     expect(result.success).toBe(true);
   });
@@ -373,7 +375,7 @@ You can enable debug mode by setting the `debug` field to `True` in the `Scenari
 Debug mode allows you to see the messages in slow motion step by step, and intervene with your own inputs to debug your agent from the middle of the conversation.
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", debug=True)
+scenario.configure(default_model="openai/gpt-4.1", debug=True)
 ```
 
 or
@@ -387,7 +389,7 @@ pytest -s tests/test_vegetarian_recipe_agent.py --debug
 Each time the scenario runs, the testing agent might chose a different input to start, this is good to make sure it covers the variance of real users as well, however we understand that the non-deterministic nature of it might make it less repeatable, costly and harder to debug. To solve for it, you can use the `cache_key` field in the `Scenario.configure` method or in the specific scenario you are running, this will make the testing agent give the same input for given the same scenario:
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", cache_key="42")
+scenario.configure(default_model="openai/gpt-4.1", cache_key="42")
 ```
 
 To bust the cache, you can simply pass a different `cache_key`, disable it, or delete the cache files located at `~/.scenario/cache`.

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/langwatch_scenario.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.3
+Version: 0.7.8
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT
@@ -30,12 +30,12 @@ Requires-Dist: pksuid>=1.1.2
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: rx>=3.2.0
 Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: pydantic-settings>=2.9.1
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
-Requires-Dist: commitizen; extra == "dev"
 Requires-Dist: pyright; extra == "dev"
 Requires-Dist: pydantic-ai; extra == "dev"
 Requires-Dist: function-schema; extra == "dev"
@@ -88,7 +88,7 @@ result = await scenario.run(
     # Define the agents that will play this simulation
     agents=[
         WeatherAgent(),
-        scenario.UserSimulatorAgent(model="openai/gpt-4.1-mini"),
+        scenario.UserSimulatorAgent(model="openai/gpt-4.1"),
     ],
 
     # (Optional) Control the simulation
@@ -159,7 +159,7 @@ import pytest
 import scenario
 import litellm
 
-scenario.configure(default_model="openai/gpt-4.1-mini")
+scenario.configure(default_model="openai/gpt-4.1")
 
 
 @pytest.mark.agent_test
@@ -189,6 +189,7 @@ async def test_vegetarian_recipe_agent():
                 ]
             ),
         ],
+        set_id="python-examples",
     )
 
     # Assert for pytest to know whether the test passed
@@ -202,7 +203,7 @@ import litellm
 @scenario.cache()
 def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
     response = litellm.completion(
-        model="openai/gpt-4.1-mini",
+        model="openai/gpt-4.1",
         messages=[
             {
                 "role": "system",
@@ -227,17 +228,17 @@ def vegetarian_recipe_agent(messages) -> scenario.AgentReturnTypes:
 Save it as `tests/vegetarian-recipe-agent.test.ts`:
 
 ```typescript
+import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario";
 import { openai } from "@ai-sdk/openai";
-import * as scenario from "@langwatch/scenario";
 import { generateText } from "ai";
 import { describe, it, expect } from "vitest";
 
 describe("Vegetarian Recipe Agent", () => {
-  const agent: scenario.AgentAdapter = {
-    role: scenario.AgentRole.AGENT,
+  const agent: AgentAdapter = {
+    role: AgentRole.AGENT,
     call: async (input) => {
       const response = await generateText({
-        model: openai("gpt-4.1-mini"),
+        model: openai("gpt-4.1"),
         messages: [
           {
             role: "system",
@@ -258,7 +259,7 @@ describe("Vegetarian Recipe Agent", () => {
         agent,
         scenario.userSimulatorAgent(),
         scenario.judgeAgent({
-          model: openai("gpt-4.1-mini"),
+          model: openai("gpt-4.1"),
           criteria: [
             "Agent should not ask more than two follow-up questions",
             "Agent should generate a recipe",
@@ -268,6 +269,7 @@ describe("Vegetarian Recipe Agent", () => {
           ],
         }),
       ],
+      setId: "javascript-examples",
     });
     expect(result.success).toBe(true);
   });
@@ -417,7 +419,7 @@ You can enable debug mode by setting the `debug` field to `True` in the `Scenari
 Debug mode allows you to see the messages in slow motion step by step, and intervene with your own inputs to debug your agent from the middle of the conversation.
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", debug=True)
+scenario.configure(default_model="openai/gpt-4.1", debug=True)
 ```
 
 or
@@ -431,7 +433,7 @@ pytest -s tests/test_vegetarian_recipe_agent.py --debug
 Each time the scenario runs, the testing agent might chose a different input to start, this is good to make sure it covers the variance of real users as well, however we understand that the non-deterministic nature of it might make it less repeatable, costly and harder to debug. To solve for it, you can use the `cache_key` field in the `Scenario.configure` method or in the specific scenario you are running, this will make the testing agent give the same input for given the same scenario:
 
 ```python
-scenario.configure(default_model="openai/gpt-4.1-mini", cache_key="42")
+scenario.configure(default_model="openai/gpt-4.1", cache_key="42")
 ```
 
 To bust the cache, you can simply pass a different `cache_key`, disable it, or delete the cache files located at `~/.scenario/cache`.

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/langwatch_scenario.egg-info/SOURCES.txt

@@ -11,7 +11,6 @@ scenario/__init__.py
 scenario/_error_messages.py
 scenario/agent_adapter.py
 scenario/cache.py
-scenario/config.py
 scenario/judge_agent.py
 scenario/pytest_plugin.py
 scenario/scenario_executor.py
@@ -20,6 +19,7 @@ scenario/script.py
 scenario/types.py
 scenario/user_simulator_agent.py
 scenario/_events/__init__.py
+scenario/_events/event_alert_message_logger.py
 scenario/_events/event_bus.py
 scenario/_events/event_reporter.py
 scenario/_events/events.py
@@ -239,7 +239,12 @@ scenario/_utils/__init__.py
 scenario/_utils/ids.py
 scenario/_utils/message_conversion.py
 scenario/_utils/utils.py
+scenario/config/__init__.py
+scenario/config/langwatch.py
+scenario/config/model.py
+scenario/config/scenario.py
 tests/test_event_reporter.py
+tests/test_model_config.py
 tests/test_scenario.py
 tests/test_scenario_agent.py
 tests/test_scenario_event_bus.py

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/langwatch_scenario.egg-info/requires.txt

@@ -12,13 +12,13 @@ pksuid>=1.1.2
 httpx>=0.27.0
 rx>=3.2.0
 python-dateutil>=2.9.0.post0
+pydantic-settings>=2.9.1
 
 [dev]
 black
 isort
 pytest-cov
 pre-commit
-commitizen
 pyright
 pydantic-ai
 function-schema

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "langwatch-scenario"
-version = "0.7.3"
+version = "0.7.8"
 description = "The end-to-end agent testing library"
 readme = "README.md"
 authors = [{ name = "LangWatch Team", email = "support@langwatch.ai" }]
@@ -35,6 +35,7 @@ dependencies = [
     "httpx>=0.27.0",
     "rx>=3.2.0",
     "python-dateutil>=2.9.0.post0",
+    "pydantic-settings>=2.9.1",
 ]
 
 [project.optional-dependencies]
@@ -43,7 +44,6 @@ dev = [
     "isort",
     "pytest-cov",
     "pre-commit",
-    "commitizen",
     "pyright",
     "pydantic-ai",
     "function-schema",
@@ -67,7 +67,7 @@ markers = ["agent_test: marks tests as agent scenario tests"]
 
 [dependency-groups]
 dev = [
-    "commitizen>=4.8.3",
+
     "function-schema>=0.4.5",
     "pre-commit>=4.2.0",
     "pydantic-ai>=0.0.52",
@@ -77,11 +77,4 @@ dev = [
     "respx>=0.22.0",
 ]
 
-[tool.commitizen]
-name = "cz_conventional_commits"
-version = "0.7.0"
-tag_format = "python/v$version"
-legacy_tag_formats = ["v$version"]
-version_files = ["pyproject.toml:version"]
-bump_message = "bump: version $current_version → $new_version"
-major_version_zero = true
+

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/scenario/__init__.py

@@ -25,7 +25,7 @@ Basic Usage:
     import scenario
 
     # Configure global settings
-    scenario.configure(default_model="openai/gpt-4.1-mini")
+    scenario.configure(default_model="openai/gpt-4.1")
 
     # Create your agent adapter
     class MyAgent(scenario.AgentAdapter):

{langwatch_scenario-0.7.3 → langwatch_scenario-0.7.8}/scenario/_error_messages.py

@@ -8,12 +8,12 @@ def agent_not_configured_error_message(class_name: str):
 
  {termcolor.colored("->", "cyan")} {class_name} was initialized without a model, please set the model when defining the testing agent, for example:
 
-    {class_name}(model="openai/gpt-4.1-mini")
+    {class_name}(model="openai/gpt-4.1")
     {termcolor.colored("^" * (29 + len(class_name)), "green")}
 
  {termcolor.colored("->", "cyan")} Alternatively, you can set the default model globally, for example:
 
-    scenario.configure(default_model="openai/gpt-4.1-mini")
+    scenario.configure(default_model="openai/gpt-4.1")
     {termcolor.colored("^" * 55, "green")}
 """
 

langwatch_scenario-0.7.8/scenario/_events/event_alert_message_logger.py

@@ -0,0 +1,95 @@
+import os
+from typing import Set
+from .._utils.ids import get_batch_run_id
+
+
+class EventAlertMessageLogger:
+    """
+    Handles console output of alert messages for scenario events.
+
+    Single responsibility: Display user-friendly messages about event reporting status
+    and simulation watching instructions.
+    """
+
+    _shown_batch_ids: Set[str] = set()
+
+    def handle_greeting(self) -> None:
+        """
+        Shows a fancy greeting message about simulation reporting status.
+        Only shows once per batch run to avoid spam.
+        """
+        if self._is_greeting_disabled():
+            return
+
+        batch_run_id = get_batch_run_id()
+
+        if batch_run_id in EventAlertMessageLogger._shown_batch_ids:
+            return
+
+        EventAlertMessageLogger._shown_batch_ids.add(batch_run_id)
+        self._display_greeting(batch_run_id)
+
+    def handle_watch_message(self, set_url: str) -> None:
+        """
+        Shows a fancy message about how to watch the simulation.
+        Called when a run started event is received with a session ID.
+        """
+        if self._is_greeting_disabled():
+            return
+
+        self._display_watch_message(set_url)
+
+    def _is_greeting_disabled(self) -> bool:
+        """Check if greeting messages are disabled via environment variable."""
+        return bool(os.getenv("SCENARIO_DISABLE_SIMULATION_REPORT_INFO"))
+
+    def _display_greeting(self, batch_run_id: str) -> None:
+        """Display the greeting message with simulation reporting status."""
+        separator = "─" * 60
+
+        if not os.getenv("LANGWATCH_API_KEY"):
+            print(f"\n{separator}")
+            print("🚀  LangWatch Simulation Reporting")
+            print(f"{separator}")
+            print("➡️  API key not configured")
+            print("   Simulations will only output final results")
+            print("")
+            print("💡 To visualize conversations in real time:")
+            print("   • Set LANGWATCH_API_KEY environment variable")
+            print("   • Or configure apiKey in scenario.config.js")
+            print("")
+            print(f"📦 Batch Run ID: {batch_run_id}")
+            print("")
+            print("🔇 To disable these messages:")
+            print("   • Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true")
+            print(f"{separator}\n")
+        else:
+            endpoint = os.getenv("LANGWATCH_ENDPOINT", "https://app.langwatch.ai")
+            api_key = os.getenv("LANGWATCH_API_KEY", "")
+
+            print(f"\n{separator}")
+            print("🚀  LangWatch Simulation Reporting")
+            print(f"{separator}")
+            print("✅ Simulation reporting enabled")
+            print(f"   Endpoint: {endpoint}")
+            print(f"   API Key: {'Configured' if api_key else 'Not configured'}")
+            print("")
+            print(f"📦 Batch Run ID: {batch_run_id}")
+            print("")
+            print("🔇 To disable these messages:")
+            print("   • Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true")
+            print(f"{separator}\n")
+
+    def _display_watch_message(self, set_url: str) -> None:
+        """Display the watch message with URLs for viewing the simulation."""
+        separator = "─" * 60
+        batch_url = f"{set_url}/{get_batch_run_id()}"
+
+        print(f"\n{separator}")
+        print("👀 Watch Your Simulation Live")
+        print(f"{separator}")
+        print("🌐 Open in your browser:")
+        print(f"   Scenario Set: {set_url}")
+        print(f"   Batch Run: {batch_url}")
+        print("")
+        print(f"{separator}\n")