langwatch-scenario 0.7.8__tar.gz → 0.7.10__tar.gz

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.8
+Version: 0.7.10
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: pytest>=8.1.1
 Requires-Dist: litellm>=1.49.0
@@ -31,6 +31,7 @@ Requires-Dist: httpx>=0.27.0
 Requires-Dist: rx>=3.2.0
 Requires-Dist: python-dateutil>=2.9.0.post0
 Requires-Dist: pydantic-settings>=2.9.1
+Requires-Dist: langwatch>=0.2.19
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -457,7 +458,7 @@ This will cache any function call you decorate when running the tests and make t
 While optional, we strongly recommend setting stable identifiers for your scenarios, sets, and batches for better organization and tracking in LangWatch.
 
 - **set_id**: Groups related scenarios into a test suite. This corresponds to the "Simulation Set" in the UI.
-- **batch_run_id**: Groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
+- **SCENARIO_BATCH_RUN_ID**: Env variable that groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
 
 ```python
 import os

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/README.md

@@ -413,7 +413,7 @@ This will cache any function call you decorate when running the tests and make t
 While optional, we strongly recommend setting stable identifiers for your scenarios, sets, and batches for better organization and tracking in LangWatch.
 
 - **set_id**: Groups related scenarios into a test suite. This corresponds to the "Simulation Set" in the UI.
-- **batch_run_id**: Groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
+- **SCENARIO_BATCH_RUN_ID**: Env variable that groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
 
 ```python
 import os

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/langwatch_scenario.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.8
+Version: 0.7.10
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: pytest>=8.1.1
 Requires-Dist: litellm>=1.49.0
@@ -31,6 +31,7 @@ Requires-Dist: httpx>=0.27.0
 Requires-Dist: rx>=3.2.0
 Requires-Dist: python-dateutil>=2.9.0.post0
 Requires-Dist: pydantic-settings>=2.9.1
+Requires-Dist: langwatch>=0.2.19
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -457,7 +458,7 @@ This will cache any function call you decorate when running the tests and make t
 While optional, we strongly recommend setting stable identifiers for your scenarios, sets, and batches for better organization and tracking in LangWatch.
 
 - **set_id**: Groups related scenarios into a test suite. This corresponds to the "Simulation Set" in the UI.
-- **batch_run_id**: Groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
+- **SCENARIO_BATCH_RUN_ID**: Env variable that groups all scenarios that were run together in a single execution (e.g., a single CI job). This is automatically generated but can be overridden.
 
 ```python
 import os

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/langwatch_scenario.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ scenario/_error_messages.py
 scenario/agent_adapter.py
 scenario/cache.py
 scenario/judge_agent.py
+scenario/py.typed
 scenario/pytest_plugin.py
 scenario/scenario_executor.py
 scenario/scenario_state.py

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/langwatch_scenario.egg-info/requires.txt

@@ -13,6 +13,7 @@ httpx>=0.27.0
 rx>=3.2.0
 python-dateutil>=2.9.0.post0
 pydantic-settings>=2.9.1
+langwatch>=0.2.19
 
 [dev]
 black

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/pyproject.toml

@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "langwatch-scenario"
-version = "0.7.8"
+version = "0.7.10"
 description = "The end-to-end agent testing library"
 readme = "README.md"
 authors = [{ name = "LangWatch Team", email = "support@langwatch.ai" }]
 license = { text = "MIT" }
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
@@ -36,6 +36,7 @@ dependencies = [
     "rx>=3.2.0",
     "python-dateutil>=2.9.0.post0",
     "pydantic-settings>=2.9.1",
+    "langwatch>=0.2.19",
 ]
 
 [project.optional-dependencies]
@@ -67,14 +68,11 @@ markers = ["agent_test: marks tests as agent scenario tests"]
 
 [dependency-groups]
 dev = [
-
     "function-schema>=0.4.5",
     "pre-commit>=4.2.0",
     "pydantic-ai>=0.0.52",
-    "pyright>=1.1.401",
+    "pyright>=1.1.405",
     "pytest-asyncio-concurrent>=0.4.1",
     "pdoc3>=0.11.6",
     "respx>=0.22.0",
 ]
-
-

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/_events/event_alert_message_logger.py

@@ -1,5 +1,8 @@
 import os
+import webbrowser
 from typing import Set
+
+from ..config.scenario import ScenarioConfig
 from .._utils.ids import get_batch_run_id
 
 
@@ -12,6 +15,7 @@ class EventAlertMessageLogger:
     """
 
     _shown_batch_ids: Set[str] = set()
+    _shown_watch_urls: Set[str] = set()
 
     def handle_greeting(self) -> None:
         """
@@ -37,6 +41,10 @@ class EventAlertMessageLogger:
         if self._is_greeting_disabled():
             return
 
+        if set_url in EventAlertMessageLogger._shown_watch_urls:
+            return
+
+        EventAlertMessageLogger._shown_watch_urls.add(set_url)
         self._display_watch_message(set_url)
 
     def _is_greeting_disabled(self) -> bool:
@@ -49,35 +57,13 @@ class EventAlertMessageLogger:
 
         if not os.getenv("LANGWATCH_API_KEY"):
             print(f"\n{separator}")
-            print("🚀  LangWatch Simulation Reporting")
+            print("🎭  Running Scenario Tests")
             print(f"{separator}")
-            print("➡️  API key not configured")
+            print("➡️  LangWatch API key not configured")
             print("   Simulations will only output final results")
             print("")
             print("💡 To visualize conversations in real time:")
             print("   • Set LANGWATCH_API_KEY environment variable")
-            print("   • Or configure apiKey in scenario.config.js")
-            print("")
-            print(f"📦 Batch Run ID: {batch_run_id}")
-            print("")
-            print("🔇 To disable these messages:")
-            print("   • Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true")
-            print(f"{separator}\n")
-        else:
-            endpoint = os.getenv("LANGWATCH_ENDPOINT", "https://app.langwatch.ai")
-            api_key = os.getenv("LANGWATCH_API_KEY", "")
-
-            print(f"\n{separator}")
-            print("🚀  LangWatch Simulation Reporting")
-            print(f"{separator}")
-            print("✅ Simulation reporting enabled")
-            print(f"   Endpoint: {endpoint}")
-            print(f"   API Key: {'Configured' if api_key else 'Not configured'}")
-            print("")
-            print(f"📦 Batch Run ID: {batch_run_id}")
-            print("")
-            print("🔇 To disable these messages:")
-            print("   • Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true")
             print(f"{separator}\n")
 
     def _display_watch_message(self, set_url: str) -> None:
@@ -86,10 +72,15 @@ class EventAlertMessageLogger:
         batch_url = f"{set_url}/{get_batch_run_id()}"
 
         print(f"\n{separator}")
-        print("👀 Watch Your Simulation Live")
+        print("🎭  Running Scenario Tests")
         print(f"{separator}")
-        print("🌐 Open in your browser:")
-        print(f"   Scenario Set: {set_url}")
-        print(f"   Batch Run: {batch_url}")
-        print("")
+        print(f"Follow it live: {batch_url}")
         print(f"{separator}\n")
+
+        config = ScenarioConfig.default_config
+        if config and not config.headless:
+            # Open the URL in the default browser (cross-platform)
+            try:
+                webbrowser.open(batch_url)
+            except Exception:
+                pass

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/_events/event_bus.py

@@ -3,6 +3,7 @@ from typing import Optional, Any, Dict
 from .events import ScenarioEvent
 from .event_reporter import EventReporter
 from .event_alert_message_logger import EventAlertMessageLogger
+from ..config.scenario import ScenarioConfig
 
 import asyncio
 import queue
@@ -35,7 +36,9 @@ class ScenarioEventBus:
     """
 
     def __init__(
-        self, event_reporter: Optional[EventReporter] = None, max_retries: int = 3
+        self,
+        event_reporter: Optional[EventReporter] = None,
+        max_retries: int = 3,
     ):
         """
         Initialize the event bus with optional event reporter and retry configuration.

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/_events/event_reporter.py

@@ -3,7 +3,7 @@ import httpx
 from typing import Optional, Dict, Any
 from .events import ScenarioEvent
 from .event_alert_message_logger import EventAlertMessageLogger
-from scenario.config import LangWatchSettings
+from scenario.config import LangWatchSettings, ScenarioConfig
 
 
 class EventReporter:
@@ -26,7 +26,11 @@ class EventReporter:
         reporter = EventReporter(api_key="your-api-key")
     """
 
-    def __init__(self, endpoint: Optional[str] = None, api_key: Optional[str] = None):
+    def __init__(
+        self,
+        endpoint: Optional[str] = None,
+        api_key: Optional[str] = None,
+    ):
         # Load settings from environment variables
         langwatch_settings = LangWatchSettings()
 
@@ -69,6 +73,7 @@ class EventReporter:
                         "Content-Type": "application/json",
                         "X-Auth-Token": self.api_key,
                     },
+                    timeout=httpx.Timeout(30.0),
                 )
                 self.logger.info(
                     f"[{event_type}] POST response status: {response.status_code} ({event.scenario_run_id})"
@@ -92,7 +97,7 @@ class EventReporter:
                     )
         except Exception as error:
             self.logger.error(
-                f"[{event_type}] Event POST error: {error}, event={event}, endpoint={self.endpoint}"
+                f"[{event_type}] Event POST error: {repr(error)}, event={event}, endpoint={self.endpoint}"
             )
 
         return result

langwatch_scenario-0.7.10/scenario/_events/utils.py

@@ -0,0 +1,113 @@
+import warnings
+
+from ..types import ChatCompletionMessageParamWithTrace
+from .events import MessageType
+from .messages import (
+    SystemMessage,
+    AssistantMessage,
+    UserMessage,
+    ToolMessage,
+    ToolCall,
+    FunctionCall,
+)
+from typing import List
+from pksuid import PKSUID
+
+
+def convert_messages_to_api_client_messages(
+    messages: list[ChatCompletionMessageParamWithTrace],
+) -> list[MessageType]:
+    """
+    Converts OpenAI ChatCompletionMessageParam messages to API client Message format.
+
+    This function transforms messages from OpenAI's format to the API client format
+    that matches the expected schema for ScenarioMessageSnapshotEvent.
+
+    Args:
+        messages: List of OpenAI ChatCompletionMessageParam messages
+
+    Returns:
+        List of API client Message objects
+
+    Raises:
+        ValueError: If message role is not supported or message format is invalid
+    """
+
+    converted_messages: list[MessageType] = []
+
+    for i, message in enumerate(messages):
+        # Generate unique ID for each message
+        message_id = message.get("id") or str(PKSUID("scenariomsg"))
+
+        role = message.get("role")
+        content = message.get("content")
+
+        if role == "user":
+            if not content:
+                raise ValueError(f"User message at index {i} missing required content")
+            message_ = UserMessage(
+                id=message_id,
+                role="user",
+                content=str(content),
+            )
+            message_.additional_properties = {"trace_id": message.get("trace_id")}
+            converted_messages.append(message_)
+        elif role == "assistant":
+            # Handle tool calls if present
+            tool_calls = message.get("tool_calls")
+            api_tool_calls: List[ToolCall] = []
+
+            if tool_calls:
+                for tool_call in tool_calls:
+                    api_tool_calls.append(
+                        ToolCall(
+                            id=tool_call.get("id", str(PKSUID("scenariotoolcall"))),
+                            type_="function",
+                            function=FunctionCall(
+                                name=tool_call["function"].get("name", "unknown"),
+                                arguments=tool_call["function"].get("arguments", "{}"),
+                            ),
+                        )
+                    )
+
+            message_ = AssistantMessage(
+                id=message_id,
+                role="assistant",
+                content=str(content),
+                tool_calls=api_tool_calls,
+            )
+            message_.additional_properties = {"trace_id": message.get("trace_id")}
+            converted_messages.append(message_)
+        elif role == "system":
+            if not content:
+                raise ValueError(
+                    f"System message at index {i} missing required content"
+                )
+            message_ = SystemMessage(id=message_id, role="system", content=str(content))
+            message_.additional_properties = {"trace_id": message.get("trace_id")}
+            converted_messages.append(message_)
+        elif role == "tool":
+            tool_call_id = message.get("tool_call_id")
+            if not tool_call_id:
+                warnings.warn(
+                    f"Tool message at index {i} missing required tool_call_id, skipping tool message"
+                )
+                continue
+            if not content:
+                warnings.warn(
+                    f"Tool message at index {i} missing required content, skipping tool message"
+                )
+                continue
+
+            message_ = ToolMessage(
+                id=message_id,
+                role="tool",
+                content=str(content),
+                tool_call_id=tool_call_id,
+            )
+            message_.additional_properties = {"trace_id": message.get("trace_id")}
+            converted_messages.append(message_)
+        else:
+            raise ValueError(f"Unsupported message role '{role}' at index {i}")
+
+    return converted_messages

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/_utils/__init__.py

@@ -14,7 +14,7 @@ from .ids import (
     generate_scenario_id,
     generate_thread_id,
     generate_message_id,
-    safe_parse_uuid,
+    safe_parse_ksuid,
 )
 from .utils import (
     SerializableAndPydanticEncoder,
@@ -34,7 +34,7 @@ __all__ = [
     "generate_scenario_id",
     "generate_thread_id",
     "generate_message_id",
-    "safe_parse_uuid",
+    "safe_parse_ksuid",
     "SerializableAndPydanticEncoder",
     "SerializableWithStringFallback",
     "print_openai_messages",

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/_utils/ids.py

@@ -7,7 +7,7 @@ and scenario tracking.
 """
 
 import os
-import uuid
+from pksuid import PKSUID
 
 
 def generate_thread_id() -> str:
@@ -17,7 +17,7 @@ def generate_thread_id() -> str:
     Returns:
         str: A new thread ID.
     """
-    return f"thread_{uuid.uuid4()}"
+    return f"{PKSUID('scenariothread')}"
 
 
 def generate_scenario_run_id() -> str:
@@ -27,7 +27,7 @@ def generate_scenario_run_id() -> str:
     Returns:
         str: A new scenario run ID.
     """
-    return f"scenariorun_{uuid.uuid4()}"
+    return f"{PKSUID('scenariorun')}"
 
 
 def generate_scenario_id() -> str:
@@ -37,7 +37,7 @@ def generate_scenario_id() -> str:
     Returns:
         str: A new scenario ID.
     """
-    return f"scenario_{uuid.uuid4()}"
+    return f"{PKSUID('scenario')}"
 
 
 def get_batch_run_id() -> str:
@@ -52,7 +52,7 @@ def get_batch_run_id() -> str:
     batch_run_id = os.environ.get("SCENARIO_BATCH_RUN_ID")
     if not batch_run_id:
         # Generate new batch ID if not set
-        batch_run_id = f"scenariobatchrun_{uuid.uuid4()}"
+        batch_run_id = f"{PKSUID('scenariobatch')}"
         os.environ["SCENARIO_BATCH_RUN_ID"] = batch_run_id
 
     return batch_run_id
@@ -65,23 +65,23 @@ def generate_message_id() -> str:
     Returns:
         str: A new message ID.
     """
-    return f"scenariomsg_{uuid.uuid4()}"
+    return f"{PKSUID('scenariomsg')}"
 
 
-def safe_parse_uuid(id_str: str) -> bool:
+def safe_parse_ksuid(id_str: str) -> bool:
     """
-    Safely parses a UUID string.
+    Safely parses a Ksuid string.
 
     Args:
-        id_str: The UUID string to parse.
+        id_str: The Ksuid string to parse.
 
     Returns:
-        bool: True if the UUID string is valid, false otherwise.
+        bool: True if the Ksuid string is valid, false otherwise.
     """
     try:
-        uuid.UUID(id_str)
+        PKSUID.parse(id_str)
         return True
-    except (ValueError, TypeError):
+    except Exception:
         return False
 
 

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/config/scenario.py

@@ -5,6 +5,7 @@ This module provides the main configuration class for customizing the behavior
 of the Scenario testing framework, including execution parameters and debugging options.
 """
 
+import os
 from typing import Optional, Union, ClassVar
 from pydantic import BaseModel
 
@@ -53,6 +54,11 @@ class ScenarioConfig(BaseModel):
     verbose: Optional[Union[bool, int]] = True
     cache_key: Optional[str] = None
     debug: Optional[bool] = False
+    headless: Optional[bool] = os.getenv("SCENARIO_HEADLESS", "false").lower() not in [
+        "false",
+        "0",
+        "",
+    ]
 
     default_config: ClassVar[Optional["ScenarioConfig"]] = None
 
@@ -64,6 +70,7 @@ class ScenarioConfig(BaseModel):
         verbose: Optional[Union[bool, int]] = None,
         cache_key: Optional[str] = None,
         debug: Optional[bool] = None,
+        headless: Optional[bool] = None,
     ) -> None:
         """
         Set global configuration settings for all scenario executions.
@@ -107,6 +114,7 @@ class ScenarioConfig(BaseModel):
                 verbose=verbose,
                 cache_key=cache_key,
                 debug=debug,
+                headless=headless,
             )
         )
 

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/judge_agent.py

@@ -12,7 +12,8 @@ import logging
 import re
 from typing import List, Optional, cast
 
-from litellm import Choices, completion
+import litellm
+from litellm import Choices
 from litellm.files.main import ModelResponse
 
 from scenario.cache import scenario_cache
@@ -356,7 +357,7 @@ if you don't have enough information to make a verdict, say inconclusive with ma
 
         response = cast(
             ModelResponse,
-            completion(
+            litellm.completion(
                 model=self.model,
                 messages=messages,
                 temperature=self.temperature,
@@ -398,7 +399,7 @@ if you don't have enough information to make a verdict, say inconclusive with ma
                         failed_criteria = [
                             self.criteria[idx]
                             for idx, criterion in enumerate(criteria.values())
-                            if criterion == False
+                            if criterion == False or criterion == "inconclusive"
                         ]
 
                         # Return the appropriate ScenarioResult based on the verdict

{langwatch_scenario-0.7.8 → langwatch_scenario-0.7.10}/scenario/pytest_plugin.py

@@ -199,6 +199,8 @@ class ScenarioReporter:
 # Store the original run method
 original_run = ScenarioExecutor.run
 
+def pytest_addoption(parser):
+    parser.addoption("--headless", action="store_true")
 
 @pytest.hookimpl(trylast=True)
 def pytest_configure(config):
@@ -240,6 +242,9 @@ def pytest_configure(config):
         print(colored("\nScenario debug mode enabled (--debug).", "yellow"))
         ScenarioConfig.configure(verbose=True, debug=True)
 
+    if config.getoption("--headless"):
+        ScenarioConfig.configure(headless=True)
+
     # Create a global reporter instance
     config._scenario_reporter = ScenarioReporter()