langwatch-scenario 0.7.11__tar.gz → 0.7.13__tar.gz

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.11
+Version: 0.7.13
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/langwatch_scenario.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.11
+Version: 0.7.13
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/langwatch_scenario.egg-info/SOURCES.txt

@@ -245,9 +245,11 @@ scenario/config/langwatch.py
 scenario/config/model.py
 scenario/config/scenario.py
 tests/test_event_reporter.py
+tests/test_judge_agent.py
 tests/test_model_config.py
 tests/test_scenario.py
 tests/test_scenario_agent.py
 tests/test_scenario_event_bus.py
 tests/test_scenario_executor.py
-tests/test_scenario_executor_events.py
+tests/test_scenario_executor_events.py
+tests/test_user_simulator_agent.py

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "langwatch-scenario"
-version = "0.7.11"
+version = "0.7.13"
 description = "The end-to-end agent testing library"
 readme = "README.md"
 authors = [{ name = "LangWatch Team", email = "support@langwatch.ai" }]

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/scenario/config/model.py

@@ -6,7 +6,7 @@ user simulator and judge agents in the Scenario framework.
 """
 
 from typing import Optional
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class ModelConfig(BaseModel):
@@ -16,6 +16,9 @@ class ModelConfig(BaseModel):
     This class encapsulates all the parameters needed to configure an LLM model
     for use with user simulator and judge agents in the Scenario framework.
 
+    The ModelConfig accepts any additional parameters that litellm supports,
+    including headers, timeout, client, and other provider-specific options.
+
     Attributes:
         model: The model identifier (e.g., "openai/gpt-4.1", "anthropic/claude-3-sonnet")
         api_base: Optional base URL where the model is hosted
@@ -25,6 +28,7 @@ class ModelConfig(BaseModel):
 
     Example:
         ```
+        # Basic configuration
         model_config = ModelConfig(
             model="openai/gpt-4.1",
             api_base="https://api.openai.com/v1",
@@ -32,9 +36,29 @@ class ModelConfig(BaseModel):
             temperature=0.1,
             max_tokens=1000
         )
+
+        # With custom headers and timeout
+        model_config = ModelConfig(
+            model="openai/gpt-4",
+            headers={"X-Custom-Header": "value"},
+            timeout=60,
+            num_retries=3
+        )
+
+        # With custom OpenAI client
+        from openai import OpenAI
+        model_config = ModelConfig(
+            model="openai/gpt-4",
+            client=OpenAI(
+                base_url="https://custom.com",
+                default_headers={"X-Auth": "token"}
+            )
+        )
         ```
     """
 
+    model_config = ConfigDict(extra="allow")
+
     model: str
     api_base: Optional[str] = None
     api_key: Optional[str] = None

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/scenario/config/scenario.py

@@ -65,7 +65,7 @@ class ScenarioConfig(BaseModel):
     @classmethod
     def configure(
         cls,
-        default_model: Optional[str] = None,
+        default_model: Optional[Union[str, ModelConfig]] = None,
         max_turns: Optional[int] = None,
         verbose: Optional[Union[bool, int]] = None,
         cache_key: Optional[str] = None,

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/scenario/judge_agent.py

@@ -105,6 +105,7 @@ class JudgeAgent(AgentAdapter):
     max_tokens: Optional[int]
     criteria: List[str]
     system_prompt: Optional[str]
+    _extra_params: dict
 
     def __init__(
         self,
@@ -116,6 +117,7 @@ class JudgeAgent(AgentAdapter):
         temperature: float = 0.0,
         max_tokens: Optional[int] = None,
         system_prompt: Optional[str] = None,
+        **extra_params,
     ):
         """
         Initialize a judge agent with evaluation criteria.
@@ -159,8 +161,12 @@ class JudgeAgent(AgentAdapter):
                 system_prompt="You are a senior software engineer reviewing code for production use."
             )
             ```
+
+        Note:
+            Advanced usage: Additional parameters can be passed as keyword arguments
+            (e.g., headers, timeout, client) for specialized configurations. These are
+            experimental and may not be supported in future versions.
         """
-        # Override the default system prompt for the judge agent
         self.criteria = criteria or []
         self.api_base = api_base
         self.api_key = api_key
@@ -175,6 +181,7 @@ class JudgeAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, str
         ):
             self.model = model or ScenarioConfig.default_config.default_model
+            self._extra_params = extra_params
         elif ScenarioConfig.default_config is not None and isinstance(
             ScenarioConfig.default_config.default_model, ModelConfig
         ):
@@ -191,9 +198,22 @@ class JudgeAgent(AgentAdapter):
             self.max_tokens = (
                 max_tokens or ScenarioConfig.default_config.default_model.max_tokens
             )
+            # Extract extra params from ModelConfig
+            config_dict = ScenarioConfig.default_config.default_model.model_dump(
+                exclude_none=True
+            )
+            config_dict.pop("model", None)
+            config_dict.pop("api_base", None)
+            config_dict.pop("api_key", None)
+            config_dict.pop("temperature", None)
+            config_dict.pop("max_tokens", None)
+            # Merge: config extras < agent extra_params
+            self._extra_params = {**config_dict, **extra_params}
+        else:
+            self._extra_params = extra_params
 
         if not hasattr(self, "model"):
-            raise Exception(agent_not_configured_error_message("TestingAgent"))
+            raise Exception(agent_not_configured_error_message("JudgeAgent"))
 
     @scenario_cache()
     async def call(
@@ -370,6 +390,7 @@ if you don't have enough information to make a verdict, say inconclusive with ma
                     if (is_last_message or enforce_judgment) and has_criteria
                     else "required"
                 ),
+                **self._extra_params,
             ),
         )
 

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/scenario/user_simulator_agent.py

@@ -87,6 +87,7 @@ class UserSimulatorAgent(AgentAdapter):
     temperature: float
     max_tokens: Optional[int]
     system_prompt: Optional[str]
+    _extra_params: dict
 
     def __init__(
         self,
@@ -97,6 +98,7 @@ class UserSimulatorAgent(AgentAdapter):
         temperature: float = 0.0,
         max_tokens: Optional[int] = None,
         system_prompt: Optional[str] = None,
+        **extra_params,
     ):
         """
         Initialize a user simulator agent.
@@ -133,8 +135,12 @@ class UserSimulatorAgent(AgentAdapter):
                 '''
             )
             ```
+
+        Note:
+            Advanced usage: Additional parameters can be passed as keyword arguments
+            (e.g., headers, timeout, client) for specialized configurations. These are
+            experimental and may not be supported in future versions.
         """
-        # Override the default system prompt for the user simulator agent
         self.api_base = api_base
         self.api_key = api_key
         self.temperature = temperature
@@ -148,6 +154,7 @@ class UserSimulatorAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, str
         ):
             self.model = model or ScenarioConfig.default_config.default_model
+            self._extra_params = extra_params
         elif ScenarioConfig.default_config is not None and isinstance(
             ScenarioConfig.default_config.default_model, ModelConfig
         ):
@@ -164,9 +171,22 @@ class UserSimulatorAgent(AgentAdapter):
             self.max_tokens = (
                 max_tokens or ScenarioConfig.default_config.default_model.max_tokens
             )
+            # Extract extra params from ModelConfig
+            config_dict = ScenarioConfig.default_config.default_model.model_dump(
+                exclude_none=True
+            )
+            config_dict.pop("model", None)
+            config_dict.pop("api_base", None)
+            config_dict.pop("api_key", None)
+            config_dict.pop("temperature", None)
+            config_dict.pop("max_tokens", None)
+            # Merge: config extras < agent extra_params
+            self._extra_params = {**config_dict, **extra_params}
+        else:
+            self._extra_params = extra_params
 
         if not hasattr(self, "model"):
-            raise Exception(agent_not_configured_error_message("TestingAgent"))
+            raise Exception(agent_not_configured_error_message("UserSimulatorAgent"))
 
     @scenario_cache()
     async def call(
@@ -237,6 +257,7 @@ Your goal (assistant) is to interact with the Agent Under Test (user) as if you
                 api_base=self.api_base,
                 max_tokens=self.max_tokens,
                 tools=[],
+                **self._extra_params,
             ),
         )
 

langwatch_scenario-0.7.13/tests/test_judge_agent.py

@@ -0,0 +1,151 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from openai import OpenAI
+from scenario import JudgeAgent
+from scenario.config import ModelConfig, ScenarioConfig
+from scenario.types import AgentInput
+from scenario.cache import context_scenario
+from scenario.scenario_executor import ScenarioExecutor
+
+
+class FakeOpenAIClient:
+    """Fake client for testing without requiring API keys."""
+
+    def __init__(self, base_url=None, default_headers=None):
+        self.base_url = base_url
+        self.default_headers = default_headers
+
+
+@pytest.mark.asyncio
+async def test_judge_agent_merges_global_config_and_agent_params():
+    """JudgeAgent merges ModelConfig defaults with agent-specific overrides, including custom client."""
+    # Setup custom client
+    custom_client = FakeOpenAIClient(
+        base_url="https://custom.com", default_headers={"X-Global": "global-value"}
+    )
+
+    # Setup global config with extra params
+    ScenarioConfig.default_config = ScenarioConfig(
+        default_model=ModelConfig(
+            model="openai/gpt-4",
+            api_base="https://custom.com",
+            headers={"X-Global": "global-value"},  # type: ignore  # extra param via ConfigDict(extra="allow")
+            timeout=30,  # type: ignore  # extra param via ConfigDict(extra="allow")
+            client=custom_client,  # type: ignore  # extra param via ConfigDict(extra="allow")
+        )
+    )
+
+    judge = JudgeAgent(
+        criteria=["Test criterion"],
+        temperature=0.5,
+        timeout=60,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+    mock_scenario_state.current_turn = 1
+    mock_scenario_state.config.max_turns = 10
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[{"role": "user", "content": "Hello"}],
+        new_messages=[],
+        judgment_request=True,
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.tool_calls = [MagicMock()]
+    mock_response.choices[0].message.tool_calls[0].function.name = "finish_test"
+    mock_response.choices[0].message.tool_calls[
+        0
+    ].function.arguments = '{"verdict": "success", "reasoning": "Test passed", "criteria": {"test_criterion": true}}'
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.judge_agent.litellm.completion", return_value=mock_response
+        ) as mock_completion:
+            await judge.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+
+            # Verify merged params: config defaults + agent overrides
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["api_base"] == "https://custom.com"
+            assert call_kwargs["temperature"] == 0.5  # Agent override
+            assert call_kwargs["timeout"] == 60  # Agent override
+            assert call_kwargs["headers"] == {"X-Global": "global-value"}  # From config
+            assert (
+                call_kwargs["client"] == custom_client
+            )  # Custom client passed through
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None
+
+
+@pytest.mark.asyncio
+async def test_judge_agent_with_string_default_model_config():
+    """JudgeAgent should initialize _extra_params when default_model is a string."""
+    # Setup global config with string default_model
+    ScenarioConfig.default_config = ScenarioConfig(default_model="openai/gpt-4")
+
+    judge = JudgeAgent(
+        criteria=["Test criterion"],
+        temperature=0.5,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+    mock_scenario_state.current_turn = 1
+    mock_scenario_state.config.max_turns = 10
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[{"role": "user", "content": "Hello"}],
+        new_messages=[],
+        judgment_request=True,
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.tool_calls = [MagicMock()]
+    mock_response.choices[0].message.tool_calls[0].function.name = "finish_test"
+    mock_response.choices[0].message.tool_calls[
+        0
+    ].function.arguments = '{"verdict": "success", "reasoning": "Test passed", "criteria": {"test_criterion": true}}'
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.judge_agent.litellm.completion", return_value=mock_response
+        ) as mock_completion:
+            # This should not raise AttributeError: 'JudgeAgent' object has no attribute '_extra_params'
+            await judge.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["temperature"] == 0.5
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None

{langwatch_scenario-0.7.11 → langwatch_scenario-0.7.13}/tests/test_model_config.py

@@ -35,3 +35,29 @@ async def test_user_simulator_agent_uses_modelconfig_api_base():
             mock_completion.call_args.kwargs["api_base"]
             == "https://custom-api-base.example.com"
         )
+
+
+def test_modelconfig_accepts_extra_litellm_params():
+    """ModelConfig accepts arbitrary litellm parameters via extra='allow'."""
+    from openai import OpenAI
+
+    custom_client = MagicMock(spec=OpenAI)
+
+    config = ModelConfig(
+        model="openai/gpt-4",
+        api_base="https://custom.com",
+        headers={"X-Custom-Header": "test-value"},  # type: ignore  # extra param via ConfigDict(extra="allow")
+        timeout=60,  # type: ignore  # extra param via ConfigDict(extra="allow")
+        num_retries=3,  # type: ignore  # extra param via ConfigDict(extra="allow")
+        client=custom_client,  # type: ignore  # extra param via ConfigDict(extra="allow")
+    )
+
+    assert config.model == "openai/gpt-4"
+    assert config.api_base == "https://custom.com"
+
+    # Verify extra params are stored
+    dump = config.model_dump()
+    assert dump["headers"] == {"X-Custom-Header": "test-value"}
+    assert dump["timeout"] == 60
+    assert dump["num_retries"] == 3
+    assert dump["client"] == custom_client

langwatch_scenario-0.7.13/tests/test_user_simulator_agent.py

@@ -0,0 +1,118 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from scenario import UserSimulatorAgent
+from scenario.config import ModelConfig, ScenarioConfig
+from scenario.types import AgentInput
+from scenario.cache import context_scenario
+from scenario.scenario_executor import ScenarioExecutor
+
+
+@pytest.mark.asyncio
+async def test_user_simulator_agent_merges_global_config_and_agent_params():
+    """UserSimulatorAgent merges ModelConfig defaults with agent-specific overrides."""
+    # Setup global config with extra params
+    ScenarioConfig.default_config = ScenarioConfig(
+        default_model=ModelConfig(
+            model="openai/gpt-4",
+            headers={"X-Auth": "token-123"},  # type: ignore  # extra param via ConfigDict(extra="allow")
+            max_retries=5,  # type: ignore  # extra param via ConfigDict(extra="allow")
+        )
+    )
+
+    user_sim = UserSimulatorAgent(
+        temperature=0.7,
+        num_retries=2,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[],
+        new_messages=[],
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.content = "test user message"
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.user_simulator_agent.litellm.completion",
+            return_value=mock_response,
+        ) as mock_completion:
+            await user_sim.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+
+            # Verify merged params
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["temperature"] == 0.7  # Agent override
+            assert call_kwargs["headers"] == {"X-Auth": "token-123"}  # From config
+            assert call_kwargs["max_retries"] == 5  # From config
+            assert call_kwargs["num_retries"] == 2  # Agent-specific
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None
+
+
+@pytest.mark.asyncio
+async def test_user_simulator_agent_with_string_default_model_config():
+    """UserSimulatorAgent should initialize _extra_params when default_model is a string."""
+    # Setup global config with string default_model
+    ScenarioConfig.default_config = ScenarioConfig(default_model="openai/gpt-4")
+
+    user_sim = UserSimulatorAgent(
+        temperature=0.7,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[],
+        new_messages=[],
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.content = "test user message"
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.user_simulator_agent.litellm.completion",
+            return_value=mock_response,
+        ) as mock_completion:
+            # This should not raise AttributeError: 'UserSimulatorAgent' object has no attribute '_extra_params'
+            await user_sim.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["temperature"] == 0.7
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None