langwatch-scenario 0.7.7__tar.gz → 0.7.8__tar.gz

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.7
+Version: 0.7.8
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/langwatch_scenario.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.7
+Version: 0.7.8
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/langwatch_scenario.egg-info/SOURCES.txt

@@ -244,6 +244,7 @@ scenario/config/langwatch.py
 scenario/config/model.py
 scenario/config/scenario.py
 tests/test_event_reporter.py
+tests/test_model_config.py
 tests/test_scenario.py
 tests/test_scenario_agent.py
 tests/test_scenario_event_bus.py

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "langwatch-scenario"
-version = "0.7.7"
+version = "0.7.8"
 description = "The end-to-end agent testing library"
 readme = "README.md"
 authors = [{ name = "LangWatch Team", email = "support@langwatch.ai" }]

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/scenario/config/model.py

@@ -17,7 +17,8 @@ class ModelConfig(BaseModel):
     for use with user simulator and judge agents in the Scenario framework.
 
     Attributes:
-        model: The model identifier (e.g., "openai/gpt-4.1-mini", "anthropic/claude-3-sonnet")
+        model: The model identifier (e.g., "openai/gpt-4.1", "anthropic/claude-3-sonnet")
+        api_base: Optional base URL where the model is hosted
         api_key: Optional API key for the model provider
         temperature: Sampling temperature for response generation (0.0 = deterministic, 1.0 = creative)
         max_tokens: Maximum number of tokens to generate in responses
@@ -26,6 +27,7 @@ class ModelConfig(BaseModel):
         ```
         model_config = ModelConfig(
             model="openai/gpt-4.1",
+            api_base="https://api.openai.com/v1",
             api_key="your-api-key",
             temperature=0.1,
             max_tokens=1000
@@ -34,6 +36,7 @@ class ModelConfig(BaseModel):
     """
 
     model: str
+    api_base: Optional[str] = None
     api_key: Optional[str] = None
     temperature: float = 0.0
     max_tokens: Optional[int] = None

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/scenario/judge_agent.py

@@ -41,6 +41,7 @@ class JudgeAgent(AgentAdapter):
     Attributes:
         role: Always AgentRole.JUDGE for judge agents
         model: LLM model identifier to use for evaluation
+        api_base: Optional base URL where the model is hosted
         api_key: Optional API key for the model provider
         temperature: Sampling temperature for evaluation consistency
         max_tokens: Maximum tokens for judge reasoning
@@ -97,6 +98,7 @@ class JudgeAgent(AgentAdapter):
     role = AgentRole.JUDGE
 
     model: str
+    api_base: Optional[str]
     api_key: Optional[str]
     temperature: float
     max_tokens: Optional[int]
@@ -108,6 +110,7 @@ class JudgeAgent(AgentAdapter):
         *,
         criteria: Optional[List[str]] = None,
         model: Optional[str] = None,
+        api_base: Optional[str] = None,
         api_key: Optional[str] = None,
         temperature: float = 0.0,
         max_tokens: Optional[int] = None,
@@ -122,6 +125,8 @@ class JudgeAgent(AgentAdapter):
                      and negative constraints ("Agent should not provide personal information").
             model: LLM model identifier (e.g., "openai/gpt-4.1").
                    If not provided, uses the default model from global configuration.
+            api_base: Optional base URL where the model is hosted. If not provided,
+                      uses the base URL from global configuration.
             api_key: API key for the model provider. If not provided,
                      uses the key from global configuration or environment.
             temperature: Sampling temperature for evaluation (0.0-1.0).
@@ -156,6 +161,7 @@ class JudgeAgent(AgentAdapter):
         """
         # Override the default system prompt for the judge agent
         self.criteria = criteria or []
+        self.api_base = api_base
         self.api_key = api_key
         self.temperature = temperature
         self.max_tokens = max_tokens
@@ -172,6 +178,9 @@ class JudgeAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, ModelConfig
         ):
             self.model = model or ScenarioConfig.default_config.default_model.model
+            self.api_base = (
+                api_base or ScenarioConfig.default_config.default_model.api_base
+            )
             self.api_key = (
                 api_key or ScenarioConfig.default_config.default_model.api_key
             )
@@ -351,6 +360,8 @@ if you don't have enough information to make a verdict, say inconclusive with ma
                 model=self.model,
                 messages=messages,
                 temperature=self.temperature,
+                api_key=self.api_key,
+                api_base=self.api_base,
                 max_tokens=self.max_tokens,
                 tools=tools,
                 tool_choice=(

{langwatch_scenario-0.7.7 → langwatch_scenario-0.7.8}/scenario/user_simulator_agent.py

@@ -37,6 +37,7 @@ class UserSimulatorAgent(AgentAdapter):
     Attributes:
         role: Always AgentRole.USER for user simulator agents
         model: LLM model identifier to use for generating user messages
+        api_base: Optional base URL where the model is hosted
         api_key: Optional API key for the model provider
         temperature: Sampling temperature for response generation
         max_tokens: Maximum tokens to generate in user messages
@@ -76,9 +77,11 @@ class UserSimulatorAgent(AgentAdapter):
         - Messages are generated in a casual, human-like style (lowercase, brief, etc.)
         - The simulator will not act as an assistant - it only generates user inputs
     """
+
     role = AgentRole.USER
 
     model: str
+    api_base: Optional[str]
     api_key: Optional[str]
     temperature: float
     max_tokens: Optional[int]
@@ -88,6 +91,7 @@ class UserSimulatorAgent(AgentAdapter):
         self,
         *,
         model: Optional[str] = None,
+        api_base: Optional[str] = None,
         api_key: Optional[str] = None,
         temperature: float = 0.0,
         max_tokens: Optional[int] = None,
@@ -99,6 +103,8 @@ class UserSimulatorAgent(AgentAdapter):
         Args:
             model: LLM model identifier (e.g., "openai/gpt-4.1").
                    If not provided, uses the default model from global configuration.
+            api_base: Optional base URL where the model is hosted. If not provided,
+                      uses the base URL from global configuration.
             api_key: API key for the model provider. If not provided,
                      uses the key from global configuration or environment.
             temperature: Sampling temperature for message generation (0.0-1.0).
@@ -128,6 +134,7 @@ class UserSimulatorAgent(AgentAdapter):
             ```
         """
         # Override the default system prompt for the user simulator agent
+        self.api_base = api_base
         self.api_key = api_key
         self.temperature = temperature
         self.max_tokens = max_tokens
@@ -144,6 +151,9 @@ class UserSimulatorAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, ModelConfig
         ):
             self.model = model or ScenarioConfig.default_config.default_model.model
+            self.api_base = (
+                api_base or ScenarioConfig.default_config.default_model.api_base
+            )
             self.api_key = (
                 api_key or ScenarioConfig.default_config.default_model.api_key
             )
@@ -222,6 +232,8 @@ Your goal (assistant) is to interact with the Agent Under Test (user) as if you
                 model=self.model,
                 messages=messages,
                 temperature=self.temperature,
+                api_key=self.api_key,
+                api_base=self.api_base,
                 max_tokens=self.max_tokens,
                 tools=[],
             ),

langwatch_scenario-0.7.8/tests/test_model_config.py

@@ -0,0 +1,37 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from scenario.types import AgentInput
+from scenario.config import ModelConfig
+from typing import Optional
+
+
+class UserSimulatorAgent:
+    def __init__(self, *, model: str, api_base: Optional[str]):
+        self.model = model
+        self.api_base = api_base or ""
+
+    async def call(self, input: AgentInput):
+        from litellm import completion
+
+        completion(model=self.model, messages=[], api_base=self.api_base)
+
+
+def test_modelconfig_api_base_field():
+    config = ModelConfig(model="foo", api_base="https://bar.com")
+    assert config.api_base == "https://bar.com"
+
+
+@pytest.mark.asyncio
+async def test_user_simulator_agent_uses_modelconfig_api_base():
+    model_config = ModelConfig(
+        model="openai/gpt-4.1", api_base="https://custom-api-base.example.com"
+    )
+    agent = UserSimulatorAgent(model=model_config.model, api_base=model_config.api_base)
+    agent_input = MagicMock(spec=AgentInput)
+    with patch("litellm.completion") as mock_completion:
+        await agent.call(agent_input)
+        assert mock_completion.called
+        assert (
+            mock_completion.call_args.kwargs["api_base"]
+            == "https://custom-api-base.example.com"
+        )