langwatch-scenario 0.7.12__tar.gz → 0.7.14__tar.gz

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.12
+Version: 0.7.14
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/langwatch_scenario.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langwatch-scenario
-Version: 0.7.12
+Version: 0.7.14
 Summary: The end-to-end agent testing library
 Author-email: LangWatch Team <support@langwatch.ai>
 License: MIT

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "langwatch-scenario"
-version = "0.7.12"
+version = "0.7.14"
 description = "The end-to-end agent testing library"
 readme = "README.md"
 authors = [{ name = "LangWatch Team", email = "support@langwatch.ai" }]

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/scenario/judge_agent.py

@@ -181,6 +181,7 @@ class JudgeAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, str
         ):
             self.model = model or ScenarioConfig.default_config.default_model
+            self._extra_params = extra_params
         elif ScenarioConfig.default_config is not None and isinstance(
             ScenarioConfig.default_config.default_model, ModelConfig
         ):
@@ -338,7 +339,8 @@ if you don't have enough information to make a verdict, say inconclusive with ma
                                 "type": "object",
                                 "properties": {
                                     criteria_names[idx]: {
-                                        "enum": [True, False, "inconclusive"],
+                                        "type": "string",
+                                        "enum": ["true", "false", "inconclusive"],
                                         "description": criterion,
                                     }
                                     for idx, criterion in enumerate(self.criteria)
@@ -414,12 +416,12 @@ if you don't have enough information to make a verdict, say inconclusive with ma
                         passed_criteria = [
                             self.criteria[idx]
                             for idx, criterion in enumerate(criteria.values())
-                            if criterion == True
+                            if criterion == "true"
                         ]
                         failed_criteria = [
                             self.criteria[idx]
                             for idx, criterion in enumerate(criteria.values())
-                            if criterion == False or criterion == "inconclusive"
+                            if criterion == "false" or criterion == "inconclusive"
                         ]
 
                         # Return the appropriate ScenarioResult based on the verdict

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/scenario/user_simulator_agent.py

@@ -154,6 +154,7 @@ class UserSimulatorAgent(AgentAdapter):
             ScenarioConfig.default_config.default_model, str
         ):
             self.model = model or ScenarioConfig.default_config.default_model
+            self._extra_params = extra_params
         elif ScenarioConfig.default_config is not None and isinstance(
             ScenarioConfig.default_config.default_model, ModelConfig
         ):

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/tests/test_judge_agent.py

@@ -92,3 +92,60 @@ async def test_judge_agent_merges_global_config_and_agent_params():
         context_scenario.reset(token)
         # Cleanup
         ScenarioConfig.default_config = None
+
+
+@pytest.mark.asyncio
+async def test_judge_agent_with_string_default_model_config():
+    """JudgeAgent should initialize _extra_params when default_model is a string."""
+    # Setup global config with string default_model
+    ScenarioConfig.default_config = ScenarioConfig(default_model="openai/gpt-4")
+
+    judge = JudgeAgent(
+        criteria=["Test criterion"],
+        temperature=0.5,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+    mock_scenario_state.current_turn = 1
+    mock_scenario_state.config.max_turns = 10
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[{"role": "user", "content": "Hello"}],
+        new_messages=[],
+        judgment_request=True,
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.tool_calls = [MagicMock()]
+    mock_response.choices[0].message.tool_calls[0].function.name = "finish_test"
+    mock_response.choices[0].message.tool_calls[
+        0
+    ].function.arguments = '{"verdict": "success", "reasoning": "Test passed", "criteria": {"test_criterion": true}}'
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.judge_agent.litellm.completion", return_value=mock_response
+        ) as mock_completion:
+            # This should not raise AttributeError: 'JudgeAgent' object has no attribute '_extra_params'
+            await judge.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["temperature"] == 0.5
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None

{langwatch_scenario-0.7.12 → langwatch_scenario-0.7.14}/tests/test_user_simulator_agent.py

@@ -66,3 +66,53 @@ async def test_user_simulator_agent_merges_global_config_and_agent_params():
         context_scenario.reset(token)
         # Cleanup
         ScenarioConfig.default_config = None
+
+
+@pytest.mark.asyncio
+async def test_user_simulator_agent_with_string_default_model_config():
+    """UserSimulatorAgent should initialize _extra_params when default_model is a string."""
+    # Setup global config with string default_model
+    ScenarioConfig.default_config = ScenarioConfig(default_model="openai/gpt-4")
+
+    user_sim = UserSimulatorAgent(
+        temperature=0.7,
+    )
+
+    # Create mock input
+    mock_scenario_state = MagicMock()
+    mock_scenario_state.description = "Test scenario"
+
+    agent_input = AgentInput(
+        thread_id="test",
+        messages=[],
+        new_messages=[],
+        scenario_state=mock_scenario_state,
+    )
+
+    # Mock litellm.completion response
+    mock_response = MagicMock()
+    mock_response.choices = [MagicMock()]
+    mock_response.choices[0].message.content = "test user message"
+
+    # Mock scenario context for cache decorator
+    mock_executor = MagicMock()
+    mock_executor.config = MagicMock()
+    mock_executor.config.cache_key = None
+    token = context_scenario.set(mock_executor)
+
+    try:
+        with patch(
+            "scenario.user_simulator_agent.litellm.completion",
+            return_value=mock_response,
+        ) as mock_completion:
+            # This should not raise AttributeError: 'UserSimulatorAgent' object has no attribute '_extra_params'
+            await user_sim.call(agent_input)
+
+            assert mock_completion.called
+            call_kwargs = mock_completion.call_args.kwargs
+            assert call_kwargs["model"] == "openai/gpt-4"
+            assert call_kwargs["temperature"] == 0.7
+    finally:
+        context_scenario.reset(token)
+        # Cleanup
+        ScenarioConfig.default_config = None