langwatch-scenario 0.4.0__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langwatch_scenario-0.4.0.dist-info → langwatch_scenario-0.6.0.dist-info}/METADATA +93 -71
- langwatch_scenario-0.6.0.dist-info/RECORD +27 -0
- scenario/__init__.py +11 -114
- scenario/_utils/__init__.py +32 -0
- scenario/_utils/ids.py +58 -0
- scenario/_utils/message_conversion.py +103 -0
- scenario/{utils.py → _utils/utils.py} +21 -110
- scenario/agent_adapter.py +8 -4
- scenario/cache.py +4 -3
- scenario/config.py +7 -5
- scenario/events/__init__.py +66 -0
- scenario/events/event_bus.py +175 -0
- scenario/events/event_reporter.py +83 -0
- scenario/events/events.py +169 -0
- scenario/events/messages.py +84 -0
- scenario/events/utils.py +86 -0
- scenario/judge_agent.py +7 -28
- scenario/pytest_plugin.py +2 -47
- scenario/scenario_executor.py +268 -84
- scenario/scenario_state.py +6 -6
- scenario/script.py +9 -9
- scenario/types.py +10 -6
- scenario/user_simulator_agent.py +4 -11
- langwatch_scenario-0.4.0.dist-info/RECORD +0 -18
- {langwatch_scenario-0.4.0.dist-info → langwatch_scenario-0.6.0.dist-info}/WHEEL +0 -0
- {langwatch_scenario-0.4.0.dist-info → langwatch_scenario-0.6.0.dist-info}/entry_points.txt +0 -0
- {langwatch_scenario-0.4.0.dist-info → langwatch_scenario-0.6.0.dist-info}/top_level.txt +0 -0
- /scenario/{error_messages.py → _error_messages.py} +0 -0
scenario/script.py
CHANGED
@@ -32,7 +32,7 @@ def message(message: ChatCompletionMessageParam) -> ScriptStep:
|
|
32
32
|
ScriptStep function that can be used in scenario scripts
|
33
33
|
|
34
34
|
Example:
|
35
|
-
```
|
35
|
+
```
|
36
36
|
result = await scenario.run(
|
37
37
|
name="tool response test",
|
38
38
|
description="Testing tool call responses",
|
@@ -76,7 +76,7 @@ def user(
|
|
76
76
|
ScriptStep function that can be used in scenario scripts
|
77
77
|
|
78
78
|
Example:
|
79
|
-
```
|
79
|
+
```
|
80
80
|
result = await scenario.run(
|
81
81
|
name="user interaction test",
|
82
82
|
description="Testing specific user inputs",
|
@@ -95,7 +95,7 @@ def user(
|
|
95
95
|
scenario.agent(),
|
96
96
|
|
97
97
|
# Structured user message with multimodal content
|
98
|
-
scenario.
|
98
|
+
scenario.message({
|
99
99
|
"role": "user",
|
100
100
|
"content": [
|
101
101
|
{"type": "text", "text": "What's in this image?"},
|
@@ -128,7 +128,7 @@ def agent(
|
|
128
128
|
ScriptStep function that can be used in scenario scripts
|
129
129
|
|
130
130
|
Example:
|
131
|
-
```
|
131
|
+
```
|
132
132
|
result = await scenario.run(
|
133
133
|
name="agent response test",
|
134
134
|
description="Testing agent responses",
|
@@ -148,7 +148,7 @@ def agent(
|
|
148
148
|
scenario.user(), # See how user simulator reacts
|
149
149
|
|
150
150
|
# Structured agent response with tool calls
|
151
|
-
scenario.
|
151
|
+
scenario.message({
|
152
152
|
"role": "assistant",
|
153
153
|
"content": "Let me search for that information",
|
154
154
|
"tool_calls": [{"id": "call_123", "type": "function", ...}]
|
@@ -179,7 +179,7 @@ def judge(
|
|
179
179
|
ScriptStep function that can be used in scenario scripts
|
180
180
|
|
181
181
|
Example:
|
182
|
-
```
|
182
|
+
```
|
183
183
|
result = await scenario.run(
|
184
184
|
name="judge evaluation test",
|
185
185
|
description="Testing judge at specific points",
|
@@ -238,7 +238,7 @@ def proceed(
|
|
238
238
|
ScriptStep function that can be used in scenario scripts
|
239
239
|
|
240
240
|
Example:
|
241
|
-
```
|
241
|
+
```
|
242
242
|
def log_progress(state: ScenarioState) -> None:
|
243
243
|
print(f"Turn {state.current_turn}: {len(state.messages)} messages")
|
244
244
|
|
@@ -288,7 +288,7 @@ def succeed(reasoning: Optional[str] = None) -> ScriptStep:
|
|
288
288
|
ScriptStep function that can be used in scenario scripts
|
289
289
|
|
290
290
|
Example:
|
291
|
-
```
|
291
|
+
```
|
292
292
|
def custom_success_check(state: ScenarioState) -> None:
|
293
293
|
last_msg = state.last_message()
|
294
294
|
if "solution" in last_msg.get("content", "").lower():
|
@@ -331,7 +331,7 @@ def fail(reasoning: Optional[str] = None) -> ScriptStep:
|
|
331
331
|
ScriptStep function that can be used in scenario scripts
|
332
332
|
|
333
333
|
Example:
|
334
|
-
```
|
334
|
+
```
|
335
335
|
def safety_check(state: ScenarioState) -> None:
|
336
336
|
last_msg = state.last_message()
|
337
337
|
content = last_msg.get("content", "")
|
scenario/types.py
CHANGED
@@ -56,7 +56,7 @@ class AgentInput(BaseModel):
|
|
56
56
|
scenario_state: Current state of the scenario execution
|
57
57
|
|
58
58
|
Example:
|
59
|
-
```
|
59
|
+
```
|
60
60
|
class MyAgent(AgentAdapter):
|
61
61
|
async def call(self, input: AgentInput) -> str:
|
62
62
|
# Get the latest user message
|
@@ -89,7 +89,7 @@ class AgentInput(BaseModel):
|
|
89
89
|
ValueError: If no new user messages are found
|
90
90
|
|
91
91
|
Example:
|
92
|
-
```
|
92
|
+
```
|
93
93
|
user_message = input.last_new_user_message()
|
94
94
|
content = user_message["content"]
|
95
95
|
```
|
@@ -115,7 +115,7 @@ class AgentInput(BaseModel):
|
|
115
115
|
ValueError: If no new user messages found or if the message content is not a string
|
116
116
|
|
117
117
|
Example:
|
118
|
-
```
|
118
|
+
```
|
119
119
|
user_text = input.last_new_user_message_str()
|
120
120
|
response = f"You said: {user_text}"
|
121
121
|
```
|
@@ -146,7 +146,7 @@ class ScenarioResult(BaseModel):
|
|
146
146
|
agent_time: Time spent in agent calls in seconds (if measured)
|
147
147
|
|
148
148
|
Example:
|
149
|
-
```
|
149
|
+
```
|
150
150
|
result = await scenario.run(
|
151
151
|
name="weather query",
|
152
152
|
description="User asks about weather",
|
@@ -193,13 +193,17 @@ AgentReturnTypes = Union[
|
|
193
193
|
Union type representing all valid return types for agent adapter call methods.
|
194
194
|
|
195
195
|
Agent adapters can return any of these types:
|
196
|
+
|
196
197
|
- str: Simple text response
|
198
|
+
|
197
199
|
- ChatCompletionMessageParam: Single OpenAI-compatible message
|
200
|
+
|
198
201
|
- List[ChatCompletionMessageParam]: Multiple OpenAI-compatible messages (for multi-step responses)
|
202
|
+
|
199
203
|
- ScenarioResult: Direct test result (typically used by judge agents to end scenarios)
|
200
204
|
|
201
205
|
Example:
|
202
|
-
```
|
206
|
+
```
|
203
207
|
class MyAgent(AgentAdapter):
|
204
208
|
async def call(self, input: AgentInput) -> AgentReturnTypes:
|
205
209
|
# Can return a simple string
|
@@ -234,7 +238,7 @@ scenario state and can optionally return a result to end the scenario.
|
|
234
238
|
The functions can be either synchronous or asynchronous.
|
235
239
|
|
236
240
|
Example:
|
237
|
-
```
|
241
|
+
```
|
238
242
|
def check_tool_call(state: ScenarioState) -> None:
|
239
243
|
assert state.has_tool_call("get_weather")
|
240
244
|
|
scenario/user_simulator_agent.py
CHANGED
@@ -15,10 +15,10 @@ from litellm.files.main import ModelResponse
|
|
15
15
|
|
16
16
|
from scenario.cache import scenario_cache
|
17
17
|
from scenario.agent_adapter import AgentAdapter
|
18
|
-
from scenario.utils import reverse_roles
|
18
|
+
from scenario._utils.utils import reverse_roles
|
19
19
|
from scenario.config import ModelConfig, ScenarioConfig
|
20
20
|
|
21
|
-
from .
|
21
|
+
from ._error_messages import agent_not_configured_error_message
|
22
22
|
from .types import AgentInput, AgentReturnTypes, AgentRole
|
23
23
|
|
24
24
|
|
@@ -43,7 +43,7 @@ class UserSimulatorAgent(AgentAdapter):
|
|
43
43
|
system_prompt: Custom system prompt to override default user simulation behavior
|
44
44
|
|
45
45
|
Example:
|
46
|
-
```
|
46
|
+
```
|
47
47
|
import scenario
|
48
48
|
|
49
49
|
# Basic user simulator with default behavior
|
@@ -112,7 +112,7 @@ class UserSimulatorAgent(AgentAdapter):
|
|
112
112
|
Exception: If no model is configured either in parameters or global config
|
113
113
|
|
114
114
|
Example:
|
115
|
-
```
|
115
|
+
```
|
116
116
|
# Basic user simulator
|
117
117
|
user_sim = UserSimulatorAgent(model="openai/gpt-4.1-mini")
|
118
118
|
|
@@ -175,13 +175,6 @@ class UserSimulatorAgent(AgentAdapter):
|
|
175
175
|
Returns:
|
176
176
|
AgentReturnTypes: A user message in OpenAI format that continues the conversation
|
177
177
|
|
178
|
-
Example:
|
179
|
-
Given a scenario about seeking coding help and previous messages:
|
180
|
-
- Agent: "Hello! How can I help you today?"
|
181
|
-
|
182
|
-
The user simulator might generate:
|
183
|
-
- User: "hi, need help with python error"
|
184
|
-
|
185
178
|
Note:
|
186
179
|
- Messages are generated in a casual, human-like style
|
187
180
|
- The simulator follows the scenario description to stay contextually relevant
|
@@ -1,18 +0,0 @@
|
|
1
|
-
scenario/__init__.py,sha256=oMh5le4c4sIN2K1Ylv2xnkyKHpcOzBeqvW58fTWAFlU,7794
|
2
|
-
scenario/agent_adapter.py,sha256=pd3BdNUWna8h_9hykn1FvcyareMzUofQKKvXaAfQluY,4338
|
3
|
-
scenario/cache.py,sha256=iPpMmjKruLnnxCeLnRiQjiH89LhcVIfQQXKH5etU_m4,6217
|
4
|
-
scenario/config.py,sha256=AeDbKE-_Rrxkan64tDDDynaSNyijoIKHxWaRMqGd4oY,6121
|
5
|
-
scenario/error_messages.py,sha256=6lEx3jBGMbPx0kG0eX5zoZE-ENVM3O_ZkIbVMlnidYs,3892
|
6
|
-
scenario/judge_agent.py,sha256=7fKK_oevXzWKXDioBjHzgGSDpS0aby3oRcrc6oaip68,16973
|
7
|
-
scenario/pytest_plugin.py,sha256=s2M2mll9JSCSWB5SKDQIWT5DOCvzZOo_8JCCfJzyy8k,12849
|
8
|
-
scenario/scenario_executor.py,sha256=oz7Odv41HNLcNd_7sKUW-AKKdY-on_PyVLaxpvKjrGE,27211
|
9
|
-
scenario/scenario_state.py,sha256=I_fWoY_LvNuKCBL-b62z5bQOAI25dx55FuZNWwtIeVs,7075
|
10
|
-
scenario/script.py,sha256=7wsHZxdSgFaYLflkV6sysDxefkkag79mySR7yp7N3ug,12278
|
11
|
-
scenario/types.py,sha256=CsexCupg2WUi4dToYF5RqFdNIHx1JhaRaRRBs78YVd0,9498
|
12
|
-
scenario/user_simulator_agent.py,sha256=o8sZLMWOcTf7BKgPO_a5rPnC6GgdZQe3HujqwjPzjV8,9346
|
13
|
-
scenario/utils.py,sha256=ryJYcMoSAjVzA_f5V6Mcga5GkipYbCzaYNNpBjAQI_g,16992
|
14
|
-
langwatch_scenario-0.4.0.dist-info/METADATA,sha256=d9tNTNioHH5_1q8oIvIABaTgC6J9XmEJR4Tjim3sFks,13827
|
15
|
-
langwatch_scenario-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
16
|
-
langwatch_scenario-0.4.0.dist-info/entry_points.txt,sha256=WlEnJ_gku0i18bIa3DSuGqXRX-QDQLe_s0YmRzK45TI,45
|
17
|
-
langwatch_scenario-0.4.0.dist-info/top_level.txt,sha256=45Mn28aedJsetnBMB5xSmrJ-yo701QLH89Zlz4r1clE,9
|
18
|
-
langwatch_scenario-0.4.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|