pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
"""Result models for agent execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from pytest_copilot.copilot.requests import RequestAudit
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(slots=True)
|
|
13
|
+
class UsageInfo:
|
|
14
|
+
"""Token usage from a single model turn."""
|
|
15
|
+
|
|
16
|
+
model: str
|
|
17
|
+
input_tokens: int | None = None
|
|
18
|
+
output_tokens: int | None = None
|
|
19
|
+
cache_read_tokens: int | None = None
|
|
20
|
+
cache_write_tokens: int | None = None
|
|
21
|
+
reasoning_tokens: int | None = None
|
|
22
|
+
reasoning_effort: str | None = None
|
|
23
|
+
duration_ms: float | None = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
StopReason = Literal[
|
|
27
|
+
"completed",
|
|
28
|
+
"tool_budget_exceeded",
|
|
29
|
+
"timeout",
|
|
30
|
+
"request_audit_error",
|
|
31
|
+
"execution_error",
|
|
32
|
+
"cleanup_error",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(slots=True)
|
|
37
|
+
class ImageContent:
|
|
38
|
+
"""Binary image data returned by a tool call.
|
|
39
|
+
|
|
40
|
+
Example:
|
|
41
|
+
screenshots = result.tool_images_for("screenshot")
|
|
42
|
+
assert len(screenshots) > 0
|
|
43
|
+
assert screenshots[-1].media_type == "image/png"
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
data: bytes
|
|
47
|
+
media_type: str # e.g. "image/png"
|
|
48
|
+
|
|
49
|
+
def __repr__(self) -> str:
|
|
50
|
+
return f"ImageContent({self.media_type}, {len(self.data)} bytes)"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(slots=True)
|
|
54
|
+
class ToolCall:
|
|
55
|
+
"""A tool call made by the agent."""
|
|
56
|
+
|
|
57
|
+
name: str
|
|
58
|
+
arguments: dict[str, Any]
|
|
59
|
+
result: str | None = None
|
|
60
|
+
error: str | None = None
|
|
61
|
+
duration_ms: float | None = None
|
|
62
|
+
image_content: bytes | None = None
|
|
63
|
+
image_media_type: str | None = None
|
|
64
|
+
additional_images: list[ImageContent] = field(default_factory=list)
|
|
65
|
+
call_id: str | None = None
|
|
66
|
+
completion_received: bool | None = None
|
|
67
|
+
success: bool | None = None
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def evidence_complete(self) -> bool:
|
|
71
|
+
"""Whether completion, outcome, and returned content or error were captured."""
|
|
72
|
+
return (
|
|
73
|
+
self.completion_received is True
|
|
74
|
+
and self.success is not None
|
|
75
|
+
and (
|
|
76
|
+
self.result is not None
|
|
77
|
+
or self.error is not None
|
|
78
|
+
or self.image_content is not None
|
|
79
|
+
or bool(self.additional_images)
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
def __repr__(self) -> str:
|
|
84
|
+
status = "error" if self.error else "ok"
|
|
85
|
+
timing = f", {self.duration_ms:.1f}ms" if self.duration_ms else ""
|
|
86
|
+
image = ", image" if self.image_content else ""
|
|
87
|
+
return f"ToolCall({self.name}, {status}{timing}{image})"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(slots=True)
|
|
91
|
+
class ToolInfo:
|
|
92
|
+
"""Recorded metadata about an MCP tool.
|
|
93
|
+
|
|
94
|
+
Captures the description and schema exposed to the model for inspection.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
name: str
|
|
98
|
+
description: str
|
|
99
|
+
input_schema: dict[str, Any]
|
|
100
|
+
server_name: str
|
|
101
|
+
|
|
102
|
+
def __repr__(self) -> str:
|
|
103
|
+
return f"ToolInfo({self.name} from {self.server_name})"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(slots=True)
|
|
107
|
+
class MCPPromptArgument:
|
|
108
|
+
"""An argument for an MCP prompt template."""
|
|
109
|
+
|
|
110
|
+
name: str
|
|
111
|
+
description: str = ""
|
|
112
|
+
required: bool = False
|
|
113
|
+
|
|
114
|
+
def __repr__(self) -> str:
|
|
115
|
+
req = ", required" if self.required else ""
|
|
116
|
+
return f"MCPPromptArgument({self.name!r}{req})"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(slots=True)
|
|
120
|
+
class MCPPrompt:
|
|
121
|
+
"""A prompt template exposed by an MCP server.
|
|
122
|
+
|
|
123
|
+
MCP servers can bundle reusable prompt templates alongside their tools.
|
|
124
|
+
These appear in VS Code as slash commands (e.g. ``/mcp.servername.promptname``).
|
|
125
|
+
Use :meth:`MCPServerProcess.list_prompts` to discover them and
|
|
126
|
+
:meth:`MCPServerProcess.get_prompt` to render one with arguments.
|
|
127
|
+
|
|
128
|
+
Example::
|
|
129
|
+
|
|
130
|
+
server = MCPServerProcess(mcp_config)
|
|
131
|
+
await server.start()
|
|
132
|
+
prompts = await server.list_prompts()
|
|
133
|
+
messages = await server.get_prompt("code_review", {"code": "..."})
|
|
134
|
+
result = await copilot_run(agent, messages[0]["content"])
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
name: str
|
|
138
|
+
description: str = ""
|
|
139
|
+
arguments: list[MCPPromptArgument] = field(default_factory=list)
|
|
140
|
+
|
|
141
|
+
def __repr__(self) -> str:
|
|
142
|
+
args = f", {len(self.arguments)} args" if self.arguments else ""
|
|
143
|
+
return f"MCPPrompt({self.name!r}{args})"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass(slots=True)
|
|
147
|
+
class CustomAgentInfo:
|
|
148
|
+
"""Metadata about a custom agent (.agent.md file) under evaluation."""
|
|
149
|
+
|
|
150
|
+
name: str
|
|
151
|
+
description: str = ""
|
|
152
|
+
file_path: str = ""
|
|
153
|
+
|
|
154
|
+
def __repr__(self) -> str:
|
|
155
|
+
return f"CustomAgentInfo({self.name!r})"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@dataclass(slots=True)
|
|
159
|
+
class InstructionFileInfo:
|
|
160
|
+
"""Metadata about a custom instruction file under evaluation."""
|
|
161
|
+
|
|
162
|
+
name: str
|
|
163
|
+
file_path: str = ""
|
|
164
|
+
apply_to: str = "" # glob pattern from frontmatter (e.g. "**/*.py")
|
|
165
|
+
description: str = ""
|
|
166
|
+
|
|
167
|
+
def __repr__(self) -> str:
|
|
168
|
+
return f"InstructionFileInfo({self.name!r})"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass(slots=True)
|
|
172
|
+
class SkillInfo:
|
|
173
|
+
"""Recorded metadata about a skill.
|
|
174
|
+
|
|
175
|
+
Captures the skill's system prompt content and references for inspection.
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
name: str
|
|
179
|
+
description: str
|
|
180
|
+
instruction_content: str
|
|
181
|
+
reference_names: list[str] = field(default_factory=list)
|
|
182
|
+
|
|
183
|
+
def __repr__(self) -> str:
|
|
184
|
+
refs = f", {len(self.reference_names)} refs" if self.reference_names else ""
|
|
185
|
+
return f"SkillInfo({self.name}{refs})"
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass(slots=True, frozen=True)
|
|
189
|
+
class DiscoveredSkill:
|
|
190
|
+
"""Skill metadata actually returned by the public SDK, not observed use."""
|
|
191
|
+
|
|
192
|
+
name: str
|
|
193
|
+
description: str
|
|
194
|
+
source: str
|
|
195
|
+
enabled: bool
|
|
196
|
+
user_invocable: bool
|
|
197
|
+
path: str | None = None
|
|
198
|
+
plugin_name: str | None = None
|
|
199
|
+
command_name: str | None = None
|
|
200
|
+
argument_hint: str | None = None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@dataclass(slots=True)
|
|
204
|
+
class SkillDiscovery:
|
|
205
|
+
"""Pre-execution discovery evidence, including incomplete or failed checks."""
|
|
206
|
+
|
|
207
|
+
skills: list[DiscoveredSkill] = field(default_factory=list)
|
|
208
|
+
warnings: list[str] = field(default_factory=list)
|
|
209
|
+
errors: list[str] = field(default_factory=list)
|
|
210
|
+
complete: bool = False
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@dataclass(slots=True)
|
|
214
|
+
class ClarificationStats:
|
|
215
|
+
"""Statistics about clarification requests detected during execution.
|
|
216
|
+
|
|
217
|
+
Tracks when the agent asks for user input instead of executing the task.
|
|
218
|
+
Only populated when clarification_detection is enabled on the agent.
|
|
219
|
+
|
|
220
|
+
Example:
|
|
221
|
+
result = await copilot_run(agent, "Check my balance")
|
|
222
|
+
if result.clarification_stats:
|
|
223
|
+
print(f"Eval asked {result.clarification_stats.count} question(s)")
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
count: int = 0
|
|
227
|
+
turn_indices: list[int] = field(default_factory=list)
|
|
228
|
+
examples: list[str] = field(default_factory=list) # Max 3 examples stored
|
|
229
|
+
|
|
230
|
+
def __repr__(self) -> str:
|
|
231
|
+
return f"ClarificationStats(count={self.count}, turns={self.turn_indices})"
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@dataclass(slots=True)
|
|
235
|
+
class Assertion:
|
|
236
|
+
"""A single assertion result from a test."""
|
|
237
|
+
|
|
238
|
+
type: str # "semantic", "condition", etc.
|
|
239
|
+
passed: bool
|
|
240
|
+
message: str
|
|
241
|
+
details: str | None = None
|
|
242
|
+
|
|
243
|
+
def __repr__(self) -> str:
|
|
244
|
+
status = "✓" if self.passed else "✗"
|
|
245
|
+
return f"Assertion({status} {self.message})"
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@dataclass(slots=True)
|
|
249
|
+
class SubagentInvocation:
|
|
250
|
+
"""A subagent invocation observed during agent execution.
|
|
251
|
+
|
|
252
|
+
Tracks when an orchestrator agent dispatches work to a named sub-agent,
|
|
253
|
+
along with the final status and duration of that invocation.
|
|
254
|
+
|
|
255
|
+
Example:
|
|
256
|
+
result = await copilot_run(agent, "Build and test the project")
|
|
257
|
+
assert any(s.name == "coder" for s in result.subagent_invocations)
|
|
258
|
+
assert all(s.status == "completed" for s in result.subagent_invocations)
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
invocation_id: str
|
|
262
|
+
name: str
|
|
263
|
+
status: str # "selected", "started", "completed", "failed"
|
|
264
|
+
duration_ms: float | None = None
|
|
265
|
+
result: EvalResult | None = None
|
|
266
|
+
|
|
267
|
+
def __repr__(self) -> str:
|
|
268
|
+
return f"SubagentInvocation({self.name}, {self.status})"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@dataclass(slots=True)
|
|
272
|
+
class Turn:
|
|
273
|
+
"""A single conversational turn."""
|
|
274
|
+
|
|
275
|
+
role: str # "user", "assistant", "tool"
|
|
276
|
+
content: str
|
|
277
|
+
tool_calls: list[ToolCall] = field(default_factory=list)
|
|
278
|
+
|
|
279
|
+
@property
|
|
280
|
+
def text(self) -> str:
|
|
281
|
+
"""Get the text content of this turn."""
|
|
282
|
+
return self.content
|
|
283
|
+
|
|
284
|
+
def __repr__(self) -> str:
|
|
285
|
+
preview = self.content[:50] + "..." if len(self.content) > 50 else self.content
|
|
286
|
+
return f"Turn({self.role}: {preview!r})"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@dataclass(slots=True)
|
|
290
|
+
class EvalResult:
|
|
291
|
+
"""Result of running an agent with rich inspection capabilities.
|
|
292
|
+
|
|
293
|
+
Example:
|
|
294
|
+
result = await copilot_run(agent, "Hello!")
|
|
295
|
+
assert result.success
|
|
296
|
+
assert "hello" in result.final_response.lower()
|
|
297
|
+
assert result.tool_was_called("read_file")
|
|
298
|
+
|
|
299
|
+
"""
|
|
300
|
+
|
|
301
|
+
turns: list[Turn]
|
|
302
|
+
success: bool
|
|
303
|
+
error: str | None = None
|
|
304
|
+
duration_ms: float = 0.0
|
|
305
|
+
token_usage: dict[str, int] = field(default_factory=dict)
|
|
306
|
+
cost_usd: float = 0.0
|
|
307
|
+
_messages: list[Any] = field(default_factory=list)
|
|
308
|
+
session_context_count: int = 0 # Number of prior messages passed in
|
|
309
|
+
assertions: list[Assertion] = field(default_factory=list) # Assertion results
|
|
310
|
+
|
|
311
|
+
# Recorded interface metadata.
|
|
312
|
+
available_tools: list[ToolInfo] = field(default_factory=list)
|
|
313
|
+
skill_info: SkillInfo | None = None
|
|
314
|
+
skill_discovery: SkillDiscovery | None = None
|
|
315
|
+
effective_system_prompt: str | None = None
|
|
316
|
+
mcp_prompts: list[MCPPrompt] = field(default_factory=list)
|
|
317
|
+
prompt_name: str | None = None
|
|
318
|
+
custom_agent_info: CustomAgentInfo | None = None
|
|
319
|
+
premium_requests: float = 0.0
|
|
320
|
+
instruction_files: list[InstructionFileInfo] = field(default_factory=list)
|
|
321
|
+
configuration: dict[str, Any] | None = None
|
|
322
|
+
capture_errors: list[str] = field(default_factory=list)
|
|
323
|
+
evidence_complete: bool | None = None
|
|
324
|
+
request_audit: list[RequestAudit] = field(default_factory=list)
|
|
325
|
+
stop_reason: StopReason | None = None
|
|
326
|
+
usage: list[UsageInfo] = field(default_factory=list)
|
|
327
|
+
tool_calls_admitted: int = 0
|
|
328
|
+
model_used: str | None = None
|
|
329
|
+
reasoning_traces: list[str] = field(default_factory=list)
|
|
330
|
+
permission_requested: bool = False
|
|
331
|
+
permissions: list[dict[str, Any]] = field(default_factory=list)
|
|
332
|
+
subagent_invocations: list[SubagentInvocation] = field(default_factory=list)
|
|
333
|
+
comparison_role: Literal["baseline", "treatment"] | None = None
|
|
334
|
+
|
|
335
|
+
@property
|
|
336
|
+
def execution_results(self) -> list[EvalResult]:
|
|
337
|
+
"""This session and separately captured framework-owned descendants."""
|
|
338
|
+
return [
|
|
339
|
+
self,
|
|
340
|
+
*(
|
|
341
|
+
result
|
|
342
|
+
for invocation in self.subagent_invocations
|
|
343
|
+
if invocation.result is not None
|
|
344
|
+
for result in invocation.result.execution_results
|
|
345
|
+
),
|
|
346
|
+
]
|
|
347
|
+
|
|
348
|
+
# Clarification detection
|
|
349
|
+
clarification_stats: ClarificationStats | None = None
|
|
350
|
+
|
|
351
|
+
@property
|
|
352
|
+
def messages(self) -> list[Any]:
|
|
353
|
+
"""Get full conversation messages for session continuity.
|
|
354
|
+
|
|
355
|
+
This is a copy of captured messages for inspection. The public
|
|
356
|
+
``copilot_run`` fixture does not accept a message-history argument or
|
|
357
|
+
reuse conversation state between calls.
|
|
358
|
+
"""
|
|
359
|
+
return list(self._messages) # Return copy to prevent mutation
|
|
360
|
+
|
|
361
|
+
@property
|
|
362
|
+
def is_session_continuation(self) -> bool:
|
|
363
|
+
"""Check if this result is part of a multi-turn session.
|
|
364
|
+
|
|
365
|
+
Returns True if prior messages were passed via the messages parameter.
|
|
366
|
+
"""
|
|
367
|
+
return self.session_context_count > 0
|
|
368
|
+
|
|
369
|
+
@property
|
|
370
|
+
def final_response(self) -> str:
|
|
371
|
+
"""Get the last assistant response."""
|
|
372
|
+
for turn in reversed(self.turns):
|
|
373
|
+
if turn.role == "assistant":
|
|
374
|
+
return turn.content
|
|
375
|
+
return ""
|
|
376
|
+
|
|
377
|
+
@property
|
|
378
|
+
def all_responses(self) -> list[str]:
|
|
379
|
+
"""Get all assistant responses."""
|
|
380
|
+
return [t.content for t in self.turns if t.role == "assistant"]
|
|
381
|
+
|
|
382
|
+
@property
|
|
383
|
+
def all_tool_calls(self) -> list[ToolCall]:
|
|
384
|
+
"""Get all tool calls across all turns."""
|
|
385
|
+
calls = []
|
|
386
|
+
for turn in self.turns:
|
|
387
|
+
calls.extend(turn.tool_calls)
|
|
388
|
+
return calls
|
|
389
|
+
|
|
390
|
+
@property
|
|
391
|
+
def tool_names_called(self) -> set[str]:
|
|
392
|
+
"""Get set of all tool names that were called."""
|
|
393
|
+
return {call.name for call in self.all_tool_calls}
|
|
394
|
+
|
|
395
|
+
def tool_was_called(self, name: str) -> bool:
|
|
396
|
+
"""Check if a specific tool was called."""
|
|
397
|
+
return name in self.tool_names_called
|
|
398
|
+
|
|
399
|
+
def tool_was_called_from_server(self, server_name: str, tool_name: str) -> bool:
|
|
400
|
+
"""Check if a specific tool from a named MCP server was called.
|
|
401
|
+
|
|
402
|
+
Useful for plugin testing where tools may be namespaced by server.
|
|
403
|
+
Checks both the plain tool name and the ``server_tool`` prefixed form.
|
|
404
|
+
|
|
405
|
+
Args:
|
|
406
|
+
server_name: Name of the MCP server (e.g., ``"filesystem"``).
|
|
407
|
+
tool_name: Name of the tool (e.g., ``"read_file"``).
|
|
408
|
+
|
|
409
|
+
Returns:
|
|
410
|
+
True if either ``tool_name`` or ``server_name_tool_name`` was called.
|
|
411
|
+
|
|
412
|
+
Example::
|
|
413
|
+
|
|
414
|
+
assert result.tool_was_called_from_server("banking", "get_balance")
|
|
415
|
+
"""
|
|
416
|
+
full_name = f"{server_name}_{tool_name}"
|
|
417
|
+
return self.tool_was_called(tool_name) or self.tool_was_called(full_name)
|
|
418
|
+
|
|
419
|
+
def tool_call_count(self, name: str) -> int:
|
|
420
|
+
"""Count how many times a specific tool was called."""
|
|
421
|
+
return len(self.tool_calls_for(name))
|
|
422
|
+
|
|
423
|
+
def tool_calls_for(self, name: str) -> list[ToolCall]:
|
|
424
|
+
"""Get all calls to a specific tool."""
|
|
425
|
+
return [c for c in self.all_tool_calls if c.name == name]
|
|
426
|
+
|
|
427
|
+
def tool_call_arg(self, tool_name: str, arg_name: str) -> Any:
|
|
428
|
+
"""Get argument value from the first call to a tool.
|
|
429
|
+
|
|
430
|
+
Args:
|
|
431
|
+
tool_name: Name of the tool
|
|
432
|
+
arg_name: Name of the argument
|
|
433
|
+
|
|
434
|
+
Returns:
|
|
435
|
+
Argument value or None if not found
|
|
436
|
+
"""
|
|
437
|
+
calls = self.tool_calls_for(tool_name)
|
|
438
|
+
if calls:
|
|
439
|
+
return calls[0].arguments.get(arg_name)
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
def tool_was_called_with(self, name: str, **expected_args: Any) -> bool:
|
|
443
|
+
"""Check if a tool was called with specific argument values.
|
|
444
|
+
|
|
445
|
+
Returns True if at least one call to the named tool has all the
|
|
446
|
+
specified argument key-value pairs.
|
|
447
|
+
|
|
448
|
+
Args:
|
|
449
|
+
name: Tool name to check.
|
|
450
|
+
**expected_args: Expected argument key-value pairs.
|
|
451
|
+
|
|
452
|
+
Example::
|
|
453
|
+
|
|
454
|
+
assert result.tool_was_called_with("size_vm", region="westeurope", cores=8)
|
|
455
|
+
assert result.tool_was_called_with("get_balance", account="checking")
|
|
456
|
+
"""
|
|
457
|
+
for call in self.tool_calls_for(name):
|
|
458
|
+
if all(call.arguments.get(k) == v for k, v in expected_args.items()):
|
|
459
|
+
return True
|
|
460
|
+
return False
|
|
461
|
+
|
|
462
|
+
def tool_images_for(self, name: str) -> list[ImageContent]:
|
|
463
|
+
"""Get all images returned by a specific tool.
|
|
464
|
+
|
|
465
|
+
Args:
|
|
466
|
+
name: Name of the tool (e.g., "screenshot")
|
|
467
|
+
|
|
468
|
+
Returns:
|
|
469
|
+
List of ImageContent objects from tool calls that returned images.
|
|
470
|
+
|
|
471
|
+
Example:
|
|
472
|
+
screenshots = result.tool_images_for("screenshot")
|
|
473
|
+
assert len(screenshots) > 0
|
|
474
|
+
assert screenshots[-1].media_type == "image/png"
|
|
475
|
+
"""
|
|
476
|
+
images: list[ImageContent] = []
|
|
477
|
+
for call in self.tool_calls_for(name):
|
|
478
|
+
if call.image_content is not None:
|
|
479
|
+
images.append(
|
|
480
|
+
ImageContent(call.image_content, call.image_media_type or "image/png")
|
|
481
|
+
)
|
|
482
|
+
images.extend(call.additional_images)
|
|
483
|
+
return images
|
|
484
|
+
|
|
485
|
+
@property
|
|
486
|
+
def asked_for_clarification(self) -> bool:
|
|
487
|
+
"""Check if the agent asked for clarification instead of acting.
|
|
488
|
+
|
|
489
|
+
Returns True if clarification detection was enabled AND the agent
|
|
490
|
+
asked at least one clarifying question.
|
|
491
|
+
|
|
492
|
+
Example:
|
|
493
|
+
result = await copilot_run(agent, "Check my balance")
|
|
494
|
+
assert not result.asked_for_clarification
|
|
495
|
+
"""
|
|
496
|
+
return self.clarification_stats is not None and self.clarification_stats.count > 0
|
|
497
|
+
|
|
498
|
+
@property
|
|
499
|
+
def clarification_count(self) -> int:
|
|
500
|
+
"""Number of times the agent asked for clarification."""
|
|
501
|
+
if self.clarification_stats is None:
|
|
502
|
+
return 0
|
|
503
|
+
return self.clarification_stats.count
|
|
504
|
+
|
|
505
|
+
@property
|
|
506
|
+
def tool_context(self) -> str:
|
|
507
|
+
"""Summarise tool calls and their results as plain text.
|
|
508
|
+
|
|
509
|
+
This is a readable evidence summary for investigation. It is not an
|
|
510
|
+
independent correctness check and does not replace the full trace.
|
|
511
|
+
"""
|
|
512
|
+
calls = self.all_tool_calls
|
|
513
|
+
if not calls:
|
|
514
|
+
return "No tools were called."
|
|
515
|
+
lines: list[str] = []
|
|
516
|
+
for i, call in enumerate(calls, 1):
|
|
517
|
+
lines.append(f"## Tool call {i}: {call.name}")
|
|
518
|
+
if call.arguments:
|
|
519
|
+
args = ", ".join(f"{k}={v!r}" for k, v in call.arguments.items())
|
|
520
|
+
lines.append(f"Arguments: {args}")
|
|
521
|
+
if call.error:
|
|
522
|
+
lines.append(f"Error: {call.error}")
|
|
523
|
+
elif call.result:
|
|
524
|
+
lines.append(f"Result: {call.result}")
|
|
525
|
+
lines.append("")
|
|
526
|
+
return "\n".join(lines)
|
|
527
|
+
|
|
528
|
+
def __repr__(self) -> str:
|
|
529
|
+
status = "SUCCESS" if self.success else f"FAILED: {self.error}"
|
|
530
|
+
tools = ", ".join(sorted(self.tool_names_called)) or "none"
|
|
531
|
+
cost_str = f"${self.cost_usd:.6f}" if self.cost_usd > 0 else "N/A"
|
|
532
|
+
tokens = self.token_usage.get("prompt", 0) + self.token_usage.get("completion", 0)
|
|
533
|
+
return (
|
|
534
|
+
f"EvalResult({status})\n"
|
|
535
|
+
f" Turns: {len(self.turns)}\n"
|
|
536
|
+
f" Tools called: {tools}\n"
|
|
537
|
+
f" Duration: {self.duration_ms:.0f}ms\n"
|
|
538
|
+
f" Tokens: {tokens} | Cost: {cost_str}\n"
|
|
539
|
+
f" Final: {self.final_response[:100]!r}..."
|
|
540
|
+
)
|
|
541
|
+
|
|
542
|
+
def __bool__(self) -> bool:
|
|
543
|
+
return self.success
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Lossless serialization and strict loading of the current native report schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import binascii
|
|
7
|
+
import sys
|
|
8
|
+
from dataclasses import fields, is_dataclass
|
|
9
|
+
from types import UnionType
|
|
10
|
+
from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union, get_args, get_origin, get_type_hints
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from pytest_copilot.reporting.collector import SuiteReport
|
|
14
|
+
|
|
15
|
+
T = TypeVar("T")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def serialize_dataclass(obj: Any) -> Any:
|
|
19
|
+
"""Convert public dataclass fields recursively, encoding bytes as base64."""
|
|
20
|
+
if is_dataclass(obj) and not isinstance(obj, type):
|
|
21
|
+
return {
|
|
22
|
+
field.name: serialize_dataclass(getattr(obj, field.name))
|
|
23
|
+
for field in fields(obj)
|
|
24
|
+
if not field.name.startswith("_")
|
|
25
|
+
}
|
|
26
|
+
if isinstance(obj, (list, tuple)):
|
|
27
|
+
return [serialize_dataclass(item) for item in obj]
|
|
28
|
+
if isinstance(obj, dict):
|
|
29
|
+
return {key: serialize_dataclass(value) for key, value in obj.items()}
|
|
30
|
+
if isinstance(obj, bytes):
|
|
31
|
+
return base64.b64encode(obj).decode("ascii")
|
|
32
|
+
return obj
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _decode_dataclass(cls: type[T], data: Any, *, path: str) -> T:
|
|
36
|
+
from pytest_copilot.copilot.requests import RequestAudit
|
|
37
|
+
from pytest_copilot.core.result import EvalResult
|
|
38
|
+
|
|
39
|
+
if not is_dataclass(cls):
|
|
40
|
+
raise TypeError(f"{cls} is not a native report dataclass")
|
|
41
|
+
context = f"{cls.__name__} at {path}"
|
|
42
|
+
if not isinstance(data, dict):
|
|
43
|
+
raise ValueError(f"{context} must be an object")
|
|
44
|
+
public_fields = [field for field in fields(cls) if not field.name.startswith("_")]
|
|
45
|
+
unexpected = data.keys() - {field.name for field in public_fields}
|
|
46
|
+
if unexpected:
|
|
47
|
+
raise ValueError(f"{context} has unexpected fields: {sorted(unexpected)}")
|
|
48
|
+
annotations = get_type_hints(
|
|
49
|
+
cls,
|
|
50
|
+
globalns={
|
|
51
|
+
**vars(sys.modules[cls.__module__]),
|
|
52
|
+
"RequestAudit": RequestAudit,
|
|
53
|
+
"EvalResult": EvalResult,
|
|
54
|
+
},
|
|
55
|
+
)
|
|
56
|
+
values: dict[str, Any] = {}
|
|
57
|
+
for field in public_fields:
|
|
58
|
+
if field.name not in data:
|
|
59
|
+
raise ValueError(f"{context} is missing required field {field.name!r}")
|
|
60
|
+
values[field.name] = _decode_value(
|
|
61
|
+
annotations[field.name], data[field.name], path=f"{path}.{field.name}"
|
|
62
|
+
)
|
|
63
|
+
return cls(**values)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _decode_value(annotation: Any, value: Any, *, path: str) -> Any:
|
|
67
|
+
origin = get_origin(annotation)
|
|
68
|
+
arguments = get_args(annotation)
|
|
69
|
+
if annotation is Any:
|
|
70
|
+
return value
|
|
71
|
+
if origin in (Union, UnionType):
|
|
72
|
+
if value is None and type(None) in arguments:
|
|
73
|
+
return None
|
|
74
|
+
alternatives = [item for item in arguments if item is not type(None)]
|
|
75
|
+
if len(alternatives) != 1:
|
|
76
|
+
raise TypeError(f"Unsupported native report field type at {path}: {annotation}")
|
|
77
|
+
return _decode_value(alternatives[0], value, path=path)
|
|
78
|
+
if origin is Literal:
|
|
79
|
+
if value not in arguments:
|
|
80
|
+
raise ValueError(f"{path} must be one of {arguments!r}")
|
|
81
|
+
return value
|
|
82
|
+
if origin in (list, tuple):
|
|
83
|
+
if not isinstance(value, list):
|
|
84
|
+
raise ValueError(f"{path} must be a list")
|
|
85
|
+
if origin is list:
|
|
86
|
+
return [
|
|
87
|
+
_decode_value(arguments[0], item, path=f"{path}[{index}]")
|
|
88
|
+
for index, item in enumerate(value)
|
|
89
|
+
]
|
|
90
|
+
if len(value) != len(arguments):
|
|
91
|
+
raise ValueError(f"{path} must contain {len(arguments)} values")
|
|
92
|
+
return tuple(
|
|
93
|
+
_decode_value(kind, item, path=f"{path}[{index}]")
|
|
94
|
+
for index, (kind, item) in enumerate(zip(arguments, value, strict=True))
|
|
95
|
+
)
|
|
96
|
+
if origin is dict:
|
|
97
|
+
if not isinstance(value, dict):
|
|
98
|
+
raise ValueError(f"{path} must be an object")
|
|
99
|
+
return {
|
|
100
|
+
_decode_value(arguments[0], key, path=f"{path} key"): _decode_value(
|
|
101
|
+
arguments[1], item, path=f"{path}[{key!r}]"
|
|
102
|
+
)
|
|
103
|
+
for key, item in value.items()
|
|
104
|
+
}
|
|
105
|
+
if isinstance(annotation, type) and is_dataclass(annotation):
|
|
106
|
+
return _decode_dataclass(annotation, value, path=path)
|
|
107
|
+
if annotation is bytes:
|
|
108
|
+
if not isinstance(value, str):
|
|
109
|
+
raise ValueError(f"{path} must be a base64 string")
|
|
110
|
+
try:
|
|
111
|
+
return base64.b64decode(value, validate=True)
|
|
112
|
+
except (ValueError, binascii.Error) as exc:
|
|
113
|
+
raise ValueError(f"{path} contains invalid base64 image data") from exc
|
|
114
|
+
if annotation is float:
|
|
115
|
+
if type(value) not in (int, float):
|
|
116
|
+
raise ValueError(f"{path} must be a number")
|
|
117
|
+
return float(value)
|
|
118
|
+
if annotation in (str, int, bool) and type(value) is annotation:
|
|
119
|
+
return value
|
|
120
|
+
raise ValueError(f"{path} must be {annotation.__name__}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
|
|
124
|
+
"""Load every current producer field; missing evidence is never inferred."""
|
|
125
|
+
from pytest_copilot.reporting.collector import SuiteReport
|
|
126
|
+
|
|
127
|
+
payload = {key: value for key, value in data.items() if key != "schema_version"}
|
|
128
|
+
return _decode_dataclass(SuiteReport, payload, path="suite")
|