pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,543 @@
1
+ """Result models for agent execution."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from typing import TYPE_CHECKING, Any, Literal
7
+
8
+ if TYPE_CHECKING:
9
+ from pytest_copilot.copilot.requests import RequestAudit
10
+
11
+
12
+ @dataclass(slots=True)
13
+ class UsageInfo:
14
+ """Token usage from a single model turn."""
15
+
16
+ model: str
17
+ input_tokens: int | None = None
18
+ output_tokens: int | None = None
19
+ cache_read_tokens: int | None = None
20
+ cache_write_tokens: int | None = None
21
+ reasoning_tokens: int | None = None
22
+ reasoning_effort: str | None = None
23
+ duration_ms: float | None = None
24
+
25
+
26
+ StopReason = Literal[
27
+ "completed",
28
+ "tool_budget_exceeded",
29
+ "timeout",
30
+ "request_audit_error",
31
+ "execution_error",
32
+ "cleanup_error",
33
+ ]
34
+
35
+
36
+ @dataclass(slots=True)
37
+ class ImageContent:
38
+ """Binary image data returned by a tool call.
39
+
40
+ Example:
41
+ screenshots = result.tool_images_for("screenshot")
42
+ assert len(screenshots) > 0
43
+ assert screenshots[-1].media_type == "image/png"
44
+ """
45
+
46
+ data: bytes
47
+ media_type: str # e.g. "image/png"
48
+
49
+ def __repr__(self) -> str:
50
+ return f"ImageContent({self.media_type}, {len(self.data)} bytes)"
51
+
52
+
53
+ @dataclass(slots=True)
54
+ class ToolCall:
55
+ """A tool call made by the agent."""
56
+
57
+ name: str
58
+ arguments: dict[str, Any]
59
+ result: str | None = None
60
+ error: str | None = None
61
+ duration_ms: float | None = None
62
+ image_content: bytes | None = None
63
+ image_media_type: str | None = None
64
+ additional_images: list[ImageContent] = field(default_factory=list)
65
+ call_id: str | None = None
66
+ completion_received: bool | None = None
67
+ success: bool | None = None
68
+
69
+ @property
70
+ def evidence_complete(self) -> bool:
71
+ """Whether completion, outcome, and returned content or error were captured."""
72
+ return (
73
+ self.completion_received is True
74
+ and self.success is not None
75
+ and (
76
+ self.result is not None
77
+ or self.error is not None
78
+ or self.image_content is not None
79
+ or bool(self.additional_images)
80
+ )
81
+ )
82
+
83
+ def __repr__(self) -> str:
84
+ status = "error" if self.error else "ok"
85
+ timing = f", {self.duration_ms:.1f}ms" if self.duration_ms else ""
86
+ image = ", image" if self.image_content else ""
87
+ return f"ToolCall({self.name}, {status}{timing}{image})"
88
+
89
+
90
+ @dataclass(slots=True)
91
+ class ToolInfo:
92
+ """Recorded metadata about an MCP tool.
93
+
94
+ Captures the description and schema exposed to the model for inspection.
95
+ """
96
+
97
+ name: str
98
+ description: str
99
+ input_schema: dict[str, Any]
100
+ server_name: str
101
+
102
+ def __repr__(self) -> str:
103
+ return f"ToolInfo({self.name} from {self.server_name})"
104
+
105
+
106
+ @dataclass(slots=True)
107
+ class MCPPromptArgument:
108
+ """An argument for an MCP prompt template."""
109
+
110
+ name: str
111
+ description: str = ""
112
+ required: bool = False
113
+
114
+ def __repr__(self) -> str:
115
+ req = ", required" if self.required else ""
116
+ return f"MCPPromptArgument({self.name!r}{req})"
117
+
118
+
119
+ @dataclass(slots=True)
120
+ class MCPPrompt:
121
+ """A prompt template exposed by an MCP server.
122
+
123
+ MCP servers can bundle reusable prompt templates alongside their tools.
124
+ These appear in VS Code as slash commands (e.g. ``/mcp.servername.promptname``).
125
+ Use :meth:`MCPServerProcess.list_prompts` to discover them and
126
+ :meth:`MCPServerProcess.get_prompt` to render one with arguments.
127
+
128
+ Example::
129
+
130
+ server = MCPServerProcess(mcp_config)
131
+ await server.start()
132
+ prompts = await server.list_prompts()
133
+ messages = await server.get_prompt("code_review", {"code": "..."})
134
+ result = await copilot_run(agent, messages[0]["content"])
135
+ """
136
+
137
+ name: str
138
+ description: str = ""
139
+ arguments: list[MCPPromptArgument] = field(default_factory=list)
140
+
141
+ def __repr__(self) -> str:
142
+ args = f", {len(self.arguments)} args" if self.arguments else ""
143
+ return f"MCPPrompt({self.name!r}{args})"
144
+
145
+
146
+ @dataclass(slots=True)
147
+ class CustomAgentInfo:
148
+ """Metadata about a custom agent (.agent.md file) under evaluation."""
149
+
150
+ name: str
151
+ description: str = ""
152
+ file_path: str = ""
153
+
154
+ def __repr__(self) -> str:
155
+ return f"CustomAgentInfo({self.name!r})"
156
+
157
+
158
+ @dataclass(slots=True)
159
+ class InstructionFileInfo:
160
+ """Metadata about a custom instruction file under evaluation."""
161
+
162
+ name: str
163
+ file_path: str = ""
164
+ apply_to: str = "" # glob pattern from frontmatter (e.g. "**/*.py")
165
+ description: str = ""
166
+
167
+ def __repr__(self) -> str:
168
+ return f"InstructionFileInfo({self.name!r})"
169
+
170
+
171
+ @dataclass(slots=True)
172
+ class SkillInfo:
173
+ """Recorded metadata about a skill.
174
+
175
+ Captures the skill's system prompt content and references for inspection.
176
+ """
177
+
178
+ name: str
179
+ description: str
180
+ instruction_content: str
181
+ reference_names: list[str] = field(default_factory=list)
182
+
183
+ def __repr__(self) -> str:
184
+ refs = f", {len(self.reference_names)} refs" if self.reference_names else ""
185
+ return f"SkillInfo({self.name}{refs})"
186
+
187
+
188
+ @dataclass(slots=True, frozen=True)
189
+ class DiscoveredSkill:
190
+ """Skill metadata actually returned by the public SDK, not observed use."""
191
+
192
+ name: str
193
+ description: str
194
+ source: str
195
+ enabled: bool
196
+ user_invocable: bool
197
+ path: str | None = None
198
+ plugin_name: str | None = None
199
+ command_name: str | None = None
200
+ argument_hint: str | None = None
201
+
202
+
203
+ @dataclass(slots=True)
204
+ class SkillDiscovery:
205
+ """Pre-execution discovery evidence, including incomplete or failed checks."""
206
+
207
+ skills: list[DiscoveredSkill] = field(default_factory=list)
208
+ warnings: list[str] = field(default_factory=list)
209
+ errors: list[str] = field(default_factory=list)
210
+ complete: bool = False
211
+
212
+
213
+ @dataclass(slots=True)
214
+ class ClarificationStats:
215
+ """Statistics about clarification requests detected during execution.
216
+
217
+ Tracks when the agent asks for user input instead of executing the task.
218
+ Only populated when clarification_detection is enabled on the agent.
219
+
220
+ Example:
221
+ result = await copilot_run(agent, "Check my balance")
222
+ if result.clarification_stats:
223
+ print(f"Eval asked {result.clarification_stats.count} question(s)")
224
+ """
225
+
226
+ count: int = 0
227
+ turn_indices: list[int] = field(default_factory=list)
228
+ examples: list[str] = field(default_factory=list) # Max 3 examples stored
229
+
230
+ def __repr__(self) -> str:
231
+ return f"ClarificationStats(count={self.count}, turns={self.turn_indices})"
232
+
233
+
234
+ @dataclass(slots=True)
235
+ class Assertion:
236
+ """A single assertion result from a test."""
237
+
238
+ type: str # "semantic", "condition", etc.
239
+ passed: bool
240
+ message: str
241
+ details: str | None = None
242
+
243
+ def __repr__(self) -> str:
244
+ status = "✓" if self.passed else "✗"
245
+ return f"Assertion({status} {self.message})"
246
+
247
+
248
+ @dataclass(slots=True)
249
+ class SubagentInvocation:
250
+ """A subagent invocation observed during agent execution.
251
+
252
+ Tracks when an orchestrator agent dispatches work to a named sub-agent,
253
+ along with the final status and duration of that invocation.
254
+
255
+ Example:
256
+ result = await copilot_run(agent, "Build and test the project")
257
+ assert any(s.name == "coder" for s in result.subagent_invocations)
258
+ assert all(s.status == "completed" for s in result.subagent_invocations)
259
+ """
260
+
261
+ invocation_id: str
262
+ name: str
263
+ status: str # "selected", "started", "completed", "failed"
264
+ duration_ms: float | None = None
265
+ result: EvalResult | None = None
266
+
267
+ def __repr__(self) -> str:
268
+ return f"SubagentInvocation({self.name}, {self.status})"
269
+
270
+
271
+ @dataclass(slots=True)
272
+ class Turn:
273
+ """A single conversational turn."""
274
+
275
+ role: str # "user", "assistant", "tool"
276
+ content: str
277
+ tool_calls: list[ToolCall] = field(default_factory=list)
278
+
279
+ @property
280
+ def text(self) -> str:
281
+ """Get the text content of this turn."""
282
+ return self.content
283
+
284
+ def __repr__(self) -> str:
285
+ preview = self.content[:50] + "..." if len(self.content) > 50 else self.content
286
+ return f"Turn({self.role}: {preview!r})"
287
+
288
+
289
+ @dataclass(slots=True)
290
+ class EvalResult:
291
+ """Result of running an agent with rich inspection capabilities.
292
+
293
+ Example:
294
+ result = await copilot_run(agent, "Hello!")
295
+ assert result.success
296
+ assert "hello" in result.final_response.lower()
297
+ assert result.tool_was_called("read_file")
298
+
299
+ """
300
+
301
+ turns: list[Turn]
302
+ success: bool
303
+ error: str | None = None
304
+ duration_ms: float = 0.0
305
+ token_usage: dict[str, int] = field(default_factory=dict)
306
+ cost_usd: float = 0.0
307
+ _messages: list[Any] = field(default_factory=list)
308
+ session_context_count: int = 0 # Number of prior messages passed in
309
+ assertions: list[Assertion] = field(default_factory=list) # Assertion results
310
+
311
+ # Recorded interface metadata.
312
+ available_tools: list[ToolInfo] = field(default_factory=list)
313
+ skill_info: SkillInfo | None = None
314
+ skill_discovery: SkillDiscovery | None = None
315
+ effective_system_prompt: str | None = None
316
+ mcp_prompts: list[MCPPrompt] = field(default_factory=list)
317
+ prompt_name: str | None = None
318
+ custom_agent_info: CustomAgentInfo | None = None
319
+ premium_requests: float = 0.0
320
+ instruction_files: list[InstructionFileInfo] = field(default_factory=list)
321
+ configuration: dict[str, Any] | None = None
322
+ capture_errors: list[str] = field(default_factory=list)
323
+ evidence_complete: bool | None = None
324
+ request_audit: list[RequestAudit] = field(default_factory=list)
325
+ stop_reason: StopReason | None = None
326
+ usage: list[UsageInfo] = field(default_factory=list)
327
+ tool_calls_admitted: int = 0
328
+ model_used: str | None = None
329
+ reasoning_traces: list[str] = field(default_factory=list)
330
+ permission_requested: bool = False
331
+ permissions: list[dict[str, Any]] = field(default_factory=list)
332
+ subagent_invocations: list[SubagentInvocation] = field(default_factory=list)
333
+ comparison_role: Literal["baseline", "treatment"] | None = None
334
+
335
+ @property
336
+ def execution_results(self) -> list[EvalResult]:
337
+ """This session and separately captured framework-owned descendants."""
338
+ return [
339
+ self,
340
+ *(
341
+ result
342
+ for invocation in self.subagent_invocations
343
+ if invocation.result is not None
344
+ for result in invocation.result.execution_results
345
+ ),
346
+ ]
347
+
348
+ # Clarification detection
349
+ clarification_stats: ClarificationStats | None = None
350
+
351
+ @property
352
+ def messages(self) -> list[Any]:
353
+ """Get full conversation messages for session continuity.
354
+
355
+ This is a copy of captured messages for inspection. The public
356
+ ``copilot_run`` fixture does not accept a message-history argument or
357
+ reuse conversation state between calls.
358
+ """
359
+ return list(self._messages) # Return copy to prevent mutation
360
+
361
+ @property
362
+ def is_session_continuation(self) -> bool:
363
+ """Check if this result is part of a multi-turn session.
364
+
365
+ Returns True if prior messages were passed via the messages parameter.
366
+ """
367
+ return self.session_context_count > 0
368
+
369
+ @property
370
+ def final_response(self) -> str:
371
+ """Get the last assistant response."""
372
+ for turn in reversed(self.turns):
373
+ if turn.role == "assistant":
374
+ return turn.content
375
+ return ""
376
+
377
+ @property
378
+ def all_responses(self) -> list[str]:
379
+ """Get all assistant responses."""
380
+ return [t.content for t in self.turns if t.role == "assistant"]
381
+
382
+ @property
383
+ def all_tool_calls(self) -> list[ToolCall]:
384
+ """Get all tool calls across all turns."""
385
+ calls = []
386
+ for turn in self.turns:
387
+ calls.extend(turn.tool_calls)
388
+ return calls
389
+
390
+ @property
391
+ def tool_names_called(self) -> set[str]:
392
+ """Get set of all tool names that were called."""
393
+ return {call.name for call in self.all_tool_calls}
394
+
395
+ def tool_was_called(self, name: str) -> bool:
396
+ """Check if a specific tool was called."""
397
+ return name in self.tool_names_called
398
+
399
+ def tool_was_called_from_server(self, server_name: str, tool_name: str) -> bool:
400
+ """Check if a specific tool from a named MCP server was called.
401
+
402
+ Useful for plugin testing where tools may be namespaced by server.
403
+ Checks both the plain tool name and the ``server_tool`` prefixed form.
404
+
405
+ Args:
406
+ server_name: Name of the MCP server (e.g., ``"filesystem"``).
407
+ tool_name: Name of the tool (e.g., ``"read_file"``).
408
+
409
+ Returns:
410
+ True if either ``tool_name`` or ``server_name_tool_name`` was called.
411
+
412
+ Example::
413
+
414
+ assert result.tool_was_called_from_server("banking", "get_balance")
415
+ """
416
+ full_name = f"{server_name}_{tool_name}"
417
+ return self.tool_was_called(tool_name) or self.tool_was_called(full_name)
418
+
419
+ def tool_call_count(self, name: str) -> int:
420
+ """Count how many times a specific tool was called."""
421
+ return len(self.tool_calls_for(name))
422
+
423
+ def tool_calls_for(self, name: str) -> list[ToolCall]:
424
+ """Get all calls to a specific tool."""
425
+ return [c for c in self.all_tool_calls if c.name == name]
426
+
427
+ def tool_call_arg(self, tool_name: str, arg_name: str) -> Any:
428
+ """Get argument value from the first call to a tool.
429
+
430
+ Args:
431
+ tool_name: Name of the tool
432
+ arg_name: Name of the argument
433
+
434
+ Returns:
435
+ Argument value or None if not found
436
+ """
437
+ calls = self.tool_calls_for(tool_name)
438
+ if calls:
439
+ return calls[0].arguments.get(arg_name)
440
+ return None
441
+
442
+ def tool_was_called_with(self, name: str, **expected_args: Any) -> bool:
443
+ """Check if a tool was called with specific argument values.
444
+
445
+ Returns True if at least one call to the named tool has all the
446
+ specified argument key-value pairs.
447
+
448
+ Args:
449
+ name: Tool name to check.
450
+ **expected_args: Expected argument key-value pairs.
451
+
452
+ Example::
453
+
454
+ assert result.tool_was_called_with("size_vm", region="westeurope", cores=8)
455
+ assert result.tool_was_called_with("get_balance", account="checking")
456
+ """
457
+ for call in self.tool_calls_for(name):
458
+ if all(call.arguments.get(k) == v for k, v in expected_args.items()):
459
+ return True
460
+ return False
461
+
462
+ def tool_images_for(self, name: str) -> list[ImageContent]:
463
+ """Get all images returned by a specific tool.
464
+
465
+ Args:
466
+ name: Name of the tool (e.g., "screenshot")
467
+
468
+ Returns:
469
+ List of ImageContent objects from tool calls that returned images.
470
+
471
+ Example:
472
+ screenshots = result.tool_images_for("screenshot")
473
+ assert len(screenshots) > 0
474
+ assert screenshots[-1].media_type == "image/png"
475
+ """
476
+ images: list[ImageContent] = []
477
+ for call in self.tool_calls_for(name):
478
+ if call.image_content is not None:
479
+ images.append(
480
+ ImageContent(call.image_content, call.image_media_type or "image/png")
481
+ )
482
+ images.extend(call.additional_images)
483
+ return images
484
+
485
+ @property
486
+ def asked_for_clarification(self) -> bool:
487
+ """Check if the agent asked for clarification instead of acting.
488
+
489
+ Returns True if clarification detection was enabled AND the agent
490
+ asked at least one clarifying question.
491
+
492
+ Example:
493
+ result = await copilot_run(agent, "Check my balance")
494
+ assert not result.asked_for_clarification
495
+ """
496
+ return self.clarification_stats is not None and self.clarification_stats.count > 0
497
+
498
+ @property
499
+ def clarification_count(self) -> int:
500
+ """Number of times the agent asked for clarification."""
501
+ if self.clarification_stats is None:
502
+ return 0
503
+ return self.clarification_stats.count
504
+
505
+ @property
506
+ def tool_context(self) -> str:
507
+ """Summarise tool calls and their results as plain text.
508
+
509
+ This is a readable evidence summary for investigation. It is not an
510
+ independent correctness check and does not replace the full trace.
511
+ """
512
+ calls = self.all_tool_calls
513
+ if not calls:
514
+ return "No tools were called."
515
+ lines: list[str] = []
516
+ for i, call in enumerate(calls, 1):
517
+ lines.append(f"## Tool call {i}: {call.name}")
518
+ if call.arguments:
519
+ args = ", ".join(f"{k}={v!r}" for k, v in call.arguments.items())
520
+ lines.append(f"Arguments: {args}")
521
+ if call.error:
522
+ lines.append(f"Error: {call.error}")
523
+ elif call.result:
524
+ lines.append(f"Result: {call.result}")
525
+ lines.append("")
526
+ return "\n".join(lines)
527
+
528
+ def __repr__(self) -> str:
529
+ status = "SUCCESS" if self.success else f"FAILED: {self.error}"
530
+ tools = ", ".join(sorted(self.tool_names_called)) or "none"
531
+ cost_str = f"${self.cost_usd:.6f}" if self.cost_usd > 0 else "N/A"
532
+ tokens = self.token_usage.get("prompt", 0) + self.token_usage.get("completion", 0)
533
+ return (
534
+ f"EvalResult({status})\n"
535
+ f" Turns: {len(self.turns)}\n"
536
+ f" Tools called: {tools}\n"
537
+ f" Duration: {self.duration_ms:.0f}ms\n"
538
+ f" Tokens: {tokens} | Cost: {cost_str}\n"
539
+ f" Final: {self.final_response[:100]!r}..."
540
+ )
541
+
542
+ def __bool__(self) -> bool:
543
+ return self.success
@@ -0,0 +1,128 @@
1
+ """Lossless serialization and strict loading of the current native report schema."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import binascii
7
+ import sys
8
+ from dataclasses import fields, is_dataclass
9
+ from types import UnionType
10
+ from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union, get_args, get_origin, get_type_hints
11
+
12
+ if TYPE_CHECKING:
13
+ from pytest_copilot.reporting.collector import SuiteReport
14
+
15
+ T = TypeVar("T")
16
+
17
+
18
+ def serialize_dataclass(obj: Any) -> Any:
19
+ """Convert public dataclass fields recursively, encoding bytes as base64."""
20
+ if is_dataclass(obj) and not isinstance(obj, type):
21
+ return {
22
+ field.name: serialize_dataclass(getattr(obj, field.name))
23
+ for field in fields(obj)
24
+ if not field.name.startswith("_")
25
+ }
26
+ if isinstance(obj, (list, tuple)):
27
+ return [serialize_dataclass(item) for item in obj]
28
+ if isinstance(obj, dict):
29
+ return {key: serialize_dataclass(value) for key, value in obj.items()}
30
+ if isinstance(obj, bytes):
31
+ return base64.b64encode(obj).decode("ascii")
32
+ return obj
33
+
34
+
35
+ def _decode_dataclass(cls: type[T], data: Any, *, path: str) -> T:
36
+ from pytest_copilot.copilot.requests import RequestAudit
37
+ from pytest_copilot.core.result import EvalResult
38
+
39
+ if not is_dataclass(cls):
40
+ raise TypeError(f"{cls} is not a native report dataclass")
41
+ context = f"{cls.__name__} at {path}"
42
+ if not isinstance(data, dict):
43
+ raise ValueError(f"{context} must be an object")
44
+ public_fields = [field for field in fields(cls) if not field.name.startswith("_")]
45
+ unexpected = data.keys() - {field.name for field in public_fields}
46
+ if unexpected:
47
+ raise ValueError(f"{context} has unexpected fields: {sorted(unexpected)}")
48
+ annotations = get_type_hints(
49
+ cls,
50
+ globalns={
51
+ **vars(sys.modules[cls.__module__]),
52
+ "RequestAudit": RequestAudit,
53
+ "EvalResult": EvalResult,
54
+ },
55
+ )
56
+ values: dict[str, Any] = {}
57
+ for field in public_fields:
58
+ if field.name not in data:
59
+ raise ValueError(f"{context} is missing required field {field.name!r}")
60
+ values[field.name] = _decode_value(
61
+ annotations[field.name], data[field.name], path=f"{path}.{field.name}"
62
+ )
63
+ return cls(**values)
64
+
65
+
66
+ def _decode_value(annotation: Any, value: Any, *, path: str) -> Any:
67
+ origin = get_origin(annotation)
68
+ arguments = get_args(annotation)
69
+ if annotation is Any:
70
+ return value
71
+ if origin in (Union, UnionType):
72
+ if value is None and type(None) in arguments:
73
+ return None
74
+ alternatives = [item for item in arguments if item is not type(None)]
75
+ if len(alternatives) != 1:
76
+ raise TypeError(f"Unsupported native report field type at {path}: {annotation}")
77
+ return _decode_value(alternatives[0], value, path=path)
78
+ if origin is Literal:
79
+ if value not in arguments:
80
+ raise ValueError(f"{path} must be one of {arguments!r}")
81
+ return value
82
+ if origin in (list, tuple):
83
+ if not isinstance(value, list):
84
+ raise ValueError(f"{path} must be a list")
85
+ if origin is list:
86
+ return [
87
+ _decode_value(arguments[0], item, path=f"{path}[{index}]")
88
+ for index, item in enumerate(value)
89
+ ]
90
+ if len(value) != len(arguments):
91
+ raise ValueError(f"{path} must contain {len(arguments)} values")
92
+ return tuple(
93
+ _decode_value(kind, item, path=f"{path}[{index}]")
94
+ for index, (kind, item) in enumerate(zip(arguments, value, strict=True))
95
+ )
96
+ if origin is dict:
97
+ if not isinstance(value, dict):
98
+ raise ValueError(f"{path} must be an object")
99
+ return {
100
+ _decode_value(arguments[0], key, path=f"{path} key"): _decode_value(
101
+ arguments[1], item, path=f"{path}[{key!r}]"
102
+ )
103
+ for key, item in value.items()
104
+ }
105
+ if isinstance(annotation, type) and is_dataclass(annotation):
106
+ return _decode_dataclass(annotation, value, path=path)
107
+ if annotation is bytes:
108
+ if not isinstance(value, str):
109
+ raise ValueError(f"{path} must be a base64 string")
110
+ try:
111
+ return base64.b64decode(value, validate=True)
112
+ except (ValueError, binascii.Error) as exc:
113
+ raise ValueError(f"{path} contains invalid base64 image data") from exc
114
+ if annotation is float:
115
+ if type(value) not in (int, float):
116
+ raise ValueError(f"{path} must be a number")
117
+ return float(value)
118
+ if annotation in (str, int, bool) and type(value) is annotation:
119
+ return value
120
+ raise ValueError(f"{path} must be {annotation.__name__}")
121
+
122
+
123
+ def deserialize_suite_report(data: dict[str, Any]) -> SuiteReport:
124
+ """Load every current producer field; missing evidence is never inferred."""
125
+ from pytest_copilot.reporting.collector import SuiteReport
126
+
127
+ payload = {key: value for key, value in data.items() if key != "schema_version"}
128
+ return _decode_dataclass(SuiteReport, payload, path="suite")