pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,350 @@
1
+ """Privacy-preserving evidence from the SDK's outbound model requests."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import hashlib
7
+ import json
8
+ from contextlib import suppress
9
+ from dataclasses import dataclass
10
+ from typing import Any, Literal
11
+
12
+ import httpx
13
+ from copilot import (
14
+ CopilotRequestContext,
15
+ CopilotRequestHandler,
16
+ CopilotWebSocketCloseStatus,
17
+ CopilotWebSocketForwarder,
18
+ CopilotWebSocketHandler,
19
+ )
20
+
21
+ ImageDetail = Literal["auto", "low", "high"]
22
+ TransportPhase = Literal["open", "send", "receive"]
23
+
24
+
25
+ @dataclass(slots=True, frozen=True)
26
+ class RequestAudit:
27
+ """Observed wire values, never inferred from the eval configuration."""
28
+
29
+ request_id: str
30
+ model: str
31
+ tool_names: list[str]
32
+ reasoning_effort: str | None
33
+ image_details: list[str | None]
34
+ image_count: int
35
+ instructions_sha256: str
36
+
37
+
38
+ def _transport_failure_message(phase: TransportPhase, error: BaseException) -> str:
39
+ """Describe transport failures without retaining provider messages or request data."""
40
+ details: list[str] = []
41
+ if isinstance(error, RuntimeError):
42
+ category = {
43
+ "Copilot request response used after RPC connection closed.": "sdk_rpc_closed",
44
+ "Copilot request was cancelled by the runtime.": "runtime_cancelled",
45
+ "Copilot request response write() called after end()/error().": "response_finished",
46
+ }.get(str(error), "runtime_error")
47
+ details.append(f"category={category}")
48
+ received = getattr(error, "rcvd", None)
49
+ close_code = getattr(received, "code", None)
50
+ if isinstance(close_code, int):
51
+ details.append(f"close_code={close_code}")
52
+ if error.__cause__ is not None:
53
+ details.append(f"cause={type(error.__cause__).__name__}")
54
+ suffix = f" ({', '.join(details)})" if details else ""
55
+ return f"WebSocket transport {phase} failed: {type(error).__name__}{suffix}"
56
+
57
+
58
+ def _instruction_text(content: Any) -> str:
59
+ if isinstance(content, str):
60
+ return content
61
+ if isinstance(content, list):
62
+ if all(
63
+ isinstance(part, dict)
64
+ and part.get("type") in ("text", "input_text")
65
+ and isinstance(part.get("text"), str)
66
+ for part in content
67
+ ):
68
+ return "\n\n".join(part["text"] for part in content)
69
+ raise ValueError("Unsupported system/developer instruction content")
70
+
71
+
72
+ def _audit_payload(payload: Any, request_id: str, detail: ImageDetail | None) -> RequestAudit:
73
+ if (
74
+ not isinstance(payload, dict)
75
+ or not isinstance(payload.get("model"), str)
76
+ or not payload["model"]
77
+ ):
78
+ raise ValueError("Unsupported model request: missing model")
79
+ if "system" in payload or ("messages" in payload and "input" in payload):
80
+ raise ValueError("Unsupported model request schema")
81
+ instructions: list[str] = []
82
+ if "messages" in payload:
83
+ messages = payload["messages"]
84
+ effort = payload.get("reasoning_effort")
85
+ elif "input" in payload:
86
+ messages = payload["input"]
87
+ if isinstance(messages, str):
88
+ messages = []
89
+ reasoning = payload.get("reasoning")
90
+ if reasoning is not None and not isinstance(reasoning, dict):
91
+ raise ValueError("Unsupported reasoning configuration")
92
+ effort = reasoning.get("effort") if reasoning else None
93
+ if payload.get("instructions") is not None:
94
+ instructions.append(_instruction_text(payload["instructions"]))
95
+ else:
96
+ raise ValueError("Unsupported model request: expected messages or input")
97
+ if effort is not None and not isinstance(effort, str):
98
+ raise ValueError("Unsupported reasoning effort")
99
+ if not isinstance(messages, list) or not all(isinstance(m, dict) for m in messages):
100
+ raise ValueError("Unsupported model request messages")
101
+ for message in messages:
102
+ if message.get("role") in ("system", "developer"):
103
+ instructions.append(_instruction_text(message.get("content")))
104
+
105
+ tools = payload.get("tools", [])
106
+ if not isinstance(tools, list):
107
+ raise ValueError("Unsupported advertised tool list")
108
+ names: list[str] = []
109
+ for tool in tools:
110
+ if not isinstance(tool, dict) or tool.get("type") != "function":
111
+ raise ValueError("Unsupported advertised tool schema")
112
+ function = tool.get("function", tool)
113
+ if (
114
+ not isinstance(function, dict)
115
+ or not isinstance(function.get("name"), str)
116
+ or not function["name"]
117
+ ):
118
+ raise ValueError("Unsupported advertised tool name")
119
+ names.append(function["name"])
120
+
121
+ images: list[str | None] = []
122
+
123
+ def visit(value: Any) -> None:
124
+ if isinstance(value, list):
125
+ for part in value:
126
+ visit(part)
127
+ elif isinstance(value, dict):
128
+ kind = value.get("type")
129
+ if kind in ("input_image", "image_url"):
130
+ target = value if kind == "input_image" else value.get("image_url")
131
+ if not isinstance(target, dict):
132
+ raise ValueError("Unsupported image payload")
133
+ if kind == "image_url" and not isinstance(target.get("url"), str):
134
+ raise ValueError("Unsupported image URL")
135
+ if kind == "input_image" and not (
136
+ isinstance(value.get("image_url"), str) or isinstance(value.get("file_id"), str)
137
+ ):
138
+ raise ValueError("Unsupported input image source")
139
+ if detail is not None:
140
+ target["detail"] = detail
141
+ actual = target.get("detail")
142
+ if actual not in (None, "auto", "low", "high"):
143
+ raise ValueError("Unsupported image detail")
144
+ images.append(actual)
145
+ return
146
+ if kind == "image":
147
+ raise ValueError("Unsupported image schema: expected OpenAI image content")
148
+ for part in value.values():
149
+ visit(part)
150
+
151
+ visit(messages)
152
+ return RequestAudit(
153
+ request_id=request_id,
154
+ model=payload["model"],
155
+ tool_names=names,
156
+ reasoning_effort=effort,
157
+ image_details=images,
158
+ image_count=len(images),
159
+ instructions_sha256=hashlib.sha256("\n\n".join(instructions).encode("utf-8")).hexdigest(),
160
+ )
161
+
162
+
163
+ class _AuditedWebSocket(CopilotWebSocketForwarder):
164
+ """Audit each Responses message before the SDK forwards it."""
165
+
166
+ def __init__(self, context: CopilotRequestContext, owner: RequestAuditHandler) -> None:
167
+ super().__init__(context)
168
+ self.owner = owner
169
+
170
+ async def open(self) -> None:
171
+ if self.owner.failed.is_set():
172
+ raise ValueError("Request audit already failed")
173
+ try:
174
+ await super().open()
175
+ except Exception as exc:
176
+ if not self.owner.inspect_requests:
177
+ raise
178
+ message = _transport_failure_message("open", exc)
179
+ self.owner.fail(message)
180
+ raise ValueError(message) from None
181
+
182
+ async def send_request_message(self, data: str | bytes) -> None:
183
+ if self.owner.failed.is_set():
184
+ raise ValueError("Request audit already failed")
185
+ if self.owner.forwarding_stopped.is_set():
186
+ await self.context.cancel_event.wait()
187
+ raise asyncio.CancelledError
188
+ if not self.owner.inspect_requests:
189
+ await super().send_request_message(data)
190
+ return
191
+ try:
192
+ if not isinstance(data, str):
193
+ raise ValueError("Unsupported WebSocket frame: expected JSON text")
194
+ payload = json.loads(data)
195
+ if not isinstance(payload, dict) or payload.get("type") != "response.create":
196
+ raise ValueError("Unsupported WebSocket frame: expected response.create")
197
+ if "input" not in payload or "messages" in payload:
198
+ raise ValueError("Unsupported WebSocket Responses request")
199
+ if payload.get("previous_response_id") is not None and not all(
200
+ field in payload for field in ("model", "instructions", "tools")
201
+ ):
202
+ raise ValueError("Unsupported continuation: request settings must be explicit")
203
+ record = _audit_payload(payload, self.context.request_id, self.owner.image_detail)
204
+ content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
205
+ except (ValueError, TypeError, UnicodeError) as exc:
206
+ message = (
207
+ "Unsupported WebSocket request: invalid JSON"
208
+ if isinstance(exc, json.JSONDecodeError)
209
+ else str(exc)
210
+ )
211
+ self.owner.fail(message)
212
+ raise ValueError(message) from None
213
+ if self._upstream is None or self._closed:
214
+ self.owner.fail("WebSocket transport is not open")
215
+ raise ValueError(self.owner.error)
216
+ self.owner.observed_requests += 1
217
+ if self.owner.audit:
218
+ self.owner.records.append(record)
219
+ try:
220
+ await super().send_request_message(content)
221
+ except Exception as exc:
222
+ message = _transport_failure_message("send", exc)
223
+ self.owner.fail(message)
224
+ raise ValueError(message) from None
225
+
226
+ async def close(self, status: CopilotWebSocketCloseStatus | None = None) -> None:
227
+ if not self.owner.inspect_requests:
228
+ await super().close(status)
229
+ return
230
+ if status is not None and status.error is not None:
231
+ message = _transport_failure_message("receive", status.error)
232
+ self.owner.fail(message)
233
+ status = CopilotWebSocketCloseStatus(description=message, error=ValueError(message))
234
+ try:
235
+ if self._upstream is not None:
236
+ await self._upstream.close()
237
+ finally:
238
+ await CopilotWebSocketHandler.close(self, status)
239
+
240
+ async def aclose(self) -> None:
241
+ if not self.owner.inspect_requests:
242
+ await super().aclose()
243
+ return
244
+ try:
245
+ if self._suppress_close_on_dispose:
246
+ if self._upstream is not None:
247
+ await self._upstream.close()
248
+ else:
249
+ await self.close()
250
+ finally:
251
+ if self._receive_task is not None:
252
+ self._receive_task.cancel()
253
+ with suppress(asyncio.CancelledError):
254
+ await self._receive_task
255
+
256
+
257
+ class RequestAuditHandler(CopilotRequestHandler):
258
+ """Inspect HTTP requests and WebSocket Responses messages before forwarding."""
259
+
260
+ def __init__(
261
+ self,
262
+ *,
263
+ image_detail: ImageDetail | None,
264
+ audit: bool,
265
+ inspect_requests: bool = True,
266
+ ) -> None:
267
+ self.image_detail: ImageDetail | None = image_detail
268
+ self.audit = audit
269
+ self.inspect_requests = inspect_requests
270
+ self.records: list[RequestAudit] = []
271
+ self.observed_requests = 0
272
+ self.failed = asyncio.Event()
273
+ self.forwarding_stopped = asyncio.Event()
274
+ self.error: str | None = None
275
+ self.http: httpx.AsyncClient | None = None
276
+ self.websockets: list[_AuditedWebSocket] = []
277
+
278
+ def fail(self, message: str) -> None:
279
+ self.error = message
280
+ self.failed.set()
281
+
282
+ def stop_forwarding(self) -> None:
283
+ """Prevent post-budget model requests from reaching the provider."""
284
+ self.forwarding_stopped.set()
285
+
286
+ async def send_request(
287
+ self, request: httpx.Request, ctx: CopilotRequestContext
288
+ ) -> httpx.Response:
289
+ if self.failed.is_set():
290
+ raise ValueError("Request audit already failed")
291
+ if self.forwarding_stopped.is_set():
292
+ await ctx.cancel_event.wait()
293
+ raise asyncio.CancelledError
294
+ if not self.inspect_requests:
295
+ if self.http is None:
296
+ self.http = httpx.AsyncClient(timeout=None, follow_redirects=False)
297
+ return await self.http.send(request, stream=True)
298
+ if request.method == "GET" and not request.url.path.rstrip("/").endswith("/models"):
299
+ self.fail("Unsupported request audit: only model-catalog GET requests may bypass audit")
300
+ raise ValueError(self.error)
301
+ if request.method != "GET":
302
+ try:
303
+ if request.headers.get("content-encoding", "identity") != "identity":
304
+ raise ValueError("Unsupported compressed model request")
305
+ payload = json.loads(await request.aread())
306
+ record = _audit_payload(payload, ctx.request_id, self.image_detail)
307
+ content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()
308
+ headers = [
309
+ (key, value)
310
+ for key, value in request.headers.multi_items()
311
+ if key.lower() not in ("content-length", "transfer-encoding")
312
+ ]
313
+ request = httpx.Request(
314
+ request.method,
315
+ request.url,
316
+ headers=headers,
317
+ content=content,
318
+ extensions=request.extensions,
319
+ )
320
+ except (ValueError, TypeError, UnicodeError) as exc:
321
+ # JSON decoding errors can contain pieces of the raw request.
322
+ message = (
323
+ "Unsupported model request: invalid JSON"
324
+ if isinstance(exc, json.JSONDecodeError)
325
+ else str(exc)
326
+ )
327
+ self.fail(message)
328
+ raise ValueError(message) from None
329
+ self.observed_requests += 1
330
+ if self.audit:
331
+ self.records.append(record)
332
+ if self.http is None:
333
+ self.http = httpx.AsyncClient(timeout=None, follow_redirects=False)
334
+ return await self.http.send(request, stream=True)
335
+
336
+ async def open_websocket(self, ctx: CopilotRequestContext) -> CopilotWebSocketHandler:
337
+ if self.failed.is_set():
338
+ raise ValueError("Request audit already failed")
339
+ socket = _AuditedWebSocket(ctx, self)
340
+ self.websockets.append(socket)
341
+ return socket
342
+
343
+ async def aclose(self) -> None:
344
+ closes = [socket.aclose() for socket in self.websockets]
345
+ if self.http is not None:
346
+ closes.append(self.http.aclose())
347
+ outcomes = await asyncio.gather(*closes, return_exceptions=True)
348
+ errors = [outcome for outcome in outcomes if isinstance(outcome, BaseException)]
349
+ if errors:
350
+ raise BaseExceptionGroup("Request transport cleanup failed", errors)
@@ -0,0 +1,309 @@
1
+ """Result types for Copilot agent execution.
2
+
3
+ Shared result types are imported from pytest-copilot core.
4
+ CopilotResult adds SDK-specific execution evidence.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass, field
10
+ from pathlib import Path
11
+ from typing import Any, Literal, Protocol, TypeAlias
12
+
13
+ from pytest_copilot.copilot.requests import RequestAudit
14
+ from pytest_copilot.core.result import (
15
+ SkillDiscovery,
16
+ StopReason,
17
+ ToolCall,
18
+ Turn,
19
+ UsageInfo,
20
+ )
21
+
22
+ __all__ = [
23
+ "CopilotResult",
24
+ "StopReason",
25
+ "SubagentInvocation",
26
+ "ToolCall",
27
+ "Turn",
28
+ "UsageInfo",
29
+ ]
30
+
31
+
32
+ SubagentStatus: TypeAlias = Literal["selected", "started", "completed", "failed"]
33
+
34
+
35
+ class CopilotResultAgent(Protocol):
36
+ """Minimal agent surface stored on :class:`CopilotResult`."""
37
+
38
+ @property
39
+ def working_directory(self) -> str | None:
40
+ raise NotImplementedError
41
+
42
+
43
+ @dataclass(slots=True)
44
+ class SubagentInvocation:
45
+ """A single custom-agent dispatch observed during a run."""
46
+
47
+ invocation_id: str
48
+ name: str
49
+ status: SubagentStatus
50
+ duration_ms: float | None = None
51
+ result: CopilotResult | None = None
52
+
53
+
54
+ @dataclass(slots=True)
55
+ class CopilotResult:
56
+ """Result of running a prompt against GitHub Copilot.
57
+
58
+ Captures the full event stream from the SDK, including tool calls,
59
+ reasoning traces, subagent routing, permissions, and token usage.
60
+
61
+ Example:
62
+ result = await copilot_run(agent, "Create hello.py")
63
+ assert result.success
64
+ assert result.tool_was_called("create_file")
65
+ assert "hello" in result.final_response.lower()
66
+ """
67
+
68
+ turns: list[Turn] = field(default_factory=list)
69
+ success: bool = True
70
+ error: str | None = None
71
+ duration_ms: float = 0.0
72
+
73
+ # Token usage (aggregated across all model turns)
74
+ usage: list[UsageInfo] = field(default_factory=list)
75
+
76
+ # Reasoning traces (from assistant.reasoning events)
77
+ reasoning_traces: list[str] = field(default_factory=list)
78
+
79
+ # Subagent invocations
80
+ subagent_invocations: list[SubagentInvocation] = field(default_factory=list)
81
+
82
+ # Permission requests (True if any were requested)
83
+ permission_requested: bool = False
84
+ permissions: list[dict[str, Any]] = field(default_factory=list)
85
+
86
+ # Model actually used (from session.start or assistant.usage)
87
+ model_used: str | None = None
88
+
89
+ # Premium requests (from session.usage_info — CopilotTask only)
90
+ total_premium_requests: float = 0.0
91
+
92
+ # Raw SDK events for advanced inspection
93
+ raw_events: list[Any] = field(default_factory=list)
94
+ capture_errors: list[str] = field(default_factory=list)
95
+ request_audit: list[RequestAudit] = field(default_factory=list)
96
+ stop_reason: StopReason | None = None
97
+ tool_calls_admitted: int = 0
98
+ skill_discovery: SkillDiscovery | None = None
99
+ configuration: dict[str, Any] | None = None
100
+
101
+ @property
102
+ def evidence_complete(self) -> bool:
103
+ """Whether every observed call has a complete, correlated record."""
104
+ return (
105
+ not self.capture_errors
106
+ and all(c.evidence_complete for c in self.all_tool_calls)
107
+ and all(
108
+ invocation.result.evidence_complete
109
+ for invocation in self.subagent_invocations
110
+ if invocation.result is not None
111
+ )
112
+ )
113
+
114
+ @property
115
+ def execution_results(self) -> list[CopilotResult]:
116
+ """This session and framework-owned descendants, each included once."""
117
+ return [
118
+ self,
119
+ *(
120
+ result
121
+ for invocation in self.subagent_invocations
122
+ if invocation.result is not None
123
+ for result in invocation.result.execution_results
124
+ ),
125
+ ]
126
+
127
+ @property
128
+ def all_usage(self) -> list[UsageInfo]:
129
+ """Captured usage across this session and its descendants."""
130
+ return [usage for result in self.execution_results for usage in result.usage]
131
+
132
+ # Back-reference to the agent that produced this result.
133
+ # Set automatically by run_copilot() so the plugin hook can
134
+ # stash results for pytest-copilot without requiring the
135
+ # copilot_run fixture.
136
+ agent: CopilotResultAgent | None = field(default=None, repr=False)
137
+
138
+ @property
139
+ def final_response(self) -> str | None:
140
+ """Get the last assistant response."""
141
+ for turn in reversed(self.turns):
142
+ if turn.role == "assistant":
143
+ return turn.content
144
+ return None
145
+
146
+ @property
147
+ def all_responses(self) -> list[str]:
148
+ """Get all assistant responses."""
149
+ return [t.content for t in self.turns if t.role == "assistant"]
150
+
151
+ @property
152
+ def all_tool_calls(self) -> list[ToolCall]:
153
+ """Get all tool calls across all turns."""
154
+ calls = []
155
+ for turn in self.turns:
156
+ calls.extend(turn.tool_calls)
157
+ return calls
158
+
159
+ @property
160
+ def tool_names_called(self) -> set[str]:
161
+ """Get set of all tool names that were called."""
162
+ return {call.name for call in self.all_tool_calls}
163
+
164
+ def tool_was_called(self, name: str) -> bool:
165
+ """Check if a specific tool was called."""
166
+ return name in self.tool_names_called
167
+
168
+ def tool_call_count(self, name: str) -> int:
169
+ """Count how many times a specific tool was called."""
170
+ return len(self.tool_calls_for(name))
171
+
172
+ def tool_calls_for(self, name: str) -> list[ToolCall]:
173
+ """Get all calls to a specific tool."""
174
+ return [c for c in self.all_tool_calls if c.name == name]
175
+
176
+ def tool_was_called_with(self, name: str, **expected_args: Any) -> bool:
177
+ """Check if a tool was called with specific argument values.
178
+
179
+ Returns True if at least one call to the named tool has all the
180
+ specified argument key-value pairs.
181
+
182
+ Args:
183
+ name: Tool name to check.
184
+ **expected_args: Expected argument key-value pairs.
185
+
186
+ Example::
187
+
188
+ assert result.tool_was_called_with("size_vm", region="westeurope", cores=8)
189
+ assert result.tool_was_called_with("get_balance", account="checking")
190
+ """
191
+ for call in self.tool_calls_for(name):
192
+ if all(call.arguments.get(k) == v for k, v in expected_args.items()):
193
+ return True
194
+ return False
195
+
196
+ @property
197
+ def total_input_tokens(self) -> int | None:
198
+ """Total input tokens across all model turns."""
199
+ if any(not result.usage for result in self.execution_results):
200
+ return None
201
+ counts = [u.input_tokens for u in self.all_usage]
202
+ return sum(c for c in counts if c is not None) if counts and None not in counts else None
203
+
204
+ @property
205
+ def total_output_tokens(self) -> int | None:
206
+ """Total output tokens across all model turns."""
207
+ if any(not result.usage for result in self.execution_results):
208
+ return None
209
+ counts = [u.output_tokens for u in self.all_usage]
210
+ return sum(c for c in counts if c is not None) if counts and None not in counts else None
211
+
212
+ @property
213
+ def total_tokens(self) -> int | None:
214
+ """Total tokens (input + output) across all model turns."""
215
+ inputs, outputs = self.total_input_tokens, self.total_output_tokens
216
+ return inputs + outputs if inputs is not None and outputs is not None else None
217
+
218
+ @property
219
+ def token_usage(self) -> dict[str, int]:
220
+ """Token usage dict compatible with pytest-copilot's EvalResult.
221
+
222
+ Keys use short names (``prompt``, ``completion``, ``total``) to match
223
+ the format pytest-copilot reads in its collector and generator.
224
+ """
225
+ counts = {
226
+ "prompt": self.total_input_tokens,
227
+ "completion": self.total_output_tokens,
228
+ "total": self.total_tokens,
229
+ }
230
+ return {key: value for key, value in counts.items() if value is not None}
231
+
232
+ def __repr__(self) -> str:
233
+ status = "SUCCESS" if self.success else f"FAILED: {self.error}"
234
+ tools = ", ".join(sorted(self.tool_names_called)) or "none"
235
+ pr = self.total_premium_requests
236
+ cost_str = f"{pr:.0f} premium requests" if pr > 0 else "N/A"
237
+ final = self.final_response or ""
238
+ return (
239
+ f"CopilotResult({status})\n"
240
+ f" Turns: {len(self.turns)}\n"
241
+ f" Tools called: {tools}\n"
242
+ f" Duration: {self.duration_ms:.0f}ms\n"
243
+ f" Tokens: {self.total_tokens} | Cost: {cost_str}\n"
244
+ f" Model: {self.model_used or 'default'}\n"
245
+ f" Final: {final[:100]!r}..."
246
+ )
247
+
248
+ def __bool__(self) -> bool:
249
+ return self.success
250
+
251
+ # ── File helpers ──
252
+
253
+ @property
254
+ def working_directory(self) -> Path:
255
+ """Working directory where the agent operated.
256
+
257
+ Resolved from ``agent.working_directory`` when set; falls back to
258
+ the current working directory.
259
+ """
260
+ if self.agent and self.agent.working_directory:
261
+ return Path(self.agent.working_directory)
262
+ return Path.cwd()
263
+
264
+ def file(self, path: str) -> str:
265
+ """Read the content of a file relative to the working directory.
266
+
267
+ Args:
268
+ path: Relative file path (e.g. ``"main.py"`` or ``"src/utils.py"``).
269
+
270
+ Returns:
271
+ File content as a string.
272
+
273
+ Raises:
274
+ FileNotFoundError: If the file does not exist.
275
+ """
276
+ return (self.working_directory / path).read_text(encoding="utf-8")
277
+
278
+ def file_exists(self, path: str) -> bool:
279
+ """Check whether a file exists in the working directory.
280
+
281
+ Args:
282
+ path: Relative file path.
283
+
284
+ Returns:
285
+ ``True`` if the file exists, ``False`` otherwise.
286
+ """
287
+ return (self.working_directory / path).exists()
288
+
289
+ def files_matching(self, pattern: str = "**/*") -> list[Path]:
290
+ """Find files matching a glob pattern in the working directory.
291
+
292
+ Args:
293
+ pattern: Glob pattern relative to the working directory.
294
+ Defaults to ``"**/*"`` (all files recursively).
295
+
296
+ Returns:
297
+ Sorted list of matching ``Path`` objects (files only, no
298
+ directories).
299
+
300
+ Example::
301
+
302
+ # All Python files created by the agent
303
+ py_files = result.files_matching("**/*.py")
304
+ assert py_files, "No Python files were created"
305
+
306
+ # Top-level test files
307
+ test_files = result.files_matching("test_*.py")
308
+ """
309
+ return sorted(p for p in self.working_directory.glob(pattern) if p.is_file())