pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""Privacy-preserving evidence from the SDK's outbound model requests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
from contextlib import suppress
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any, Literal
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
from copilot import (
|
|
14
|
+
CopilotRequestContext,
|
|
15
|
+
CopilotRequestHandler,
|
|
16
|
+
CopilotWebSocketCloseStatus,
|
|
17
|
+
CopilotWebSocketForwarder,
|
|
18
|
+
CopilotWebSocketHandler,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
ImageDetail = Literal["auto", "low", "high"]
|
|
22
|
+
TransportPhase = Literal["open", "send", "receive"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(slots=True, frozen=True)
|
|
26
|
+
class RequestAudit:
|
|
27
|
+
"""Observed wire values, never inferred from the eval configuration."""
|
|
28
|
+
|
|
29
|
+
request_id: str
|
|
30
|
+
model: str
|
|
31
|
+
tool_names: list[str]
|
|
32
|
+
reasoning_effort: str | None
|
|
33
|
+
image_details: list[str | None]
|
|
34
|
+
image_count: int
|
|
35
|
+
instructions_sha256: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _transport_failure_message(phase: TransportPhase, error: BaseException) -> str:
|
|
39
|
+
"""Describe transport failures without retaining provider messages or request data."""
|
|
40
|
+
details: list[str] = []
|
|
41
|
+
if isinstance(error, RuntimeError):
|
|
42
|
+
category = {
|
|
43
|
+
"Copilot request response used after RPC connection closed.": "sdk_rpc_closed",
|
|
44
|
+
"Copilot request was cancelled by the runtime.": "runtime_cancelled",
|
|
45
|
+
"Copilot request response write() called after end()/error().": "response_finished",
|
|
46
|
+
}.get(str(error), "runtime_error")
|
|
47
|
+
details.append(f"category={category}")
|
|
48
|
+
received = getattr(error, "rcvd", None)
|
|
49
|
+
close_code = getattr(received, "code", None)
|
|
50
|
+
if isinstance(close_code, int):
|
|
51
|
+
details.append(f"close_code={close_code}")
|
|
52
|
+
if error.__cause__ is not None:
|
|
53
|
+
details.append(f"cause={type(error.__cause__).__name__}")
|
|
54
|
+
suffix = f" ({', '.join(details)})" if details else ""
|
|
55
|
+
return f"WebSocket transport {phase} failed: {type(error).__name__}{suffix}"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _instruction_text(content: Any) -> str:
|
|
59
|
+
if isinstance(content, str):
|
|
60
|
+
return content
|
|
61
|
+
if isinstance(content, list):
|
|
62
|
+
if all(
|
|
63
|
+
isinstance(part, dict)
|
|
64
|
+
and part.get("type") in ("text", "input_text")
|
|
65
|
+
and isinstance(part.get("text"), str)
|
|
66
|
+
for part in content
|
|
67
|
+
):
|
|
68
|
+
return "\n\n".join(part["text"] for part in content)
|
|
69
|
+
raise ValueError("Unsupported system/developer instruction content")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _audit_payload(payload: Any, request_id: str, detail: ImageDetail | None) -> RequestAudit:
|
|
73
|
+
if (
|
|
74
|
+
not isinstance(payload, dict)
|
|
75
|
+
or not isinstance(payload.get("model"), str)
|
|
76
|
+
or not payload["model"]
|
|
77
|
+
):
|
|
78
|
+
raise ValueError("Unsupported model request: missing model")
|
|
79
|
+
if "system" in payload or ("messages" in payload and "input" in payload):
|
|
80
|
+
raise ValueError("Unsupported model request schema")
|
|
81
|
+
instructions: list[str] = []
|
|
82
|
+
if "messages" in payload:
|
|
83
|
+
messages = payload["messages"]
|
|
84
|
+
effort = payload.get("reasoning_effort")
|
|
85
|
+
elif "input" in payload:
|
|
86
|
+
messages = payload["input"]
|
|
87
|
+
if isinstance(messages, str):
|
|
88
|
+
messages = []
|
|
89
|
+
reasoning = payload.get("reasoning")
|
|
90
|
+
if reasoning is not None and not isinstance(reasoning, dict):
|
|
91
|
+
raise ValueError("Unsupported reasoning configuration")
|
|
92
|
+
effort = reasoning.get("effort") if reasoning else None
|
|
93
|
+
if payload.get("instructions") is not None:
|
|
94
|
+
instructions.append(_instruction_text(payload["instructions"]))
|
|
95
|
+
else:
|
|
96
|
+
raise ValueError("Unsupported model request: expected messages or input")
|
|
97
|
+
if effort is not None and not isinstance(effort, str):
|
|
98
|
+
raise ValueError("Unsupported reasoning effort")
|
|
99
|
+
if not isinstance(messages, list) or not all(isinstance(m, dict) for m in messages):
|
|
100
|
+
raise ValueError("Unsupported model request messages")
|
|
101
|
+
for message in messages:
|
|
102
|
+
if message.get("role") in ("system", "developer"):
|
|
103
|
+
instructions.append(_instruction_text(message.get("content")))
|
|
104
|
+
|
|
105
|
+
tools = payload.get("tools", [])
|
|
106
|
+
if not isinstance(tools, list):
|
|
107
|
+
raise ValueError("Unsupported advertised tool list")
|
|
108
|
+
names: list[str] = []
|
|
109
|
+
for tool in tools:
|
|
110
|
+
if not isinstance(tool, dict) or tool.get("type") != "function":
|
|
111
|
+
raise ValueError("Unsupported advertised tool schema")
|
|
112
|
+
function = tool.get("function", tool)
|
|
113
|
+
if (
|
|
114
|
+
not isinstance(function, dict)
|
|
115
|
+
or not isinstance(function.get("name"), str)
|
|
116
|
+
or not function["name"]
|
|
117
|
+
):
|
|
118
|
+
raise ValueError("Unsupported advertised tool name")
|
|
119
|
+
names.append(function["name"])
|
|
120
|
+
|
|
121
|
+
images: list[str | None] = []
|
|
122
|
+
|
|
123
|
+
def visit(value: Any) -> None:
|
|
124
|
+
if isinstance(value, list):
|
|
125
|
+
for part in value:
|
|
126
|
+
visit(part)
|
|
127
|
+
elif isinstance(value, dict):
|
|
128
|
+
kind = value.get("type")
|
|
129
|
+
if kind in ("input_image", "image_url"):
|
|
130
|
+
target = value if kind == "input_image" else value.get("image_url")
|
|
131
|
+
if not isinstance(target, dict):
|
|
132
|
+
raise ValueError("Unsupported image payload")
|
|
133
|
+
if kind == "image_url" and not isinstance(target.get("url"), str):
|
|
134
|
+
raise ValueError("Unsupported image URL")
|
|
135
|
+
if kind == "input_image" and not (
|
|
136
|
+
isinstance(value.get("image_url"), str) or isinstance(value.get("file_id"), str)
|
|
137
|
+
):
|
|
138
|
+
raise ValueError("Unsupported input image source")
|
|
139
|
+
if detail is not None:
|
|
140
|
+
target["detail"] = detail
|
|
141
|
+
actual = target.get("detail")
|
|
142
|
+
if actual not in (None, "auto", "low", "high"):
|
|
143
|
+
raise ValueError("Unsupported image detail")
|
|
144
|
+
images.append(actual)
|
|
145
|
+
return
|
|
146
|
+
if kind == "image":
|
|
147
|
+
raise ValueError("Unsupported image schema: expected OpenAI image content")
|
|
148
|
+
for part in value.values():
|
|
149
|
+
visit(part)
|
|
150
|
+
|
|
151
|
+
visit(messages)
|
|
152
|
+
return RequestAudit(
|
|
153
|
+
request_id=request_id,
|
|
154
|
+
model=payload["model"],
|
|
155
|
+
tool_names=names,
|
|
156
|
+
reasoning_effort=effort,
|
|
157
|
+
image_details=images,
|
|
158
|
+
image_count=len(images),
|
|
159
|
+
instructions_sha256=hashlib.sha256("\n\n".join(instructions).encode("utf-8")).hexdigest(),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class _AuditedWebSocket(CopilotWebSocketForwarder):
|
|
164
|
+
"""Audit each Responses message before the SDK forwards it."""
|
|
165
|
+
|
|
166
|
+
def __init__(self, context: CopilotRequestContext, owner: RequestAuditHandler) -> None:
|
|
167
|
+
super().__init__(context)
|
|
168
|
+
self.owner = owner
|
|
169
|
+
|
|
170
|
+
async def open(self) -> None:
|
|
171
|
+
if self.owner.failed.is_set():
|
|
172
|
+
raise ValueError("Request audit already failed")
|
|
173
|
+
try:
|
|
174
|
+
await super().open()
|
|
175
|
+
except Exception as exc:
|
|
176
|
+
if not self.owner.inspect_requests:
|
|
177
|
+
raise
|
|
178
|
+
message = _transport_failure_message("open", exc)
|
|
179
|
+
self.owner.fail(message)
|
|
180
|
+
raise ValueError(message) from None
|
|
181
|
+
|
|
182
|
+
async def send_request_message(self, data: str | bytes) -> None:
|
|
183
|
+
if self.owner.failed.is_set():
|
|
184
|
+
raise ValueError("Request audit already failed")
|
|
185
|
+
if self.owner.forwarding_stopped.is_set():
|
|
186
|
+
await self.context.cancel_event.wait()
|
|
187
|
+
raise asyncio.CancelledError
|
|
188
|
+
if not self.owner.inspect_requests:
|
|
189
|
+
await super().send_request_message(data)
|
|
190
|
+
return
|
|
191
|
+
try:
|
|
192
|
+
if not isinstance(data, str):
|
|
193
|
+
raise ValueError("Unsupported WebSocket frame: expected JSON text")
|
|
194
|
+
payload = json.loads(data)
|
|
195
|
+
if not isinstance(payload, dict) or payload.get("type") != "response.create":
|
|
196
|
+
raise ValueError("Unsupported WebSocket frame: expected response.create")
|
|
197
|
+
if "input" not in payload or "messages" in payload:
|
|
198
|
+
raise ValueError("Unsupported WebSocket Responses request")
|
|
199
|
+
if payload.get("previous_response_id") is not None and not all(
|
|
200
|
+
field in payload for field in ("model", "instructions", "tools")
|
|
201
|
+
):
|
|
202
|
+
raise ValueError("Unsupported continuation: request settings must be explicit")
|
|
203
|
+
record = _audit_payload(payload, self.context.request_id, self.owner.image_detail)
|
|
204
|
+
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
205
|
+
except (ValueError, TypeError, UnicodeError) as exc:
|
|
206
|
+
message = (
|
|
207
|
+
"Unsupported WebSocket request: invalid JSON"
|
|
208
|
+
if isinstance(exc, json.JSONDecodeError)
|
|
209
|
+
else str(exc)
|
|
210
|
+
)
|
|
211
|
+
self.owner.fail(message)
|
|
212
|
+
raise ValueError(message) from None
|
|
213
|
+
if self._upstream is None or self._closed:
|
|
214
|
+
self.owner.fail("WebSocket transport is not open")
|
|
215
|
+
raise ValueError(self.owner.error)
|
|
216
|
+
self.owner.observed_requests += 1
|
|
217
|
+
if self.owner.audit:
|
|
218
|
+
self.owner.records.append(record)
|
|
219
|
+
try:
|
|
220
|
+
await super().send_request_message(content)
|
|
221
|
+
except Exception as exc:
|
|
222
|
+
message = _transport_failure_message("send", exc)
|
|
223
|
+
self.owner.fail(message)
|
|
224
|
+
raise ValueError(message) from None
|
|
225
|
+
|
|
226
|
+
async def close(self, status: CopilotWebSocketCloseStatus | None = None) -> None:
|
|
227
|
+
if not self.owner.inspect_requests:
|
|
228
|
+
await super().close(status)
|
|
229
|
+
return
|
|
230
|
+
if status is not None and status.error is not None:
|
|
231
|
+
message = _transport_failure_message("receive", status.error)
|
|
232
|
+
self.owner.fail(message)
|
|
233
|
+
status = CopilotWebSocketCloseStatus(description=message, error=ValueError(message))
|
|
234
|
+
try:
|
|
235
|
+
if self._upstream is not None:
|
|
236
|
+
await self._upstream.close()
|
|
237
|
+
finally:
|
|
238
|
+
await CopilotWebSocketHandler.close(self, status)
|
|
239
|
+
|
|
240
|
+
async def aclose(self) -> None:
|
|
241
|
+
if not self.owner.inspect_requests:
|
|
242
|
+
await super().aclose()
|
|
243
|
+
return
|
|
244
|
+
try:
|
|
245
|
+
if self._suppress_close_on_dispose:
|
|
246
|
+
if self._upstream is not None:
|
|
247
|
+
await self._upstream.close()
|
|
248
|
+
else:
|
|
249
|
+
await self.close()
|
|
250
|
+
finally:
|
|
251
|
+
if self._receive_task is not None:
|
|
252
|
+
self._receive_task.cancel()
|
|
253
|
+
with suppress(asyncio.CancelledError):
|
|
254
|
+
await self._receive_task
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class RequestAuditHandler(CopilotRequestHandler):
|
|
258
|
+
"""Inspect HTTP requests and WebSocket Responses messages before forwarding."""
|
|
259
|
+
|
|
260
|
+
def __init__(
|
|
261
|
+
self,
|
|
262
|
+
*,
|
|
263
|
+
image_detail: ImageDetail | None,
|
|
264
|
+
audit: bool,
|
|
265
|
+
inspect_requests: bool = True,
|
|
266
|
+
) -> None:
|
|
267
|
+
self.image_detail: ImageDetail | None = image_detail
|
|
268
|
+
self.audit = audit
|
|
269
|
+
self.inspect_requests = inspect_requests
|
|
270
|
+
self.records: list[RequestAudit] = []
|
|
271
|
+
self.observed_requests = 0
|
|
272
|
+
self.failed = asyncio.Event()
|
|
273
|
+
self.forwarding_stopped = asyncio.Event()
|
|
274
|
+
self.error: str | None = None
|
|
275
|
+
self.http: httpx.AsyncClient | None = None
|
|
276
|
+
self.websockets: list[_AuditedWebSocket] = []
|
|
277
|
+
|
|
278
|
+
def fail(self, message: str) -> None:
|
|
279
|
+
self.error = message
|
|
280
|
+
self.failed.set()
|
|
281
|
+
|
|
282
|
+
def stop_forwarding(self) -> None:
|
|
283
|
+
"""Prevent post-budget model requests from reaching the provider."""
|
|
284
|
+
self.forwarding_stopped.set()
|
|
285
|
+
|
|
286
|
+
async def send_request(
|
|
287
|
+
self, request: httpx.Request, ctx: CopilotRequestContext
|
|
288
|
+
) -> httpx.Response:
|
|
289
|
+
if self.failed.is_set():
|
|
290
|
+
raise ValueError("Request audit already failed")
|
|
291
|
+
if self.forwarding_stopped.is_set():
|
|
292
|
+
await ctx.cancel_event.wait()
|
|
293
|
+
raise asyncio.CancelledError
|
|
294
|
+
if not self.inspect_requests:
|
|
295
|
+
if self.http is None:
|
|
296
|
+
self.http = httpx.AsyncClient(timeout=None, follow_redirects=False)
|
|
297
|
+
return await self.http.send(request, stream=True)
|
|
298
|
+
if request.method == "GET" and not request.url.path.rstrip("/").endswith("/models"):
|
|
299
|
+
self.fail("Unsupported request audit: only model-catalog GET requests may bypass audit")
|
|
300
|
+
raise ValueError(self.error)
|
|
301
|
+
if request.method != "GET":
|
|
302
|
+
try:
|
|
303
|
+
if request.headers.get("content-encoding", "identity") != "identity":
|
|
304
|
+
raise ValueError("Unsupported compressed model request")
|
|
305
|
+
payload = json.loads(await request.aread())
|
|
306
|
+
record = _audit_payload(payload, ctx.request_id, self.image_detail)
|
|
307
|
+
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()
|
|
308
|
+
headers = [
|
|
309
|
+
(key, value)
|
|
310
|
+
for key, value in request.headers.multi_items()
|
|
311
|
+
if key.lower() not in ("content-length", "transfer-encoding")
|
|
312
|
+
]
|
|
313
|
+
request = httpx.Request(
|
|
314
|
+
request.method,
|
|
315
|
+
request.url,
|
|
316
|
+
headers=headers,
|
|
317
|
+
content=content,
|
|
318
|
+
extensions=request.extensions,
|
|
319
|
+
)
|
|
320
|
+
except (ValueError, TypeError, UnicodeError) as exc:
|
|
321
|
+
# JSON decoding errors can contain pieces of the raw request.
|
|
322
|
+
message = (
|
|
323
|
+
"Unsupported model request: invalid JSON"
|
|
324
|
+
if isinstance(exc, json.JSONDecodeError)
|
|
325
|
+
else str(exc)
|
|
326
|
+
)
|
|
327
|
+
self.fail(message)
|
|
328
|
+
raise ValueError(message) from None
|
|
329
|
+
self.observed_requests += 1
|
|
330
|
+
if self.audit:
|
|
331
|
+
self.records.append(record)
|
|
332
|
+
if self.http is None:
|
|
333
|
+
self.http = httpx.AsyncClient(timeout=None, follow_redirects=False)
|
|
334
|
+
return await self.http.send(request, stream=True)
|
|
335
|
+
|
|
336
|
+
async def open_websocket(self, ctx: CopilotRequestContext) -> CopilotWebSocketHandler:
|
|
337
|
+
if self.failed.is_set():
|
|
338
|
+
raise ValueError("Request audit already failed")
|
|
339
|
+
socket = _AuditedWebSocket(ctx, self)
|
|
340
|
+
self.websockets.append(socket)
|
|
341
|
+
return socket
|
|
342
|
+
|
|
343
|
+
async def aclose(self) -> None:
|
|
344
|
+
closes = [socket.aclose() for socket in self.websockets]
|
|
345
|
+
if self.http is not None:
|
|
346
|
+
closes.append(self.http.aclose())
|
|
347
|
+
outcomes = await asyncio.gather(*closes, return_exceptions=True)
|
|
348
|
+
errors = [outcome for outcome in outcomes if isinstance(outcome, BaseException)]
|
|
349
|
+
if errors:
|
|
350
|
+
raise BaseExceptionGroup("Request transport cleanup failed", errors)
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""Result types for Copilot agent execution.
|
|
2
|
+
|
|
3
|
+
Shared result types are imported from pytest-copilot core.
|
|
4
|
+
CopilotResult adds SDK-specific execution evidence.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Literal, Protocol, TypeAlias
|
|
12
|
+
|
|
13
|
+
from pytest_copilot.copilot.requests import RequestAudit
|
|
14
|
+
from pytest_copilot.core.result import (
|
|
15
|
+
SkillDiscovery,
|
|
16
|
+
StopReason,
|
|
17
|
+
ToolCall,
|
|
18
|
+
Turn,
|
|
19
|
+
UsageInfo,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"CopilotResult",
|
|
24
|
+
"StopReason",
|
|
25
|
+
"SubagentInvocation",
|
|
26
|
+
"ToolCall",
|
|
27
|
+
"Turn",
|
|
28
|
+
"UsageInfo",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
SubagentStatus: TypeAlias = Literal["selected", "started", "completed", "failed"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CopilotResultAgent(Protocol):
|
|
36
|
+
"""Minimal agent surface stored on :class:`CopilotResult`."""
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def working_directory(self) -> str | None:
|
|
40
|
+
raise NotImplementedError
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(slots=True)
|
|
44
|
+
class SubagentInvocation:
|
|
45
|
+
"""A single custom-agent dispatch observed during a run."""
|
|
46
|
+
|
|
47
|
+
invocation_id: str
|
|
48
|
+
name: str
|
|
49
|
+
status: SubagentStatus
|
|
50
|
+
duration_ms: float | None = None
|
|
51
|
+
result: CopilotResult | None = None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(slots=True)
|
|
55
|
+
class CopilotResult:
|
|
56
|
+
"""Result of running a prompt against GitHub Copilot.
|
|
57
|
+
|
|
58
|
+
Captures the full event stream from the SDK, including tool calls,
|
|
59
|
+
reasoning traces, subagent routing, permissions, and token usage.
|
|
60
|
+
|
|
61
|
+
Example:
|
|
62
|
+
result = await copilot_run(agent, "Create hello.py")
|
|
63
|
+
assert result.success
|
|
64
|
+
assert result.tool_was_called("create_file")
|
|
65
|
+
assert "hello" in result.final_response.lower()
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
turns: list[Turn] = field(default_factory=list)
|
|
69
|
+
success: bool = True
|
|
70
|
+
error: str | None = None
|
|
71
|
+
duration_ms: float = 0.0
|
|
72
|
+
|
|
73
|
+
# Token usage (aggregated across all model turns)
|
|
74
|
+
usage: list[UsageInfo] = field(default_factory=list)
|
|
75
|
+
|
|
76
|
+
# Reasoning traces (from assistant.reasoning events)
|
|
77
|
+
reasoning_traces: list[str] = field(default_factory=list)
|
|
78
|
+
|
|
79
|
+
# Subagent invocations
|
|
80
|
+
subagent_invocations: list[SubagentInvocation] = field(default_factory=list)
|
|
81
|
+
|
|
82
|
+
# Permission requests (True if any were requested)
|
|
83
|
+
permission_requested: bool = False
|
|
84
|
+
permissions: list[dict[str, Any]] = field(default_factory=list)
|
|
85
|
+
|
|
86
|
+
# Model actually used (from session.start or assistant.usage)
|
|
87
|
+
model_used: str | None = None
|
|
88
|
+
|
|
89
|
+
# Premium requests (from session.usage_info — CopilotTask only)
|
|
90
|
+
total_premium_requests: float = 0.0
|
|
91
|
+
|
|
92
|
+
# Raw SDK events for advanced inspection
|
|
93
|
+
raw_events: list[Any] = field(default_factory=list)
|
|
94
|
+
capture_errors: list[str] = field(default_factory=list)
|
|
95
|
+
request_audit: list[RequestAudit] = field(default_factory=list)
|
|
96
|
+
stop_reason: StopReason | None = None
|
|
97
|
+
tool_calls_admitted: int = 0
|
|
98
|
+
skill_discovery: SkillDiscovery | None = None
|
|
99
|
+
configuration: dict[str, Any] | None = None
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def evidence_complete(self) -> bool:
|
|
103
|
+
"""Whether every observed call has a complete, correlated record."""
|
|
104
|
+
return (
|
|
105
|
+
not self.capture_errors
|
|
106
|
+
and all(c.evidence_complete for c in self.all_tool_calls)
|
|
107
|
+
and all(
|
|
108
|
+
invocation.result.evidence_complete
|
|
109
|
+
for invocation in self.subagent_invocations
|
|
110
|
+
if invocation.result is not None
|
|
111
|
+
)
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def execution_results(self) -> list[CopilotResult]:
|
|
116
|
+
"""This session and framework-owned descendants, each included once."""
|
|
117
|
+
return [
|
|
118
|
+
self,
|
|
119
|
+
*(
|
|
120
|
+
result
|
|
121
|
+
for invocation in self.subagent_invocations
|
|
122
|
+
if invocation.result is not None
|
|
123
|
+
for result in invocation.result.execution_results
|
|
124
|
+
),
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def all_usage(self) -> list[UsageInfo]:
|
|
129
|
+
"""Captured usage across this session and its descendants."""
|
|
130
|
+
return [usage for result in self.execution_results for usage in result.usage]
|
|
131
|
+
|
|
132
|
+
# Back-reference to the agent that produced this result.
|
|
133
|
+
# Set automatically by run_copilot() so the plugin hook can
|
|
134
|
+
# stash results for pytest-copilot without requiring the
|
|
135
|
+
# copilot_run fixture.
|
|
136
|
+
agent: CopilotResultAgent | None = field(default=None, repr=False)
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def final_response(self) -> str | None:
|
|
140
|
+
"""Get the last assistant response."""
|
|
141
|
+
for turn in reversed(self.turns):
|
|
142
|
+
if turn.role == "assistant":
|
|
143
|
+
return turn.content
|
|
144
|
+
return None
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def all_responses(self) -> list[str]:
|
|
148
|
+
"""Get all assistant responses."""
|
|
149
|
+
return [t.content for t in self.turns if t.role == "assistant"]
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def all_tool_calls(self) -> list[ToolCall]:
|
|
153
|
+
"""Get all tool calls across all turns."""
|
|
154
|
+
calls = []
|
|
155
|
+
for turn in self.turns:
|
|
156
|
+
calls.extend(turn.tool_calls)
|
|
157
|
+
return calls
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def tool_names_called(self) -> set[str]:
|
|
161
|
+
"""Get set of all tool names that were called."""
|
|
162
|
+
return {call.name for call in self.all_tool_calls}
|
|
163
|
+
|
|
164
|
+
def tool_was_called(self, name: str) -> bool:
|
|
165
|
+
"""Check if a specific tool was called."""
|
|
166
|
+
return name in self.tool_names_called
|
|
167
|
+
|
|
168
|
+
def tool_call_count(self, name: str) -> int:
|
|
169
|
+
"""Count how many times a specific tool was called."""
|
|
170
|
+
return len(self.tool_calls_for(name))
|
|
171
|
+
|
|
172
|
+
def tool_calls_for(self, name: str) -> list[ToolCall]:
|
|
173
|
+
"""Get all calls to a specific tool."""
|
|
174
|
+
return [c for c in self.all_tool_calls if c.name == name]
|
|
175
|
+
|
|
176
|
+
def tool_was_called_with(self, name: str, **expected_args: Any) -> bool:
|
|
177
|
+
"""Check if a tool was called with specific argument values.
|
|
178
|
+
|
|
179
|
+
Returns True if at least one call to the named tool has all the
|
|
180
|
+
specified argument key-value pairs.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
name: Tool name to check.
|
|
184
|
+
**expected_args: Expected argument key-value pairs.
|
|
185
|
+
|
|
186
|
+
Example::
|
|
187
|
+
|
|
188
|
+
assert result.tool_was_called_with("size_vm", region="westeurope", cores=8)
|
|
189
|
+
assert result.tool_was_called_with("get_balance", account="checking")
|
|
190
|
+
"""
|
|
191
|
+
for call in self.tool_calls_for(name):
|
|
192
|
+
if all(call.arguments.get(k) == v for k, v in expected_args.items()):
|
|
193
|
+
return True
|
|
194
|
+
return False
|
|
195
|
+
|
|
196
|
+
@property
|
|
197
|
+
def total_input_tokens(self) -> int | None:
|
|
198
|
+
"""Total input tokens across all model turns."""
|
|
199
|
+
if any(not result.usage for result in self.execution_results):
|
|
200
|
+
return None
|
|
201
|
+
counts = [u.input_tokens for u in self.all_usage]
|
|
202
|
+
return sum(c for c in counts if c is not None) if counts and None not in counts else None
|
|
203
|
+
|
|
204
|
+
@property
|
|
205
|
+
def total_output_tokens(self) -> int | None:
|
|
206
|
+
"""Total output tokens across all model turns."""
|
|
207
|
+
if any(not result.usage for result in self.execution_results):
|
|
208
|
+
return None
|
|
209
|
+
counts = [u.output_tokens for u in self.all_usage]
|
|
210
|
+
return sum(c for c in counts if c is not None) if counts and None not in counts else None
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def total_tokens(self) -> int | None:
|
|
214
|
+
"""Total tokens (input + output) across all model turns."""
|
|
215
|
+
inputs, outputs = self.total_input_tokens, self.total_output_tokens
|
|
216
|
+
return inputs + outputs if inputs is not None and outputs is not None else None
|
|
217
|
+
|
|
218
|
+
@property
|
|
219
|
+
def token_usage(self) -> dict[str, int]:
|
|
220
|
+
"""Token usage dict compatible with pytest-copilot's EvalResult.
|
|
221
|
+
|
|
222
|
+
Keys use short names (``prompt``, ``completion``, ``total``) to match
|
|
223
|
+
the format pytest-copilot reads in its collector and generator.
|
|
224
|
+
"""
|
|
225
|
+
counts = {
|
|
226
|
+
"prompt": self.total_input_tokens,
|
|
227
|
+
"completion": self.total_output_tokens,
|
|
228
|
+
"total": self.total_tokens,
|
|
229
|
+
}
|
|
230
|
+
return {key: value for key, value in counts.items() if value is not None}
|
|
231
|
+
|
|
232
|
+
def __repr__(self) -> str:
|
|
233
|
+
status = "SUCCESS" if self.success else f"FAILED: {self.error}"
|
|
234
|
+
tools = ", ".join(sorted(self.tool_names_called)) or "none"
|
|
235
|
+
pr = self.total_premium_requests
|
|
236
|
+
cost_str = f"{pr:.0f} premium requests" if pr > 0 else "N/A"
|
|
237
|
+
final = self.final_response or ""
|
|
238
|
+
return (
|
|
239
|
+
f"CopilotResult({status})\n"
|
|
240
|
+
f" Turns: {len(self.turns)}\n"
|
|
241
|
+
f" Tools called: {tools}\n"
|
|
242
|
+
f" Duration: {self.duration_ms:.0f}ms\n"
|
|
243
|
+
f" Tokens: {self.total_tokens} | Cost: {cost_str}\n"
|
|
244
|
+
f" Model: {self.model_used or 'default'}\n"
|
|
245
|
+
f" Final: {final[:100]!r}..."
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
def __bool__(self) -> bool:
|
|
249
|
+
return self.success
|
|
250
|
+
|
|
251
|
+
# ── File helpers ──
|
|
252
|
+
|
|
253
|
+
@property
|
|
254
|
+
def working_directory(self) -> Path:
|
|
255
|
+
"""Working directory where the agent operated.
|
|
256
|
+
|
|
257
|
+
Resolved from ``agent.working_directory`` when set; falls back to
|
|
258
|
+
the current working directory.
|
|
259
|
+
"""
|
|
260
|
+
if self.agent and self.agent.working_directory:
|
|
261
|
+
return Path(self.agent.working_directory)
|
|
262
|
+
return Path.cwd()
|
|
263
|
+
|
|
264
|
+
def file(self, path: str) -> str:
|
|
265
|
+
"""Read the content of a file relative to the working directory.
|
|
266
|
+
|
|
267
|
+
Args:
|
|
268
|
+
path: Relative file path (e.g. ``"main.py"`` or ``"src/utils.py"``).
|
|
269
|
+
|
|
270
|
+
Returns:
|
|
271
|
+
File content as a string.
|
|
272
|
+
|
|
273
|
+
Raises:
|
|
274
|
+
FileNotFoundError: If the file does not exist.
|
|
275
|
+
"""
|
|
276
|
+
return (self.working_directory / path).read_text(encoding="utf-8")
|
|
277
|
+
|
|
278
|
+
def file_exists(self, path: str) -> bool:
|
|
279
|
+
"""Check whether a file exists in the working directory.
|
|
280
|
+
|
|
281
|
+
Args:
|
|
282
|
+
path: Relative file path.
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
``True`` if the file exists, ``False`` otherwise.
|
|
286
|
+
"""
|
|
287
|
+
return (self.working_directory / path).exists()
|
|
288
|
+
|
|
289
|
+
def files_matching(self, pattern: str = "**/*") -> list[Path]:
|
|
290
|
+
"""Find files matching a glob pattern in the working directory.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
pattern: Glob pattern relative to the working directory.
|
|
294
|
+
Defaults to ``"**/*"`` (all files recursively).
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
Sorted list of matching ``Path`` objects (files only, no
|
|
298
|
+
directories).
|
|
299
|
+
|
|
300
|
+
Example::
|
|
301
|
+
|
|
302
|
+
# All Python files created by the agent
|
|
303
|
+
py_files = result.files_matching("**/*.py")
|
|
304
|
+
assert py_files, "No Python files were created"
|
|
305
|
+
|
|
306
|
+
# Top-level test files
|
|
307
|
+
test_files = result.files_matching("test_*.py")
|
|
308
|
+
"""
|
|
309
|
+
return sorted(p for p in self.working_directory.glob(pattern) if p.is_file())
|