pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
"""Map Copilot SDK SessionEvents to CopilotResult.
|
|
2
|
+
|
|
3
|
+
Converts the raw event stream from the Copilot SDK into structured
|
|
4
|
+
Turn/ToolCall objects for assertion and reporting.
|
|
5
|
+
|
|
6
|
+
SDK Event Types (38 values) grouped by what they map to:
|
|
7
|
+
|
|
8
|
+
Tool lifecycle:
|
|
9
|
+
tool.execution_start → Start tracking a ToolCall
|
|
10
|
+
tool.execution_complete → Complete the ToolCall with result
|
|
11
|
+
tool.execution_progress → Progress update (logged)
|
|
12
|
+
tool.user_requested → User-initiated tool call
|
|
13
|
+
|
|
14
|
+
Assistant output:
|
|
15
|
+
assistant.message → Assistant Turn with content
|
|
16
|
+
assistant.message_delta → Streaming delta (accumulated)
|
|
17
|
+
assistant.reasoning → Reasoning trace
|
|
18
|
+
assistant.reasoning_delta → Streaming reasoning delta
|
|
19
|
+
assistant.intent → Intent declaration
|
|
20
|
+
assistant.turn_start → Turn boundary marker
|
|
21
|
+
assistant.turn_end → Turn boundary marker
|
|
22
|
+
assistant.usage → Token usage / cost
|
|
23
|
+
|
|
24
|
+
Subagent routing:
|
|
25
|
+
subagent.selected → Subagent chosen
|
|
26
|
+
subagent.started → Subagent execution begins
|
|
27
|
+
subagent.completed → Subagent execution ends
|
|
28
|
+
subagent.failed → Subagent execution failed
|
|
29
|
+
|
|
30
|
+
Session lifecycle:
|
|
31
|
+
session.start → Session metadata (model, etc.)
|
|
32
|
+
session.resume → Session resumed
|
|
33
|
+
session.idle → Eval finished processing
|
|
34
|
+
session.error → Error occurred
|
|
35
|
+
session.shutdown → Session terminated; carries premium request count
|
|
36
|
+
session.info → Informational message
|
|
37
|
+
session.model_change → Model changed mid-session
|
|
38
|
+
session.usage_info → Context-window usage (tokens/limit; no premium requests)
|
|
39
|
+
session.handoff → Session handoff
|
|
40
|
+
session.truncation → Context truncation
|
|
41
|
+
session.compaction_start → Compaction started
|
|
42
|
+
session.compaction_complete → Compaction completed
|
|
43
|
+
session.snapshot_rewind → Snapshot rewind
|
|
44
|
+
|
|
45
|
+
User:
|
|
46
|
+
user.message → User Turn
|
|
47
|
+
|
|
48
|
+
Other:
|
|
49
|
+
skill.invoked → Skill activation
|
|
50
|
+
hook.start / hook.end → Hook lifecycle
|
|
51
|
+
system.message → System message
|
|
52
|
+
abort → Abort signal
|
|
53
|
+
pending_messages.modified → Queue change
|
|
54
|
+
unknown → Forward-compat catch-all
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import base64
|
|
60
|
+
import binascii
|
|
61
|
+
import logging
|
|
62
|
+
import time
|
|
63
|
+
from collections import deque
|
|
64
|
+
from datetime import timedelta
|
|
65
|
+
from typing import TYPE_CHECKING, Any
|
|
66
|
+
|
|
67
|
+
from pytest_copilot.copilot.result import (
|
|
68
|
+
CopilotResult,
|
|
69
|
+
SubagentInvocation,
|
|
70
|
+
SubagentStatus,
|
|
71
|
+
ToolCall,
|
|
72
|
+
Turn,
|
|
73
|
+
UsageInfo,
|
|
74
|
+
)
|
|
75
|
+
from pytest_copilot.core.result import ImageContent
|
|
76
|
+
|
|
77
|
+
if TYPE_CHECKING:
|
|
78
|
+
from copilot.generated.session_events import SessionEvent
|
|
79
|
+
|
|
80
|
+
logger = logging.getLogger(__name__)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class EventMapper:
|
|
84
|
+
"""Accumulates SDK events and builds a CopilotResult.
|
|
85
|
+
|
|
86
|
+
Usage:
|
|
87
|
+
mapper = EventMapper()
|
|
88
|
+
for event in events:
|
|
89
|
+
mapper.handle(event)
|
|
90
|
+
result = mapper.build()
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(self) -> None:
|
|
94
|
+
self._turns: list[Turn] = []
|
|
95
|
+
self._pending_tool_calls: dict[str, ToolCall] = {} # tool_call_id → ToolCall
|
|
96
|
+
self._early_tool_completions: dict[str, list[SessionEvent]] = {}
|
|
97
|
+
self._pending_tool_start_times: dict[str, float] = {}
|
|
98
|
+
self._current_assistant_content: list[str] = []
|
|
99
|
+
self._current_tool_calls: list[ToolCall] = []
|
|
100
|
+
self._usage: list[UsageInfo] = []
|
|
101
|
+
self._reasoning_traces: list[str] = []
|
|
102
|
+
self._reasoning_buffer: list[str] = []
|
|
103
|
+
self._subagents: list[SubagentInvocation] = []
|
|
104
|
+
self._subagents_by_invocation_id: dict[str, SubagentInvocation] = {}
|
|
105
|
+
self._pending_selected_subagents: dict[str, deque[SubagentInvocation]] = {}
|
|
106
|
+
self._open_subagent_invocations: dict[str, deque[str]] = {}
|
|
107
|
+
self._subagent_start_times: dict[str, float] = {}
|
|
108
|
+
self._tool_subagent_call_ids: dict[str, str] = {} # call_id → agent_name
|
|
109
|
+
self._permissions: list[dict[str, Any]] = []
|
|
110
|
+
self._permission_requested: bool = False
|
|
111
|
+
self._model_used: str | None = None
|
|
112
|
+
self._error: str | None = None
|
|
113
|
+
self._contract_errors: list[str] = []
|
|
114
|
+
self._raw_events: list[Any] = []
|
|
115
|
+
self._start_time: float = time.monotonic()
|
|
116
|
+
self._total_premium_requests: float = 0.0
|
|
117
|
+
|
|
118
|
+
def handle(self, event: SessionEvent) -> None:
|
|
119
|
+
"""Process a single SDK event."""
|
|
120
|
+
self._raw_events.append(event)
|
|
121
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
122
|
+
|
|
123
|
+
handler = _EVENT_HANDLERS.get(event_type)
|
|
124
|
+
if handler:
|
|
125
|
+
handler(self, event)
|
|
126
|
+
else:
|
|
127
|
+
logger.debug("Unhandled event type: %s", event_type)
|
|
128
|
+
|
|
129
|
+
def build(self) -> CopilotResult:
|
|
130
|
+
"""Build the final CopilotResult from accumulated events."""
|
|
131
|
+
# Flush any pending assistant content
|
|
132
|
+
self._flush_assistant_turn()
|
|
133
|
+
for call_id, tool_call in self._pending_tool_calls.items():
|
|
134
|
+
if not tool_call.completion_received:
|
|
135
|
+
self._record_contract_error(f"Tool call {call_id} is missing completion")
|
|
136
|
+
for call_id in self._early_tool_completions:
|
|
137
|
+
self._record_contract_error(f"Completion for unknown tool call {call_id}")
|
|
138
|
+
|
|
139
|
+
resolved_subagents = [subagent for subagent in self._subagents if subagent.invocation_id]
|
|
140
|
+
unresolved_subagents = [
|
|
141
|
+
subagent.name for subagent in self._subagents if not subagent.invocation_id
|
|
142
|
+
]
|
|
143
|
+
if unresolved_subagents:
|
|
144
|
+
self._record_contract_error(
|
|
145
|
+
"Subagent selected event was not correlated to a tool_call_id: "
|
|
146
|
+
+ ", ".join(unresolved_subagents)
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
errors = ([self._error] if self._error else []) + self._contract_errors
|
|
150
|
+
error = "; ".join(errors) or None
|
|
151
|
+
|
|
152
|
+
duration_ms = (time.monotonic() - self._start_time) * 1000
|
|
153
|
+
has_error = error is not None
|
|
154
|
+
|
|
155
|
+
return CopilotResult(
|
|
156
|
+
turns=self._turns,
|
|
157
|
+
success=not has_error,
|
|
158
|
+
error=error,
|
|
159
|
+
capture_errors=list(self._contract_errors),
|
|
160
|
+
duration_ms=duration_ms,
|
|
161
|
+
usage=self._usage,
|
|
162
|
+
reasoning_traces=self._reasoning_traces,
|
|
163
|
+
subagent_invocations=resolved_subagents,
|
|
164
|
+
permission_requested=self._permission_requested,
|
|
165
|
+
permissions=self._permissions,
|
|
166
|
+
model_used=self._model_used,
|
|
167
|
+
raw_events=self._raw_events,
|
|
168
|
+
total_premium_requests=self._total_premium_requests
|
|
169
|
+
+ sum(
|
|
170
|
+
invocation.result.total_premium_requests
|
|
171
|
+
for invocation in resolved_subagents
|
|
172
|
+
if invocation.result is not None
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# ── Assistant events ──
|
|
177
|
+
|
|
178
|
+
def _handle_assistant_message(self, event: SessionEvent) -> None:
|
|
179
|
+
"""Handle complete assistant message.
|
|
180
|
+
|
|
181
|
+
The SDK fires both streaming deltas (accumulated via turn_end)
|
|
182
|
+
AND a complete assistant.message with the same content. We must
|
|
183
|
+
avoid creating duplicate turns.
|
|
184
|
+
"""
|
|
185
|
+
content = _get_data_field(event, "content", "")
|
|
186
|
+
|
|
187
|
+
# If we have accumulated delta content that hasn't been flushed
|
|
188
|
+
# yet, the complete message supersedes it — clear and replace.
|
|
189
|
+
if self._current_assistant_content:
|
|
190
|
+
self._current_assistant_content.clear()
|
|
191
|
+
if content:
|
|
192
|
+
self._current_assistant_content.append(content)
|
|
193
|
+
elif content:
|
|
194
|
+
# Deltas were already flushed by turn_end — check if the
|
|
195
|
+
# last turn already has this exact content to avoid duplication.
|
|
196
|
+
if (
|
|
197
|
+
self._turns
|
|
198
|
+
and self._turns[-1].role == "assistant"
|
|
199
|
+
and self._turns[-1].content == content
|
|
200
|
+
):
|
|
201
|
+
# Already flushed by turn_end — skip
|
|
202
|
+
pass
|
|
203
|
+
else:
|
|
204
|
+
self._current_assistant_content.append(content)
|
|
205
|
+
|
|
206
|
+
# Check for tool_requests in the message
|
|
207
|
+
# SDK returns ToolRequest dataclass objects, not dicts
|
|
208
|
+
# Requests and execution events share one call object across turns.
|
|
209
|
+
tool_requests = _get_data_field(event, "tool_requests", None)
|
|
210
|
+
if tool_requests:
|
|
211
|
+
for req in tool_requests:
|
|
212
|
+
call_id = getattr(req, "tool_call_id", "") or ""
|
|
213
|
+
name = getattr(req, "name", "unknown")
|
|
214
|
+
arguments = getattr(req, "arguments", {})
|
|
215
|
+
if isinstance(arguments, str):
|
|
216
|
+
import json
|
|
217
|
+
|
|
218
|
+
try:
|
|
219
|
+
arguments = json.loads(arguments)
|
|
220
|
+
except json.JSONDecodeError:
|
|
221
|
+
arguments = {"raw": arguments}
|
|
222
|
+
self._track_tool_call(call_id, name, arguments or {})
|
|
223
|
+
|
|
224
|
+
def _handle_assistant_message_delta(self, event: SessionEvent) -> None:
|
|
225
|
+
"""Handle streaming delta — accumulate content."""
|
|
226
|
+
delta = _get_data_field(event, "delta_content", "")
|
|
227
|
+
if delta:
|
|
228
|
+
self._current_assistant_content.append(delta)
|
|
229
|
+
|
|
230
|
+
def _handle_assistant_reasoning(self, event: SessionEvent) -> None:
|
|
231
|
+
"""Handle complete reasoning trace."""
|
|
232
|
+
text = _get_data_field(event, "reasoning_text", "")
|
|
233
|
+
if text:
|
|
234
|
+
# Flush any buffered deltas first
|
|
235
|
+
if self._reasoning_buffer:
|
|
236
|
+
self._reasoning_traces.append("".join(self._reasoning_buffer))
|
|
237
|
+
self._reasoning_buffer.clear()
|
|
238
|
+
self._reasoning_traces.append(text)
|
|
239
|
+
|
|
240
|
+
def _handle_assistant_reasoning_delta(self, event: SessionEvent) -> None:
|
|
241
|
+
"""Handle streaming reasoning delta."""
|
|
242
|
+
delta = _get_data_field(event, "delta_content", "")
|
|
243
|
+
if delta:
|
|
244
|
+
self._reasoning_buffer.append(delta)
|
|
245
|
+
|
|
246
|
+
def _handle_assistant_turn_start(self, event: SessionEvent) -> None:
|
|
247
|
+
"""Mark the start of a new assistant turn."""
|
|
248
|
+
# Flush previous turn if any
|
|
249
|
+
self._flush_assistant_turn()
|
|
250
|
+
|
|
251
|
+
def _handle_assistant_turn_end(self, event: SessionEvent) -> None:
|
|
252
|
+
"""Mark the end of an assistant turn."""
|
|
253
|
+
# Flush reasoning buffer
|
|
254
|
+
if self._reasoning_buffer:
|
|
255
|
+
self._reasoning_traces.append("".join(self._reasoning_buffer))
|
|
256
|
+
self._reasoning_buffer.clear()
|
|
257
|
+
self._flush_assistant_turn()
|
|
258
|
+
|
|
259
|
+
def _handle_assistant_usage(self, event: SessionEvent) -> None:
|
|
260
|
+
"""Handle token usage report."""
|
|
261
|
+
model = _get_data_field(event, "model", "unknown")
|
|
262
|
+
self._model_used = model
|
|
263
|
+
duration = _get_data_field(event, "duration", None)
|
|
264
|
+
duration_ms = (
|
|
265
|
+
duration.total_seconds() * 1000 if isinstance(duration, timedelta) else duration
|
|
266
|
+
)
|
|
267
|
+
self._usage.append(
|
|
268
|
+
UsageInfo(
|
|
269
|
+
model=model,
|
|
270
|
+
input_tokens=_get_data_field(event, "input_tokens", None),
|
|
271
|
+
output_tokens=_get_data_field(event, "output_tokens", None),
|
|
272
|
+
cache_read_tokens=_get_data_field(event, "cache_read_tokens", None),
|
|
273
|
+
cache_write_tokens=_get_data_field(event, "cache_write_tokens", None),
|
|
274
|
+
reasoning_tokens=_get_data_field(event, "reasoning_tokens", None),
|
|
275
|
+
reasoning_effort=_get_data_field(event, "reasoning_effort", None),
|
|
276
|
+
duration_ms=duration_ms,
|
|
277
|
+
)
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# ── Tool events ──
|
|
281
|
+
|
|
282
|
+
# Tool names that represent subagent dispatch (native SDK tools).
|
|
283
|
+
_SUBAGENT_TOOL_NAMES = frozenset({"runSubagent", "task"})
|
|
284
|
+
|
|
285
|
+
def _track_tool_call(self, call_id: str, name: str, arguments: Any) -> ToolCall | None:
|
|
286
|
+
if not call_id:
|
|
287
|
+
self._record_contract_error(f"Tool {name} is missing tool_call_id")
|
|
288
|
+
return None
|
|
289
|
+
if arguments is None:
|
|
290
|
+
arguments = {}
|
|
291
|
+
tc = self._pending_tool_calls.get(call_id)
|
|
292
|
+
if tc is None:
|
|
293
|
+
tc = ToolCall(
|
|
294
|
+
name=name, arguments=arguments, call_id=call_id, completion_received=False
|
|
295
|
+
)
|
|
296
|
+
self._pending_tool_calls[call_id] = tc
|
|
297
|
+
self._current_tool_calls.append(tc)
|
|
298
|
+
for completion in self._early_tool_completions.pop(call_id, []):
|
|
299
|
+
self._handle_tool_execution_complete(completion)
|
|
300
|
+
elif tc.name != name or tc.arguments != arguments:
|
|
301
|
+
self._record_contract_error(
|
|
302
|
+
f"Conflicting identity or arguments for tool call {call_id}"
|
|
303
|
+
)
|
|
304
|
+
return tc
|
|
305
|
+
|
|
306
|
+
def _handle_tool_execution_start(self, event: SessionEvent) -> None:
|
|
307
|
+
"""Handle tool execution starting."""
|
|
308
|
+
call_id = _get_data_field(event, "tool_call_id", "")
|
|
309
|
+
name = _get_data_field(event, "tool_name", "unknown")
|
|
310
|
+
arguments = _get_data_field(event, "arguments", {})
|
|
311
|
+
|
|
312
|
+
if isinstance(arguments, str):
|
|
313
|
+
import json
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
arguments = json.loads(arguments)
|
|
317
|
+
except json.JSONDecodeError:
|
|
318
|
+
arguments = {"raw": arguments}
|
|
319
|
+
|
|
320
|
+
tc = self._track_tool_call(call_id, name, arguments)
|
|
321
|
+
if tc is None or tc.completion_received or call_id in self._pending_tool_start_times:
|
|
322
|
+
return
|
|
323
|
+
self._pending_tool_start_times[call_id] = time.monotonic()
|
|
324
|
+
|
|
325
|
+
# Detect subagent dispatch via native tool calls (runSubagent/task).
|
|
326
|
+
# The SDK may or may not emit separate subagent.* events, so we
|
|
327
|
+
# also track invocations here as a fallback.
|
|
328
|
+
if name in self._SUBAGENT_TOOL_NAMES:
|
|
329
|
+
agent_name = _resolve_tool_subagent_name(arguments)
|
|
330
|
+
if not call_id:
|
|
331
|
+
self._record_contract_error(f"{name} dispatch is missing tool_call_id")
|
|
332
|
+
elif agent_name is None:
|
|
333
|
+
self._record_contract_error(
|
|
334
|
+
f"{name} dispatch {call_id} is missing required agentSlug"
|
|
335
|
+
)
|
|
336
|
+
else:
|
|
337
|
+
self._tool_subagent_call_ids[call_id] = agent_name
|
|
338
|
+
self.record_subagent_start(invocation_id=call_id, name=agent_name)
|
|
339
|
+
|
|
340
|
+
def _handle_tool_execution_complete(self, event: SessionEvent) -> None:
|
|
341
|
+
"""Handle tool execution completed."""
|
|
342
|
+
call_id = _get_data_field(event, "tool_call_id", "")
|
|
343
|
+
result_data = _get_data_field(event, "result", None)
|
|
344
|
+
|
|
345
|
+
tc = self._pending_tool_calls.get(call_id)
|
|
346
|
+
if tc is None:
|
|
347
|
+
if not call_id:
|
|
348
|
+
self._record_contract_error("Tool completion is missing tool_call_id")
|
|
349
|
+
return
|
|
350
|
+
self._early_tool_completions.setdefault(call_id, []).append(event)
|
|
351
|
+
return
|
|
352
|
+
result_text: str | None = None
|
|
353
|
+
images: list[ImageContent] = []
|
|
354
|
+
if result_data is not None and hasattr(result_data, "content"):
|
|
355
|
+
result_text = result_data.content
|
|
356
|
+
elif isinstance(result_data, str):
|
|
357
|
+
result_text = result_data
|
|
358
|
+
elif result_data is not None:
|
|
359
|
+
result_text = str(result_data)
|
|
360
|
+
text_parts: list[str] = []
|
|
361
|
+
blocks = getattr(result_data, "contents", None) or []
|
|
362
|
+
for block in blocks:
|
|
363
|
+
block_type = getattr(block, "type", None)
|
|
364
|
+
if block_type == "image":
|
|
365
|
+
try:
|
|
366
|
+
data = getattr(block, "data", None)
|
|
367
|
+
media_type = getattr(block, "mime_type", None)
|
|
368
|
+
if (
|
|
369
|
+
not isinstance(data, str)
|
|
370
|
+
or not isinstance(media_type, str)
|
|
371
|
+
or not media_type
|
|
372
|
+
):
|
|
373
|
+
raise ValueError("image data and media type are required")
|
|
374
|
+
images.append(ImageContent(base64.b64decode(data, validate=True), media_type))
|
|
375
|
+
except (ValueError, binascii.Error) as exc:
|
|
376
|
+
self._record_contract_error(f"Tool call {call_id} has an invalid image: {exc}")
|
|
377
|
+
elif block_type == "text" and result_text is None:
|
|
378
|
+
text = getattr(block, "text", None)
|
|
379
|
+
if text is not None:
|
|
380
|
+
text_parts.append(text)
|
|
381
|
+
if result_text is None and text_parts:
|
|
382
|
+
result_text = "\n".join(text_parts)
|
|
383
|
+
content_images = list(images)
|
|
384
|
+
for block in getattr(result_data, "binary_results_for_llm", None) or []:
|
|
385
|
+
block_type = getattr(block, "type", None)
|
|
386
|
+
if getattr(block_type, "value", block_type) != "image":
|
|
387
|
+
continue
|
|
388
|
+
try:
|
|
389
|
+
data, media_type = block.data, block.mime_type
|
|
390
|
+
if not isinstance(data, str) or not isinstance(media_type, str) or not media_type:
|
|
391
|
+
raise ValueError("image data and media type are required")
|
|
392
|
+
image = ImageContent(base64.b64decode(data, validate=True), media_type)
|
|
393
|
+
if image not in content_images:
|
|
394
|
+
images.append(image)
|
|
395
|
+
except (ValueError, binascii.Error) as exc:
|
|
396
|
+
self._record_contract_error(f"Tool call {call_id} has an invalid image: {exc}")
|
|
397
|
+
success = _get_data_field(event, "success", None)
|
|
398
|
+
error = _stringify_tool_error(_get_data_field(event, "error", None))
|
|
399
|
+
if tc.completion_received:
|
|
400
|
+
captured_images = (
|
|
401
|
+
[ImageContent(tc.image_content, tc.image_media_type or "image/png")]
|
|
402
|
+
if tc.image_content is not None
|
|
403
|
+
else []
|
|
404
|
+
) + tc.additional_images
|
|
405
|
+
if (tc.result, tc.success, tc.error, captured_images) != (
|
|
406
|
+
result_text,
|
|
407
|
+
success,
|
|
408
|
+
error,
|
|
409
|
+
images,
|
|
410
|
+
):
|
|
411
|
+
self._record_contract_error(f"Conflicting completion for tool call {call_id}")
|
|
412
|
+
return
|
|
413
|
+
tc.completion_received = True
|
|
414
|
+
tc.success = success
|
|
415
|
+
tc.result = result_text
|
|
416
|
+
tc.error = error
|
|
417
|
+
if images:
|
|
418
|
+
tc.image_content, tc.image_media_type = images[0].data, images[0].media_type
|
|
419
|
+
tc.additional_images = images[1:]
|
|
420
|
+
if tc.success is None:
|
|
421
|
+
self._record_contract_error(f"Tool call {call_id} completion is missing success")
|
|
422
|
+
start = self._pending_tool_start_times.pop(call_id, None)
|
|
423
|
+
if start is not None:
|
|
424
|
+
tc.duration_ms = (time.monotonic() - start) * 1000
|
|
425
|
+
|
|
426
|
+
# Complete subagent tracking from tool call
|
|
427
|
+
agent_name = self._tool_subagent_call_ids.pop(call_id, None)
|
|
428
|
+
if agent_name:
|
|
429
|
+
if tc.success is True:
|
|
430
|
+
self.record_subagent_complete(invocation_id=call_id, name=agent_name)
|
|
431
|
+
else:
|
|
432
|
+
self.record_subagent_failed(invocation_id=call_id, name=agent_name)
|
|
433
|
+
|
|
434
|
+
# Add a tool turn for reporting
|
|
435
|
+
tool_name = tc.name
|
|
436
|
+
result_text = tc.result
|
|
437
|
+
self._turns.append(Turn(role="tool", content=f"[{tool_name}] {result_text or ''}"))
|
|
438
|
+
|
|
439
|
+
def _handle_abort(self, event: SessionEvent) -> None:
|
|
440
|
+
self._error = f"Session aborted: {_get_data_field(event, 'reason', None)}"
|
|
441
|
+
|
|
442
|
+
# ── Subagent recording (used by runSubagent tool handler) ──
|
|
443
|
+
|
|
444
|
+
def record_subagent_result(
|
|
445
|
+
self, *, invocation_id: str, name: str, result: CopilotResult
|
|
446
|
+
) -> None:
|
|
447
|
+
"""Link a framework-owned child without mixing its conversation into this one."""
|
|
448
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
449
|
+
if invocation.result is not None and invocation.result is not result:
|
|
450
|
+
self._record_contract_error(f"Conflicting child result for invocation {invocation_id}")
|
|
451
|
+
return
|
|
452
|
+
invocation.result = result
|
|
453
|
+
|
|
454
|
+
def record_subagent_start(self, *, invocation_id: str, name: str) -> None:
|
|
455
|
+
"""Record a subagent invocation dispatched via a tool call."""
|
|
456
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
457
|
+
self._enqueue_open_subagent(name=name, invocation_id=invocation_id)
|
|
458
|
+
self._subagent_start_times.setdefault(invocation_id, time.monotonic())
|
|
459
|
+
self._advance_subagent_status(invocation, "started")
|
|
460
|
+
|
|
461
|
+
def record_subagent_complete(self, *, invocation_id: str, name: str) -> None:
|
|
462
|
+
"""Mark a previously started subagent invocation as completed."""
|
|
463
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
464
|
+
duration = self._resolve_subagent_duration_ms(invocation_id=invocation_id)
|
|
465
|
+
self._dequeue_open_subagent(name=name, invocation_id=invocation_id)
|
|
466
|
+
self._advance_subagent_status(invocation, "completed", duration_ms=duration)
|
|
467
|
+
|
|
468
|
+
def record_subagent_failed(self, *, invocation_id: str, name: str) -> None:
|
|
469
|
+
"""Mark a previously started subagent invocation as failed."""
|
|
470
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
471
|
+
duration = self._resolve_subagent_duration_ms(invocation_id=invocation_id)
|
|
472
|
+
self._dequeue_open_subagent(name=name, invocation_id=invocation_id)
|
|
473
|
+
self._advance_subagent_status(invocation, "failed", duration_ms=duration)
|
|
474
|
+
|
|
475
|
+
# ── Subagent events ──
|
|
476
|
+
|
|
477
|
+
def _handle_subagent_selected(self, event: SessionEvent) -> None:
|
|
478
|
+
"""Handle subagent selection."""
|
|
479
|
+
name = _require_subagent_name(event)
|
|
480
|
+
if name is None:
|
|
481
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
482
|
+
self._record_contract_error(f"{event_type} is missing required agent_name")
|
|
483
|
+
return
|
|
484
|
+
|
|
485
|
+
invocation_id = self._peek_open_subagent(name)
|
|
486
|
+
if invocation_id is None:
|
|
487
|
+
pending = SubagentInvocation(invocation_id="", name=name, status="selected")
|
|
488
|
+
self._subagents.append(pending)
|
|
489
|
+
self._pending_selected_subagents.setdefault(name, deque()).append(pending)
|
|
490
|
+
return
|
|
491
|
+
|
|
492
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
493
|
+
self._advance_subagent_status(invocation, "selected")
|
|
494
|
+
|
|
495
|
+
def _handle_subagent_started(self, event: SessionEvent) -> None:
|
|
496
|
+
"""Handle subagent execution start."""
|
|
497
|
+
name = _require_subagent_name(event)
|
|
498
|
+
invocation_id = _require_invocation_id(event)
|
|
499
|
+
if name is None or invocation_id is None:
|
|
500
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
501
|
+
self._record_contract_error(
|
|
502
|
+
f"{event_type} is missing required subagent correlation fields"
|
|
503
|
+
)
|
|
504
|
+
return
|
|
505
|
+
|
|
506
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
507
|
+
self._enqueue_open_subagent(name=name, invocation_id=invocation_id)
|
|
508
|
+
self._subagent_start_times.setdefault(invocation_id, time.monotonic())
|
|
509
|
+
self._advance_subagent_status(invocation, "started")
|
|
510
|
+
|
|
511
|
+
def _handle_subagent_completed(self, event: SessionEvent) -> None:
|
|
512
|
+
"""Handle subagent execution completion."""
|
|
513
|
+
name = _require_subagent_name(event)
|
|
514
|
+
invocation_id = _require_invocation_id(event)
|
|
515
|
+
if name is None or invocation_id is None:
|
|
516
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
517
|
+
self._record_contract_error(
|
|
518
|
+
f"{event_type} is missing required subagent correlation fields"
|
|
519
|
+
)
|
|
520
|
+
return
|
|
521
|
+
|
|
522
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
523
|
+
self._dequeue_open_subagent(name=name, invocation_id=invocation_id)
|
|
524
|
+
duration = _duration_to_ms(_get_data_field(event, "duration", None))
|
|
525
|
+
if duration is None:
|
|
526
|
+
duration = self._resolve_subagent_duration_ms(invocation_id=invocation_id)
|
|
527
|
+
self._advance_subagent_status(invocation, "completed", duration_ms=duration)
|
|
528
|
+
|
|
529
|
+
def _handle_subagent_failed(self, event: SessionEvent) -> None:
|
|
530
|
+
"""Handle subagent execution failure."""
|
|
531
|
+
name = _require_subagent_name(event)
|
|
532
|
+
invocation_id = _require_invocation_id(event)
|
|
533
|
+
if name is None or invocation_id is None:
|
|
534
|
+
event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
|
|
535
|
+
self._record_contract_error(
|
|
536
|
+
f"{event_type} is missing required subagent correlation fields"
|
|
537
|
+
)
|
|
538
|
+
return
|
|
539
|
+
|
|
540
|
+
invocation = self._ensure_subagent_invocation(invocation_id=invocation_id, name=name)
|
|
541
|
+
self._dequeue_open_subagent(name=name, invocation_id=invocation_id)
|
|
542
|
+
duration = _duration_to_ms(_get_data_field(event, "duration", None))
|
|
543
|
+
if duration is None:
|
|
544
|
+
duration = self._resolve_subagent_duration_ms(invocation_id=invocation_id)
|
|
545
|
+
self._advance_subagent_status(invocation, "failed", duration_ms=duration)
|
|
546
|
+
|
|
547
|
+
# ── Session events ──
|
|
548
|
+
|
|
549
|
+
def _handle_session_start(self, event: SessionEvent) -> None:
|
|
550
|
+
"""Handle session start — capture model selection."""
|
|
551
|
+
model = _get_data_field(event, "selected_model", None)
|
|
552
|
+
if model:
|
|
553
|
+
self._model_used = model
|
|
554
|
+
|
|
555
|
+
def _handle_session_usage_info(self, event: SessionEvent) -> None:
|
|
556
|
+
"""Handle session-level context-window usage (tokens/limit).
|
|
557
|
+
|
|
558
|
+
SDK 1.x's ``session.usage_info`` no longer carries a premium-request
|
|
559
|
+
count (it's context-window stats only). Retained as a documented
|
|
560
|
+
no-op handler so the event isn't logged as unhandled; see
|
|
561
|
+
``_handle_session_shutdown`` for premium requests.
|
|
562
|
+
"""
|
|
563
|
+
|
|
564
|
+
def _handle_session_shutdown(self, event: SessionEvent) -> None:
|
|
565
|
+
"""Handle session shutdown — capture the total premium request count.
|
|
566
|
+
|
|
567
|
+
KNOWN GAP (verified with SDK 1.0.13): ``_total_premium_requests`` exists on
|
|
568
|
+
``SessionShutdownData`` per the generated schema, but empirically
|
|
569
|
+
``session.shutdown`` is never delivered to ``session.on()`` listeners
|
|
570
|
+
via the ``client.stop()`` teardown path this runner uses. The live
|
|
571
|
+
event integration test recorded zero shutdown events even with
|
|
572
|
+
its listener registered before session creation. So ``premium_requests``
|
|
573
|
+
stays 0.0 in practice today. This handler is kept because it's
|
|
574
|
+
schema-correct and harmless, in case a future SDK version emits the
|
|
575
|
+
event (or an alternate teardown path does). If this still reads 0.0
|
|
576
|
+
after upgrading github-copilot-sdk, check whether the SDK now
|
|
577
|
+
exposes a public usage/metrics RPC instead of relying on this event.
|
|
578
|
+
"""
|
|
579
|
+
self._total_premium_requests = float(
|
|
580
|
+
_get_data_field(event, "_total_premium_requests", 0) or 0
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
def _handle_session_error(self, event: SessionEvent) -> None:
|
|
584
|
+
"""Handle session error."""
|
|
585
|
+
msg = _get_data_field(event, "message", "Unknown error")
|
|
586
|
+
error_type = _get_data_field(event, "error_type", "")
|
|
587
|
+
self._error = f"{error_type}: {msg}" if error_type else msg
|
|
588
|
+
|
|
589
|
+
# ── User events ──
|
|
590
|
+
|
|
591
|
+
def _handle_user_message(self, event: SessionEvent) -> None:
|
|
592
|
+
"""Handle user message — create a user turn."""
|
|
593
|
+
content = _get_data_field(event, "content", "")
|
|
594
|
+
if content:
|
|
595
|
+
self._turns.append(Turn(role="user", content=content))
|
|
596
|
+
|
|
597
|
+
# ── Permission events ──
|
|
598
|
+
|
|
599
|
+
def _handle_permission(self, event: SessionEvent) -> None:
|
|
600
|
+
"""Handle permission request."""
|
|
601
|
+
self._permission_requested = True
|
|
602
|
+
self._permissions.append(
|
|
603
|
+
{
|
|
604
|
+
"type": _get_data_field(event, "permission_type", "unknown"),
|
|
605
|
+
"tool": _get_data_field(event, "tool_name", None),
|
|
606
|
+
"message": _get_data_field(event, "message", ""),
|
|
607
|
+
}
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
# ── Internal helpers ──
|
|
611
|
+
|
|
612
|
+
def _flush_assistant_turn(self) -> None:
|
|
613
|
+
"""Flush accumulated assistant content into a Turn."""
|
|
614
|
+
if self._current_assistant_content or self._current_tool_calls:
|
|
615
|
+
content = "".join(self._current_assistant_content)
|
|
616
|
+
self._turns.append(
|
|
617
|
+
Turn(
|
|
618
|
+
role="assistant",
|
|
619
|
+
content=content,
|
|
620
|
+
tool_calls=list(self._current_tool_calls),
|
|
621
|
+
)
|
|
622
|
+
)
|
|
623
|
+
self._current_assistant_content.clear()
|
|
624
|
+
self._current_tool_calls.clear()
|
|
625
|
+
|
|
626
|
+
def _ensure_subagent_invocation(
|
|
627
|
+
self,
|
|
628
|
+
*,
|
|
629
|
+
invocation_id: str,
|
|
630
|
+
name: str,
|
|
631
|
+
) -> SubagentInvocation:
|
|
632
|
+
"""Get or create a subagent invocation keyed by invocation ID."""
|
|
633
|
+
existing = self._subagents_by_invocation_id.get(invocation_id)
|
|
634
|
+
if existing is not None:
|
|
635
|
+
if existing.name != name:
|
|
636
|
+
self._record_contract_error(
|
|
637
|
+
"Subagent invocation "
|
|
638
|
+
f"{invocation_id} changed name from {existing.name} to {name}"
|
|
639
|
+
)
|
|
640
|
+
return existing
|
|
641
|
+
|
|
642
|
+
pending = self._consume_pending_selected(name=name)
|
|
643
|
+
if pending is not None:
|
|
644
|
+
pending.invocation_id = invocation_id
|
|
645
|
+
self._subagents_by_invocation_id[invocation_id] = pending
|
|
646
|
+
return pending
|
|
647
|
+
|
|
648
|
+
invocation = SubagentInvocation(invocation_id=invocation_id, name=name, status="started")
|
|
649
|
+
self._subagents.append(invocation)
|
|
650
|
+
self._subagents_by_invocation_id[invocation_id] = invocation
|
|
651
|
+
return invocation
|
|
652
|
+
|
|
653
|
+
def _consume_pending_selected(self, *, name: str) -> SubagentInvocation | None:
|
|
654
|
+
pending = self._pending_selected_subagents.get(name)
|
|
655
|
+
if not pending:
|
|
656
|
+
return None
|
|
657
|
+
invocation = pending.popleft()
|
|
658
|
+
if not pending:
|
|
659
|
+
self._pending_selected_subagents.pop(name, None)
|
|
660
|
+
return invocation
|
|
661
|
+
|
|
662
|
+
def _enqueue_open_subagent(self, *, name: str, invocation_id: str) -> None:
|
|
663
|
+
queue = self._open_subagent_invocations.setdefault(name, deque())
|
|
664
|
+
if invocation_id not in queue:
|
|
665
|
+
queue.append(invocation_id)
|
|
666
|
+
|
|
667
|
+
def _peek_open_subagent(self, name: str) -> str | None:
|
|
668
|
+
queue = self._open_subagent_invocations.get(name)
|
|
669
|
+
if not queue:
|
|
670
|
+
return None
|
|
671
|
+
return queue[0]
|
|
672
|
+
|
|
673
|
+
def _dequeue_open_subagent(self, *, name: str, invocation_id: str) -> None:
|
|
674
|
+
queue = self._open_subagent_invocations.get(name)
|
|
675
|
+
if not queue:
|
|
676
|
+
return
|
|
677
|
+
try:
|
|
678
|
+
queue.remove(invocation_id)
|
|
679
|
+
except ValueError:
|
|
680
|
+
return
|
|
681
|
+
if not queue:
|
|
682
|
+
self._open_subagent_invocations.pop(name, None)
|
|
683
|
+
|
|
684
|
+
def _advance_subagent_status(
|
|
685
|
+
self,
|
|
686
|
+
invocation: SubagentInvocation,
|
|
687
|
+
status: SubagentStatus,
|
|
688
|
+
*,
|
|
689
|
+
duration_ms: float | None = None,
|
|
690
|
+
) -> None:
|
|
691
|
+
status_order = {"selected": 0, "started": 1, "completed": 2, "failed": 2}
|
|
692
|
+
if status_order[status] < status_order[invocation.status]:
|
|
693
|
+
return
|
|
694
|
+
if invocation.status in {"completed", "failed"} and invocation.status != status:
|
|
695
|
+
self._record_contract_error(
|
|
696
|
+
"Subagent invocation "
|
|
697
|
+
f"{invocation.invocation_id} ended twice with conflicting states"
|
|
698
|
+
)
|
|
699
|
+
return
|
|
700
|
+
invocation.status = status
|
|
701
|
+
if duration_ms is not None:
|
|
702
|
+
invocation.duration_ms = duration_ms
|
|
703
|
+
|
|
704
|
+
def _resolve_subagent_duration_ms(self, *, invocation_id: str) -> float | None:
|
|
705
|
+
start = self._subagent_start_times.pop(invocation_id, None)
|
|
706
|
+
if start is None:
|
|
707
|
+
return None
|
|
708
|
+
return (time.monotonic() - start) * 1000
|
|
709
|
+
|
|
710
|
+
def _record_contract_error(self, message: str) -> None:
|
|
711
|
+
if message not in self._contract_errors:
|
|
712
|
+
self._contract_errors.append(message)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def _get_data_field(event: SessionEvent, field: str, default: Any = None) -> Any:
|
|
716
|
+
"""Safely get a field from event.data (which has ~90 optional fields)."""
|
|
717
|
+
return getattr(event.data, field, default)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _require_subagent_name(event: SessionEvent) -> str | None:
|
|
721
|
+
"""Extract the current SDK subagent name field."""
|
|
722
|
+
name = _get_data_field(event, "agent_name", None)
|
|
723
|
+
if isinstance(name, str) and name:
|
|
724
|
+
return name
|
|
725
|
+
return None
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def _require_invocation_id(event: SessionEvent) -> str | None:
|
|
729
|
+
"""Extract the current SDK subagent invocation correlation field."""
|
|
730
|
+
invocation_id = _get_data_field(event, "tool_call_id", None)
|
|
731
|
+
if isinstance(invocation_id, str) and invocation_id:
|
|
732
|
+
return invocation_id
|
|
733
|
+
return None
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _resolve_tool_subagent_name(arguments: Any) -> str | None:
|
|
737
|
+
"""Extract the current dispatch tool key for the target custom agent."""
|
|
738
|
+
if not isinstance(arguments, dict):
|
|
739
|
+
return None
|
|
740
|
+
for key in ("agentSlug", "agent_type", "name"):
|
|
741
|
+
agent_slug = arguments.get(key)
|
|
742
|
+
if isinstance(agent_slug, str) and agent_slug:
|
|
743
|
+
return agent_slug
|
|
744
|
+
return None
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def _duration_to_ms(value: Any) -> float | None:
|
|
748
|
+
"""Convert current SDK duration values to milliseconds."""
|
|
749
|
+
if isinstance(value, timedelta):
|
|
750
|
+
return value.total_seconds() * 1000
|
|
751
|
+
if isinstance(value, (float, int)):
|
|
752
|
+
return float(value)
|
|
753
|
+
return None
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def _stringify_tool_error(error: Any) -> str | None:
|
|
757
|
+
"""Extract a readable tool error message."""
|
|
758
|
+
if error is None:
|
|
759
|
+
return None
|
|
760
|
+
message = getattr(error, "message", None)
|
|
761
|
+
if isinstance(message, str) and message:
|
|
762
|
+
return message
|
|
763
|
+
return str(error)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
# ── Event type → handler dispatch table ──
|
|
767
|
+
|
|
768
|
+
_EVENT_HANDLERS: dict[str, Any] = {
|
|
769
|
+
# Assistant
|
|
770
|
+
"assistant.message": EventMapper._handle_assistant_message,
|
|
771
|
+
"assistant.message_delta": EventMapper._handle_assistant_message_delta,
|
|
772
|
+
"assistant.reasoning": EventMapper._handle_assistant_reasoning,
|
|
773
|
+
"assistant.reasoning_delta": EventMapper._handle_assistant_reasoning_delta,
|
|
774
|
+
"assistant.turn_start": EventMapper._handle_assistant_turn_start,
|
|
775
|
+
"assistant.turn_end": EventMapper._handle_assistant_turn_end,
|
|
776
|
+
"assistant.usage": EventMapper._handle_assistant_usage,
|
|
777
|
+
# Tools
|
|
778
|
+
"tool.execution_start": EventMapper._handle_tool_execution_start,
|
|
779
|
+
"tool.execution_complete": EventMapper._handle_tool_execution_complete,
|
|
780
|
+
# Subagents
|
|
781
|
+
"subagent.selected": EventMapper._handle_subagent_selected,
|
|
782
|
+
"subagent.started": EventMapper._handle_subagent_started,
|
|
783
|
+
"subagent.completed": EventMapper._handle_subagent_completed,
|
|
784
|
+
"subagent.failed": EventMapper._handle_subagent_failed,
|
|
785
|
+
# Session
|
|
786
|
+
"session.start": EventMapper._handle_session_start,
|
|
787
|
+
"session.error": EventMapper._handle_session_error,
|
|
788
|
+
"session.usage_info": EventMapper._handle_session_usage_info,
|
|
789
|
+
"session.shutdown": EventMapper._handle_session_shutdown,
|
|
790
|
+
"abort": EventMapper._handle_abort,
|
|
791
|
+
# User
|
|
792
|
+
"user.message": EventMapper._handle_user_message,
|
|
793
|
+
# Permissions
|
|
794
|
+
"tool.user_requested": EventMapper._handle_permission,
|
|
795
|
+
}
|