pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""CopilotRunner — Execute prompts against GitHub Copilot directly via SDK.
|
|
2
|
+
|
|
3
|
+
This is the core execution engine. It:
|
|
4
|
+
1. Creates a CopilotClient and starts the CLI server
|
|
5
|
+
2. Creates a session with the agent's config
|
|
6
|
+
3. Sends the prompt and captures ALL events
|
|
7
|
+
4. Maps events to a CopilotResult via EventMapper
|
|
8
|
+
5. Cleans up the client
|
|
9
|
+
|
|
10
|
+
No LiteLLM. No outer agent. One LLM. Direct SDK access.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import logging
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from tempfile import TemporaryDirectory
|
|
19
|
+
from typing import TYPE_CHECKING, Any
|
|
20
|
+
|
|
21
|
+
from pytest_copilot.copilot.client import (
|
|
22
|
+
approve_all_permissions,
|
|
23
|
+
create_client,
|
|
24
|
+
stop_client,
|
|
25
|
+
)
|
|
26
|
+
from pytest_copilot.copilot.contracts import CopilotTaskConfig
|
|
27
|
+
from pytest_copilot.copilot.controls import RunControls
|
|
28
|
+
from pytest_copilot.copilot.events import EventMapper
|
|
29
|
+
from pytest_copilot.copilot.requests import RequestAuditHandler
|
|
30
|
+
from pytest_copilot.copilot.result import CopilotResult, StopReason
|
|
31
|
+
from pytest_copilot.copilot.skills import discover_requested_skills, requested_skill_files
|
|
32
|
+
from pytest_copilot.core.result import SkillDiscovery
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from copilot.client import CopilotClient
|
|
36
|
+
from copilot.generated.session_events import SessionEvent
|
|
37
|
+
from copilot.session import CopilotSession
|
|
38
|
+
|
|
39
|
+
logger = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def run_copilot(agent: CopilotTaskConfig, prompt: str) -> CopilotResult:
|
|
43
|
+
"""Execute a prompt against GitHub Copilot and return structured results.
|
|
44
|
+
|
|
45
|
+
This is the primary entry point for test execution. It manages the full
|
|
46
|
+
lifecycle: client start → session creation → prompt execution → event
|
|
47
|
+
capture → client cleanup.
|
|
48
|
+
|
|
49
|
+
Each call makes one execution attempt. Failures retain their captured
|
|
50
|
+
evidence and are returned without starting a replacement session.
|
|
51
|
+
|
|
52
|
+
Authentication is resolved in this order:
|
|
53
|
+
1. ``GITHUB_TOKEN`` environment variable (ideal for CI)
|
|
54
|
+
2. ``GH_TOKEN`` environment variable
|
|
55
|
+
3. Logged-in user via ``gh`` CLI / OAuth (local development)
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
agent: CopilotTask configuration.
|
|
59
|
+
prompt: The prompt to send to Copilot.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
CopilotResult with all captured events, tool calls, usage, etc.
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
mapper = EventMapper()
|
|
66
|
+
controls = RunControls(agent.max_tool_calls)
|
|
67
|
+
capture_requests = agent.audit_requests or agent.image_detail is not None
|
|
68
|
+
audit = (
|
|
69
|
+
RequestAuditHandler(
|
|
70
|
+
image_detail=agent.image_detail,
|
|
71
|
+
audit=agent.audit_requests,
|
|
72
|
+
inspect_requests=capture_requests,
|
|
73
|
+
)
|
|
74
|
+
if capture_requests or agent.max_tool_calls is not None
|
|
75
|
+
else None
|
|
76
|
+
)
|
|
77
|
+
storage: TemporaryDirectory[str] | None = None
|
|
78
|
+
client: CopilotClient | None = None
|
|
79
|
+
session: CopilotSession | None = None
|
|
80
|
+
execution: asyncio.Task[None] | None = None
|
|
81
|
+
budget_abort: asyncio.Task[None] | None = None
|
|
82
|
+
watchers: list[asyncio.Task[bool]] = []
|
|
83
|
+
reason: StopReason = "completed"
|
|
84
|
+
error: str | None = None
|
|
85
|
+
cleanup_errors: list[str] = []
|
|
86
|
+
skill_discovery: SkillDiscovery | None = None
|
|
87
|
+
configuration: dict[str, Any] | None = None
|
|
88
|
+
send_started = False
|
|
89
|
+
try:
|
|
90
|
+
session_config = agent.build_session_config()
|
|
91
|
+
skill_directories = session_config.get("skill_directories") or []
|
|
92
|
+
requested_skills: dict[Path, str] = {}
|
|
93
|
+
if skill_directories:
|
|
94
|
+
skill_discovery = SkillDiscovery()
|
|
95
|
+
if not isinstance(skill_directories, list):
|
|
96
|
+
raise ValueError("skill_directories must be a list of directory paths")
|
|
97
|
+
requested_skills = requested_skill_files(skill_directories)
|
|
98
|
+
session_config["skill_directories"] = [
|
|
99
|
+
str(Path(directory).resolve()) for directory in skill_directories
|
|
100
|
+
]
|
|
101
|
+
caller_hooks = dict(session_config.get("hooks") or {})
|
|
102
|
+
session_config["hooks"] = dict(caller_hooks)
|
|
103
|
+
# Empty mode must not read persona instruction files or inject tools.
|
|
104
|
+
if agent.client_mode != "empty":
|
|
105
|
+
agent.persona.apply(agent, session_config, mapper, run_copilot)
|
|
106
|
+
controls.install(session_config, caller_hooks)
|
|
107
|
+
if agent.auto_confirm and "on_permission_request" not in session_config:
|
|
108
|
+
session_config["on_permission_request"] = approve_all_permissions
|
|
109
|
+
from pytest_copilot.copilot.config import snapshot_session_configuration
|
|
110
|
+
|
|
111
|
+
configuration = snapshot_session_configuration(agent, session_config)
|
|
112
|
+
|
|
113
|
+
caller_event = session_config.get("on_event")
|
|
114
|
+
|
|
115
|
+
def capture(event: SessionEvent) -> None:
|
|
116
|
+
nonlocal budget_abort
|
|
117
|
+
mapper.handle(event)
|
|
118
|
+
if controls.observe(event) and session is not None:
|
|
119
|
+
if audit is not None:
|
|
120
|
+
audit.stop_forwarding()
|
|
121
|
+
logger.info(
|
|
122
|
+
"Aborting Copilot session: phase=tool_budget_exceeded admitted=%d "
|
|
123
|
+
"started=%d completed=%d handlers_started=%d handlers_finished=%d",
|
|
124
|
+
controls.admitted,
|
|
125
|
+
controls.started,
|
|
126
|
+
controls.completed,
|
|
127
|
+
controls.handlers_started,
|
|
128
|
+
controls.handlers_finished,
|
|
129
|
+
)
|
|
130
|
+
budget_abort = asyncio.create_task(session.abort())
|
|
131
|
+
if caller_event is not None:
|
|
132
|
+
caller_event(event)
|
|
133
|
+
|
|
134
|
+
session_config["on_event"] = capture
|
|
135
|
+
if agent.client_mode == "empty":
|
|
136
|
+
storage = TemporaryDirectory(prefix="pytest-copilot-")
|
|
137
|
+
client = create_client(
|
|
138
|
+
agent.working_directory or ".",
|
|
139
|
+
mode=agent.client_mode,
|
|
140
|
+
base_directory=storage.name if storage else None,
|
|
141
|
+
request_handler=audit,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
async def execute() -> None:
|
|
145
|
+
nonlocal session, send_started
|
|
146
|
+
assert client is not None
|
|
147
|
+
async with asyncio.timeout(min(60, agent.timeout_s)):
|
|
148
|
+
await client.start()
|
|
149
|
+
async with asyncio.timeout(min(30, agent.timeout_s)):
|
|
150
|
+
session = await client.create_session(**session_config)
|
|
151
|
+
if skill_discovery is not None:
|
|
152
|
+
await discover_requested_skills(
|
|
153
|
+
session,
|
|
154
|
+
requested_skills,
|
|
155
|
+
session_config.get("disabled_skills") or [],
|
|
156
|
+
skill_discovery,
|
|
157
|
+
)
|
|
158
|
+
send_started = True
|
|
159
|
+
await session.send_and_wait(prompt, timeout=agent.timeout_s)
|
|
160
|
+
|
|
161
|
+
execution = asyncio.create_task(execute())
|
|
162
|
+
watchers = [
|
|
163
|
+
asyncio.create_task(controls.budget_exceeded.wait()),
|
|
164
|
+
asyncio.create_task(controls.failed.wait()),
|
|
165
|
+
]
|
|
166
|
+
if audit is not None:
|
|
167
|
+
watchers.append(asyncio.create_task(audit.failed.wait()))
|
|
168
|
+
done, _ = await asyncio.wait(
|
|
169
|
+
[execution, *watchers], timeout=agent.timeout_s, return_when=asyncio.FIRST_COMPLETED
|
|
170
|
+
)
|
|
171
|
+
if audit is not None and audit.failed.is_set():
|
|
172
|
+
reason, error = "request_audit_error", audit.error
|
|
173
|
+
elif controls.failed.is_set():
|
|
174
|
+
reason, error = "execution_error", controls.error
|
|
175
|
+
elif controls.budget_exceeded.is_set():
|
|
176
|
+
reason = "tool_budget_exceeded"
|
|
177
|
+
error = f"Tool-call budget exhausted ({agent.max_tool_calls})"
|
|
178
|
+
elif execution in done:
|
|
179
|
+
await execution
|
|
180
|
+
else:
|
|
181
|
+
raise TimeoutError
|
|
182
|
+
|
|
183
|
+
except TimeoutError:
|
|
184
|
+
reason, error = "timeout", f"Timeout (eval limit {agent.timeout_s}s)"
|
|
185
|
+
except Exception as exc:
|
|
186
|
+
logger.error("Copilot execution failed: %s", exc)
|
|
187
|
+
reason, error = "execution_error", str(exc)
|
|
188
|
+
if skill_discovery is not None and not skill_discovery.complete:
|
|
189
|
+
error = f"Skill setup failed before model execution: {exc}"
|
|
190
|
+
skill_discovery.errors.append(error)
|
|
191
|
+
finally:
|
|
192
|
+
controls.closed = True
|
|
193
|
+
for watcher in watchers:
|
|
194
|
+
watcher.cancel()
|
|
195
|
+
if watchers:
|
|
196
|
+
await asyncio.gather(*watchers, return_exceptions=True)
|
|
197
|
+
if budget_abort is not None:
|
|
198
|
+
try:
|
|
199
|
+
await asyncio.wait_for(budget_abort, timeout=30)
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
logger.error("Failed to abort Copilot session at budget barrier", exc_info=True)
|
|
202
|
+
cleanup_errors.append(f"Session abort failed: {type(exc).__name__}")
|
|
203
|
+
elif session is not None and (
|
|
204
|
+
error is not None or execution is None or not execution.done()
|
|
205
|
+
):
|
|
206
|
+
logger.info(
|
|
207
|
+
"Aborting Copilot session: phase=%s admitted=%d started=%d completed=%d "
|
|
208
|
+
"handlers_started=%d handlers_finished=%d",
|
|
209
|
+
reason,
|
|
210
|
+
controls.admitted,
|
|
211
|
+
controls.started,
|
|
212
|
+
controls.completed,
|
|
213
|
+
controls.handlers_started,
|
|
214
|
+
controls.handlers_finished,
|
|
215
|
+
)
|
|
216
|
+
try:
|
|
217
|
+
await asyncio.wait_for(session.abort(), timeout=30)
|
|
218
|
+
except Exception as exc:
|
|
219
|
+
logger.error("Failed to abort Copilot session", exc_info=True)
|
|
220
|
+
cleanup_errors.append(f"Session abort failed: {type(exc).__name__}")
|
|
221
|
+
if execution is not None:
|
|
222
|
+
if not execution.done():
|
|
223
|
+
execution.cancel()
|
|
224
|
+
await asyncio.gather(execution, return_exceptions=True)
|
|
225
|
+
cleanup_errors.extend(await controls.drain())
|
|
226
|
+
if audit is not None:
|
|
227
|
+
try:
|
|
228
|
+
await audit.aclose()
|
|
229
|
+
except Exception as exc:
|
|
230
|
+
logger.error("Failed to close model request transport", exc_info=True)
|
|
231
|
+
cleanup_errors.append(f"Request transport cleanup failed: {type(exc).__name__}")
|
|
232
|
+
if client is not None:
|
|
233
|
+
cleanup_errors.extend(await stop_client(client))
|
|
234
|
+
if storage is not None:
|
|
235
|
+
try:
|
|
236
|
+
storage.cleanup()
|
|
237
|
+
except OSError as exc:
|
|
238
|
+
logger.error("Failed to remove isolated runtime storage", exc_info=True)
|
|
239
|
+
cleanup_errors.append(f"Isolated storage cleanup failed: {type(exc).__name__}")
|
|
240
|
+
|
|
241
|
+
result = mapper.build()
|
|
242
|
+
result.agent = agent
|
|
243
|
+
result.skill_discovery = skill_discovery
|
|
244
|
+
result.configuration = configuration
|
|
245
|
+
result.tool_calls_admitted = controls.admitted
|
|
246
|
+
if result.capture_errors:
|
|
247
|
+
result.capture_errors.extend(controls.incomplete_diagnostics(abort_phase=reason))
|
|
248
|
+
if audit is not None and capture_requests:
|
|
249
|
+
result.request_audit = list(audit.records)
|
|
250
|
+
if audit.error or (send_started and not audit.observed_requests):
|
|
251
|
+
audit_error = audit.error or "No outbound model requests captured; audit unsupported"
|
|
252
|
+
result.capture_errors.append(audit_error)
|
|
253
|
+
if reason == "completed" or audit.error:
|
|
254
|
+
reason, error = "request_audit_error", audit_error
|
|
255
|
+
if cleanup_errors:
|
|
256
|
+
result.capture_errors.extend(cleanup_errors)
|
|
257
|
+
reason = "cleanup_error"
|
|
258
|
+
errors = [message for message in [result.error, error, *cleanup_errors] if message]
|
|
259
|
+
if result.capture_errors:
|
|
260
|
+
errors.extend(message for message in result.capture_errors if message not in errors)
|
|
261
|
+
result.error = "; ".join(errors) or None
|
|
262
|
+
result.success = result.error is None
|
|
263
|
+
result.stop_reason = (
|
|
264
|
+
"execution_error" if reason == "completed" and not result.success else reason
|
|
265
|
+
)
|
|
266
|
+
return result
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Validate explicit skill inputs and verify native SDK availability before sending."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
from pytest_copilot.core.result import DiscoveredSkill, SkillDiscovery
|
|
10
|
+
from pytest_copilot.core.skill import Skill, SkillError
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from copilot.session import CopilotSession
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def requested_skill_files(directories: list[str]) -> dict[Path, str]:
|
|
19
|
+
"""Validate individual skill directories or parents of immediate skill directories.
|
|
20
|
+
|
|
21
|
+
Relative directories are resolved from the Python caller's current directory,
|
|
22
|
+
then passed to the SDK as absolute paths.
|
|
23
|
+
"""
|
|
24
|
+
requested: dict[Path, str] = {}
|
|
25
|
+
names: dict[str, Path] = {}
|
|
26
|
+
for raw_directory in directories:
|
|
27
|
+
if not isinstance(raw_directory, str) or not raw_directory.strip():
|
|
28
|
+
raise SkillError("skill_directories entries must be non-empty directory paths")
|
|
29
|
+
directory = Path(raw_directory).resolve()
|
|
30
|
+
if not directory.is_dir():
|
|
31
|
+
raise SkillError(
|
|
32
|
+
f"Requested skill directory does not exist or is not a directory: {directory}"
|
|
33
|
+
)
|
|
34
|
+
try:
|
|
35
|
+
candidates = (
|
|
36
|
+
[directory]
|
|
37
|
+
if (directory / "SKILL.md").exists()
|
|
38
|
+
else [
|
|
39
|
+
child
|
|
40
|
+
for child in sorted(directory.iterdir())
|
|
41
|
+
if child.is_dir() and (child / "SKILL.md").exists()
|
|
42
|
+
]
|
|
43
|
+
)
|
|
44
|
+
if not candidates:
|
|
45
|
+
raise SkillError(f"No SKILL.md found in requested skill directory: {directory}")
|
|
46
|
+
for candidate in candidates:
|
|
47
|
+
skill = Skill.from_path(candidate)
|
|
48
|
+
path = (skill.path / "SKILL.md").resolve()
|
|
49
|
+
if skill.name in names and names[skill.name] != path:
|
|
50
|
+
raise SkillError(
|
|
51
|
+
f"Duplicate requested skill name '{skill.name}': "
|
|
52
|
+
f"{names[skill.name]} and {path}"
|
|
53
|
+
)
|
|
54
|
+
names[skill.name] = path
|
|
55
|
+
requested[path] = skill.name
|
|
56
|
+
except OSError as exc:
|
|
57
|
+
raise SkillError(f"Cannot read requested skill directory {directory}: {exc}") from exc
|
|
58
|
+
return requested
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def discover_requested_skills(
|
|
62
|
+
session: CopilotSession,
|
|
63
|
+
requested: dict[Path, str],
|
|
64
|
+
disabled: list[str],
|
|
65
|
+
evidence: SkillDiscovery,
|
|
66
|
+
) -> None:
|
|
67
|
+
"""Record SDK discovery and reject missing or unexpectedly disabled requested skills."""
|
|
68
|
+
await session.rpc.skills.ensure_loaded()
|
|
69
|
+
diagnostics = await session.rpc.skills.reload()
|
|
70
|
+
evidence.warnings = list(diagnostics.warnings)
|
|
71
|
+
evidence.errors = list(diagnostics.errors)
|
|
72
|
+
for warning in evidence.warnings:
|
|
73
|
+
logger.warning("SDK skill discovery: %s", warning)
|
|
74
|
+
listed = await session.rpc.skills.list()
|
|
75
|
+
evidence.skills = [
|
|
76
|
+
DiscoveredSkill(
|
|
77
|
+
name=skill.name,
|
|
78
|
+
description=skill.description,
|
|
79
|
+
source=skill.source.value,
|
|
80
|
+
enabled=skill.enabled,
|
|
81
|
+
user_invocable=skill.user_invocable,
|
|
82
|
+
path=skill.path,
|
|
83
|
+
plugin_name=skill.plugin_name,
|
|
84
|
+
command_name=skill.command_name,
|
|
85
|
+
argument_hint=skill.argument_hint,
|
|
86
|
+
)
|
|
87
|
+
for skill in listed.skills
|
|
88
|
+
]
|
|
89
|
+
evidence.complete = True
|
|
90
|
+
available = {
|
|
91
|
+
(Path(skill.path).resolve(), skill.name): skill
|
|
92
|
+
for skill in evidence.skills
|
|
93
|
+
if skill.path is not None
|
|
94
|
+
}
|
|
95
|
+
for path, name in requested.items():
|
|
96
|
+
skill = available.get((path, name))
|
|
97
|
+
if skill is None:
|
|
98
|
+
evidence.errors.append(
|
|
99
|
+
f"Requested skill '{name}' was not discovered by the SDK: {path}"
|
|
100
|
+
)
|
|
101
|
+
elif not skill.enabled and name not in disabled:
|
|
102
|
+
evidence.errors.append(f"Requested skill '{name}' is unexpectedly disabled: {path}")
|
|
103
|
+
if evidence.errors:
|
|
104
|
+
raise SkillError("Skill setup failed before model execution: " + "; ".join(evidence.errors))
|