pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,266 @@
1
+ """CopilotRunner — Execute prompts against GitHub Copilot directly via SDK.
2
+
3
+ This is the core execution engine. It:
4
+ 1. Creates a CopilotClient and starts the CLI server
5
+ 2. Creates a session with the agent's config
6
+ 3. Sends the prompt and captures ALL events
7
+ 4. Maps events to a CopilotResult via EventMapper
8
+ 5. Cleans up the client
9
+
10
+ No LiteLLM. No outer agent. One LLM. Direct SDK access.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import logging
17
+ from pathlib import Path
18
+ from tempfile import TemporaryDirectory
19
+ from typing import TYPE_CHECKING, Any
20
+
21
+ from pytest_copilot.copilot.client import (
22
+ approve_all_permissions,
23
+ create_client,
24
+ stop_client,
25
+ )
26
+ from pytest_copilot.copilot.contracts import CopilotTaskConfig
27
+ from pytest_copilot.copilot.controls import RunControls
28
+ from pytest_copilot.copilot.events import EventMapper
29
+ from pytest_copilot.copilot.requests import RequestAuditHandler
30
+ from pytest_copilot.copilot.result import CopilotResult, StopReason
31
+ from pytest_copilot.copilot.skills import discover_requested_skills, requested_skill_files
32
+ from pytest_copilot.core.result import SkillDiscovery
33
+
34
+ if TYPE_CHECKING:
35
+ from copilot.client import CopilotClient
36
+ from copilot.generated.session_events import SessionEvent
37
+ from copilot.session import CopilotSession
38
+
39
+ logger = logging.getLogger(__name__)
40
+
41
+
42
+ async def run_copilot(agent: CopilotTaskConfig, prompt: str) -> CopilotResult:
43
+ """Execute a prompt against GitHub Copilot and return structured results.
44
+
45
+ This is the primary entry point for test execution. It manages the full
46
+ lifecycle: client start → session creation → prompt execution → event
47
+ capture → client cleanup.
48
+
49
+ Each call makes one execution attempt. Failures retain their captured
50
+ evidence and are returned without starting a replacement session.
51
+
52
+ Authentication is resolved in this order:
53
+ 1. ``GITHUB_TOKEN`` environment variable (ideal for CI)
54
+ 2. ``GH_TOKEN`` environment variable
55
+ 3. Logged-in user via ``gh`` CLI / OAuth (local development)
56
+
57
+ Args:
58
+ agent: CopilotTask configuration.
59
+ prompt: The prompt to send to Copilot.
60
+
61
+ Returns:
62
+ CopilotResult with all captured events, tool calls, usage, etc.
63
+
64
+ """
65
+ mapper = EventMapper()
66
+ controls = RunControls(agent.max_tool_calls)
67
+ capture_requests = agent.audit_requests or agent.image_detail is not None
68
+ audit = (
69
+ RequestAuditHandler(
70
+ image_detail=agent.image_detail,
71
+ audit=agent.audit_requests,
72
+ inspect_requests=capture_requests,
73
+ )
74
+ if capture_requests or agent.max_tool_calls is not None
75
+ else None
76
+ )
77
+ storage: TemporaryDirectory[str] | None = None
78
+ client: CopilotClient | None = None
79
+ session: CopilotSession | None = None
80
+ execution: asyncio.Task[None] | None = None
81
+ budget_abort: asyncio.Task[None] | None = None
82
+ watchers: list[asyncio.Task[bool]] = []
83
+ reason: StopReason = "completed"
84
+ error: str | None = None
85
+ cleanup_errors: list[str] = []
86
+ skill_discovery: SkillDiscovery | None = None
87
+ configuration: dict[str, Any] | None = None
88
+ send_started = False
89
+ try:
90
+ session_config = agent.build_session_config()
91
+ skill_directories = session_config.get("skill_directories") or []
92
+ requested_skills: dict[Path, str] = {}
93
+ if skill_directories:
94
+ skill_discovery = SkillDiscovery()
95
+ if not isinstance(skill_directories, list):
96
+ raise ValueError("skill_directories must be a list of directory paths")
97
+ requested_skills = requested_skill_files(skill_directories)
98
+ session_config["skill_directories"] = [
99
+ str(Path(directory).resolve()) for directory in skill_directories
100
+ ]
101
+ caller_hooks = dict(session_config.get("hooks") or {})
102
+ session_config["hooks"] = dict(caller_hooks)
103
+ # Empty mode must not read persona instruction files or inject tools.
104
+ if agent.client_mode != "empty":
105
+ agent.persona.apply(agent, session_config, mapper, run_copilot)
106
+ controls.install(session_config, caller_hooks)
107
+ if agent.auto_confirm and "on_permission_request" not in session_config:
108
+ session_config["on_permission_request"] = approve_all_permissions
109
+ from pytest_copilot.copilot.config import snapshot_session_configuration
110
+
111
+ configuration = snapshot_session_configuration(agent, session_config)
112
+
113
+ caller_event = session_config.get("on_event")
114
+
115
+ def capture(event: SessionEvent) -> None:
116
+ nonlocal budget_abort
117
+ mapper.handle(event)
118
+ if controls.observe(event) and session is not None:
119
+ if audit is not None:
120
+ audit.stop_forwarding()
121
+ logger.info(
122
+ "Aborting Copilot session: phase=tool_budget_exceeded admitted=%d "
123
+ "started=%d completed=%d handlers_started=%d handlers_finished=%d",
124
+ controls.admitted,
125
+ controls.started,
126
+ controls.completed,
127
+ controls.handlers_started,
128
+ controls.handlers_finished,
129
+ )
130
+ budget_abort = asyncio.create_task(session.abort())
131
+ if caller_event is not None:
132
+ caller_event(event)
133
+
134
+ session_config["on_event"] = capture
135
+ if agent.client_mode == "empty":
136
+ storage = TemporaryDirectory(prefix="pytest-copilot-")
137
+ client = create_client(
138
+ agent.working_directory or ".",
139
+ mode=agent.client_mode,
140
+ base_directory=storage.name if storage else None,
141
+ request_handler=audit,
142
+ )
143
+
144
+ async def execute() -> None:
145
+ nonlocal session, send_started
146
+ assert client is not None
147
+ async with asyncio.timeout(min(60, agent.timeout_s)):
148
+ await client.start()
149
+ async with asyncio.timeout(min(30, agent.timeout_s)):
150
+ session = await client.create_session(**session_config)
151
+ if skill_discovery is not None:
152
+ await discover_requested_skills(
153
+ session,
154
+ requested_skills,
155
+ session_config.get("disabled_skills") or [],
156
+ skill_discovery,
157
+ )
158
+ send_started = True
159
+ await session.send_and_wait(prompt, timeout=agent.timeout_s)
160
+
161
+ execution = asyncio.create_task(execute())
162
+ watchers = [
163
+ asyncio.create_task(controls.budget_exceeded.wait()),
164
+ asyncio.create_task(controls.failed.wait()),
165
+ ]
166
+ if audit is not None:
167
+ watchers.append(asyncio.create_task(audit.failed.wait()))
168
+ done, _ = await asyncio.wait(
169
+ [execution, *watchers], timeout=agent.timeout_s, return_when=asyncio.FIRST_COMPLETED
170
+ )
171
+ if audit is not None and audit.failed.is_set():
172
+ reason, error = "request_audit_error", audit.error
173
+ elif controls.failed.is_set():
174
+ reason, error = "execution_error", controls.error
175
+ elif controls.budget_exceeded.is_set():
176
+ reason = "tool_budget_exceeded"
177
+ error = f"Tool-call budget exhausted ({agent.max_tool_calls})"
178
+ elif execution in done:
179
+ await execution
180
+ else:
181
+ raise TimeoutError
182
+
183
+ except TimeoutError:
184
+ reason, error = "timeout", f"Timeout (eval limit {agent.timeout_s}s)"
185
+ except Exception as exc:
186
+ logger.error("Copilot execution failed: %s", exc)
187
+ reason, error = "execution_error", str(exc)
188
+ if skill_discovery is not None and not skill_discovery.complete:
189
+ error = f"Skill setup failed before model execution: {exc}"
190
+ skill_discovery.errors.append(error)
191
+ finally:
192
+ controls.closed = True
193
+ for watcher in watchers:
194
+ watcher.cancel()
195
+ if watchers:
196
+ await asyncio.gather(*watchers, return_exceptions=True)
197
+ if budget_abort is not None:
198
+ try:
199
+ await asyncio.wait_for(budget_abort, timeout=30)
200
+ except Exception as exc:
201
+ logger.error("Failed to abort Copilot session at budget barrier", exc_info=True)
202
+ cleanup_errors.append(f"Session abort failed: {type(exc).__name__}")
203
+ elif session is not None and (
204
+ error is not None or execution is None or not execution.done()
205
+ ):
206
+ logger.info(
207
+ "Aborting Copilot session: phase=%s admitted=%d started=%d completed=%d "
208
+ "handlers_started=%d handlers_finished=%d",
209
+ reason,
210
+ controls.admitted,
211
+ controls.started,
212
+ controls.completed,
213
+ controls.handlers_started,
214
+ controls.handlers_finished,
215
+ )
216
+ try:
217
+ await asyncio.wait_for(session.abort(), timeout=30)
218
+ except Exception as exc:
219
+ logger.error("Failed to abort Copilot session", exc_info=True)
220
+ cleanup_errors.append(f"Session abort failed: {type(exc).__name__}")
221
+ if execution is not None:
222
+ if not execution.done():
223
+ execution.cancel()
224
+ await asyncio.gather(execution, return_exceptions=True)
225
+ cleanup_errors.extend(await controls.drain())
226
+ if audit is not None:
227
+ try:
228
+ await audit.aclose()
229
+ except Exception as exc:
230
+ logger.error("Failed to close model request transport", exc_info=True)
231
+ cleanup_errors.append(f"Request transport cleanup failed: {type(exc).__name__}")
232
+ if client is not None:
233
+ cleanup_errors.extend(await stop_client(client))
234
+ if storage is not None:
235
+ try:
236
+ storage.cleanup()
237
+ except OSError as exc:
238
+ logger.error("Failed to remove isolated runtime storage", exc_info=True)
239
+ cleanup_errors.append(f"Isolated storage cleanup failed: {type(exc).__name__}")
240
+
241
+ result = mapper.build()
242
+ result.agent = agent
243
+ result.skill_discovery = skill_discovery
244
+ result.configuration = configuration
245
+ result.tool_calls_admitted = controls.admitted
246
+ if result.capture_errors:
247
+ result.capture_errors.extend(controls.incomplete_diagnostics(abort_phase=reason))
248
+ if audit is not None and capture_requests:
249
+ result.request_audit = list(audit.records)
250
+ if audit.error or (send_started and not audit.observed_requests):
251
+ audit_error = audit.error or "No outbound model requests captured; audit unsupported"
252
+ result.capture_errors.append(audit_error)
253
+ if reason == "completed" or audit.error:
254
+ reason, error = "request_audit_error", audit_error
255
+ if cleanup_errors:
256
+ result.capture_errors.extend(cleanup_errors)
257
+ reason = "cleanup_error"
258
+ errors = [message for message in [result.error, error, *cleanup_errors] if message]
259
+ if result.capture_errors:
260
+ errors.extend(message for message in result.capture_errors if message not in errors)
261
+ result.error = "; ".join(errors) or None
262
+ result.success = result.error is None
263
+ result.stop_reason = (
264
+ "execution_error" if reason == "completed" and not result.success else reason
265
+ )
266
+ return result
@@ -0,0 +1,104 @@
1
+ """Validate explicit skill inputs and verify native SDK availability before sending."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from pathlib import Path
7
+ from typing import TYPE_CHECKING
8
+
9
+ from pytest_copilot.core.result import DiscoveredSkill, SkillDiscovery
10
+ from pytest_copilot.core.skill import Skill, SkillError
11
+
12
+ if TYPE_CHECKING:
13
+ from copilot.session import CopilotSession
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ def requested_skill_files(directories: list[str]) -> dict[Path, str]:
19
+ """Validate individual skill directories or parents of immediate skill directories.
20
+
21
+ Relative directories are resolved from the Python caller's current directory,
22
+ then passed to the SDK as absolute paths.
23
+ """
24
+ requested: dict[Path, str] = {}
25
+ names: dict[str, Path] = {}
26
+ for raw_directory in directories:
27
+ if not isinstance(raw_directory, str) or not raw_directory.strip():
28
+ raise SkillError("skill_directories entries must be non-empty directory paths")
29
+ directory = Path(raw_directory).resolve()
30
+ if not directory.is_dir():
31
+ raise SkillError(
32
+ f"Requested skill directory does not exist or is not a directory: {directory}"
33
+ )
34
+ try:
35
+ candidates = (
36
+ [directory]
37
+ if (directory / "SKILL.md").exists()
38
+ else [
39
+ child
40
+ for child in sorted(directory.iterdir())
41
+ if child.is_dir() and (child / "SKILL.md").exists()
42
+ ]
43
+ )
44
+ if not candidates:
45
+ raise SkillError(f"No SKILL.md found in requested skill directory: {directory}")
46
+ for candidate in candidates:
47
+ skill = Skill.from_path(candidate)
48
+ path = (skill.path / "SKILL.md").resolve()
49
+ if skill.name in names and names[skill.name] != path:
50
+ raise SkillError(
51
+ f"Duplicate requested skill name '{skill.name}': "
52
+ f"{names[skill.name]} and {path}"
53
+ )
54
+ names[skill.name] = path
55
+ requested[path] = skill.name
56
+ except OSError as exc:
57
+ raise SkillError(f"Cannot read requested skill directory {directory}: {exc}") from exc
58
+ return requested
59
+
60
+
61
+ async def discover_requested_skills(
62
+ session: CopilotSession,
63
+ requested: dict[Path, str],
64
+ disabled: list[str],
65
+ evidence: SkillDiscovery,
66
+ ) -> None:
67
+ """Record SDK discovery and reject missing or unexpectedly disabled requested skills."""
68
+ await session.rpc.skills.ensure_loaded()
69
+ diagnostics = await session.rpc.skills.reload()
70
+ evidence.warnings = list(diagnostics.warnings)
71
+ evidence.errors = list(diagnostics.errors)
72
+ for warning in evidence.warnings:
73
+ logger.warning("SDK skill discovery: %s", warning)
74
+ listed = await session.rpc.skills.list()
75
+ evidence.skills = [
76
+ DiscoveredSkill(
77
+ name=skill.name,
78
+ description=skill.description,
79
+ source=skill.source.value,
80
+ enabled=skill.enabled,
81
+ user_invocable=skill.user_invocable,
82
+ path=skill.path,
83
+ plugin_name=skill.plugin_name,
84
+ command_name=skill.command_name,
85
+ argument_hint=skill.argument_hint,
86
+ )
87
+ for skill in listed.skills
88
+ ]
89
+ evidence.complete = True
90
+ available = {
91
+ (Path(skill.path).resolve(), skill.name): skill
92
+ for skill in evidence.skills
93
+ if skill.path is not None
94
+ }
95
+ for path, name in requested.items():
96
+ skill = available.get((path, name))
97
+ if skill is None:
98
+ evidence.errors.append(
99
+ f"Requested skill '{name}' was not discovered by the SDK: {path}"
100
+ )
101
+ elif not skill.enabled and name not in disabled:
102
+ evidence.errors.append(f"Requested skill '{name}' is unexpectedly disabled: {path}")
103
+ if evidence.errors:
104
+ raise SkillError("Skill setup failed before model execution: " + "; ".join(evidence.errors))