pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,289 @@
1
+ """Pytest fixtures for GitHub Copilot testing.
2
+
3
+ Provides the ``copilot_run`` fixture that executes prompts against Copilot
4
+ and stashes results for pytest-copilot reporting.
5
+
6
+ Also provides ``ab_run``, a higher-level fixture for A/B testing two agent
7
+ configurations against the same task in isolated directories.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import dataclasses
13
+ from copy import deepcopy
14
+ from pathlib import Path
15
+ from typing import TYPE_CHECKING, Any, Literal
16
+
17
+ import pytest
18
+
19
+ from pytest_copilot.copilot.api import run_copilot
20
+
21
+ if TYPE_CHECKING:
22
+ from collections.abc import Callable, Coroutine
23
+
24
+ from _pytest.nodes import Item
25
+
26
+ from pytest_copilot.copilot.result import CopilotResult
27
+ from pytest_copilot.copilot.task import CopilotTask
28
+
29
+
30
+ @pytest.fixture
31
+ def copilot_run(
32
+ request: pytest.FixtureRequest,
33
+ ) -> Callable[..., Coroutine[Any, Any, CopilotResult]]:
34
+ """Execute a prompt against a CopilotTask and capture results.
35
+
36
+ Results are automatically stashed on the test node for pytest-copilot's
37
+ evidence collector. Native JSON preserves execution alongside ordinary pytest
38
+ outcomes; the coding agent interprets it together with source and test criteria.
39
+
40
+ Example:
41
+ async def test_file_creation(copilot_run, tmp_path):
42
+ agent = CopilotTask(
43
+ instructions="Create files as requested.",
44
+ working_directory=str(tmp_path),
45
+ )
46
+ result = await copilot_run(agent, "Create hello.py with print('hello')")
47
+ assert result.success
48
+ assert result.tool_was_called("create_file")
49
+ """
50
+
51
+ async def _run(agent: CopilotTask, prompt: str) -> CopilotResult:
52
+ result = await run_copilot(agent, prompt)
53
+
54
+ # Stash for pytest-copilot's reporting plugin.
55
+ # The plugin hook also does this automatically for tests that
56
+ # call run_copilot() directly, but explicit stashing from the
57
+ # fixture ensures it works even if the hook order changes.
58
+ stash_on_item(request.node, agent, result)
59
+
60
+ return result
61
+
62
+ return _run
63
+
64
+
65
+ def _convert_to_aitest(
66
+ agent: CopilotTask,
67
+ result: CopilotResult,
68
+ ) -> tuple[Any, Any] | None:
69
+ """Convert CopilotResult to pytest-copilot types.
70
+
71
+ Returns ``(EvalResult, agent_wrapper)`` tuple, or ``None`` if conversion
72
+ fails.
73
+
74
+ Since CopilotResult already uses pytest-copilot's Turn and ToolCall types,
75
+ the turns can be passed through directly without rebuilding.
76
+
77
+ The agent_wrapper is a simple object with the fields expected by plugin.py:
78
+ - name
79
+ - provider.model
80
+ - system_prompt_name (optional)
81
+ - mcp_servers (optional)
82
+ - allowed_tools (optional)
83
+ """
84
+ from dataclasses import dataclass
85
+
86
+ from pytest_copilot.core.result import EvalResult, SubagentInvocation
87
+ from pytest_copilot.execution.cost import estimate_cost
88
+
89
+ # Estimate USD cost from captured token usage and pricing.toml.
90
+ # Models without pricing contribute 0.0 and are recorded in
91
+ # execution.cost.models_without_pricing (saved with report evidence).
92
+ cost_usd = sum(
93
+ estimate_cost(
94
+ usage.model or result.model_used or "",
95
+ usage.input_tokens,
96
+ usage.output_tokens,
97
+ usage.cache_read_tokens or 0,
98
+ )
99
+ for usage in result.all_usage
100
+ if usage.input_tokens is not None and usage.output_tokens is not None
101
+ )
102
+
103
+ # Turns already use aitest's Turn/ToolCall types — pass through directly
104
+ aitest_result = EvalResult(
105
+ turns=list(result.turns),
106
+ success=result.success,
107
+ error=result.error,
108
+ duration_ms=result.duration_ms,
109
+ token_usage=result.token_usage,
110
+ cost_usd=cost_usd,
111
+ effective_system_prompt=(
112
+ result.configuration.get("instructions") if result.configuration is not None else None
113
+ ),
114
+ premium_requests=result.total_premium_requests,
115
+ evidence_complete=result.evidence_complete,
116
+ capture_errors=list(result.capture_errors),
117
+ request_audit=list(result.request_audit),
118
+ stop_reason=result.stop_reason,
119
+ usage=list(result.usage),
120
+ tool_calls_admitted=result.tool_calls_admitted,
121
+ skill_discovery=deepcopy(result.skill_discovery),
122
+ configuration=deepcopy(result.configuration),
123
+ model_used=result.model_used,
124
+ reasoning_traces=list(result.reasoning_traces),
125
+ permission_requested=result.permission_requested,
126
+ permissions=deepcopy(result.permissions),
127
+ )
128
+ for invocation in result.subagent_invocations:
129
+ child_result = None
130
+ if invocation.result is not None:
131
+ from pytest_copilot.copilot.task import CopilotTask
132
+
133
+ child_agent = invocation.result.agent
134
+ if not isinstance(child_agent, CopilotTask):
135
+ raise ValueError(f"Child invocation {invocation.invocation_id!r} has no eval")
136
+ converted_child = _convert_to_aitest(child_agent, invocation.result)
137
+ assert converted_child is not None
138
+ child_result = converted_child[0]
139
+ aitest_result.subagent_invocations.append(
140
+ SubagentInvocation(
141
+ invocation_id=invocation.invocation_id,
142
+ name=invocation.name,
143
+ status=invocation.status,
144
+ duration_ms=invocation.duration_ms,
145
+ result=child_result,
146
+ )
147
+ )
148
+
149
+ # Create a minimal wrapper with just the fields needed by plugin.py
150
+ @dataclass(slots=True)
151
+ class Provider:
152
+ model: str | None
153
+
154
+ @dataclass(slots=True)
155
+ class AgentWrapper:
156
+ name: str
157
+ id: str
158
+ provider: Provider
159
+ system_prompt_name: str | None = None
160
+ mcp_servers: list[Any] = None # type: ignore[assignment]
161
+ allowed_tools: list[str] | None = None
162
+ skill: Any = None
163
+
164
+ def __post_init__(self) -> None:
165
+ if self.mcp_servers is None:
166
+ self.mcp_servers = []
167
+
168
+ aitest_agent = AgentWrapper(
169
+ name=agent.name,
170
+ id=agent.name,
171
+ provider=Provider(
172
+ model=result.model_used
173
+ if result.model_used is not None
174
+ else (
175
+ result.configuration.get("model")
176
+ if result.configuration is not None
177
+ else agent.extra_config.get("model", agent.model)
178
+ )
179
+ ),
180
+ system_prompt_name=None, # CopilotTask doesn't have named prompts
181
+ mcp_servers=[], # Could convert agent.mcp_servers if needed
182
+ allowed_tools=agent.allowed_tools,
183
+ )
184
+
185
+ return aitest_result, aitest_agent
186
+
187
+
188
+ def stash_on_item(
189
+ item: Item,
190
+ agent: CopilotTask,
191
+ result: CopilotResult,
192
+ *,
193
+ comparison_role: Literal["baseline", "treatment"] | None = None,
194
+ ) -> None:
195
+ """Stash result on the test node for pytest-copilot compatibility.
196
+
197
+ The plugin reads ``node._aitest_runs`` to retain every execution.
198
+ ``node._aitest_result`` and ``node._aitest_agent`` expose the latest
199
+ execution for other report hooks.
200
+
201
+ ``comparison_role`` labels report records only; the runtime config and
202
+ result's agent back-reference remain unchanged.
203
+
204
+ Called automatically by the ``copilot_run`` fixture and by the
205
+ ``pytest_runtest_makereport`` plugin hook; consumers should rarely
206
+ need to call this directly.
207
+ """
208
+ converted = _convert_to_aitest(agent, result)
209
+ if converted is not None:
210
+ if comparison_role is not None:
211
+ report_name = f"{agent.name} ({comparison_role})"
212
+ converted[1].name = report_name
213
+ converted[1].id = report_name
214
+ converted[0].comparison_role = comparison_role
215
+ item._aitest_result = converted[0] # type: ignore[attr-defined]
216
+ item._aitest_agent = converted[1] # type: ignore[attr-defined]
217
+ runs = getattr(item, "_aitest_runs", None)
218
+ if runs is None:
219
+ runs = []
220
+ item._aitest_runs = runs # type: ignore[attr-defined]
221
+ runs.append(converted)
222
+
223
+
224
+ def _prepare_ab_configs(
225
+ baseline: CopilotTask, treatment: CopilotTask, tmp_path: Path
226
+ ) -> tuple[CopilotTask, CopilotTask]:
227
+ baseline_dir = tmp_path / "baseline"
228
+ treatment_dir = tmp_path / "treatment"
229
+ baseline_dir.mkdir(exist_ok=True)
230
+ treatment_dir.mkdir(exist_ok=True)
231
+ return (
232
+ dataclasses.replace(baseline, working_directory=str(baseline_dir)),
233
+ dataclasses.replace(treatment, working_directory=str(treatment_dir)),
234
+ )
235
+
236
+
237
+ @pytest.fixture
238
+ def ab_run(
239
+ request: pytest.FixtureRequest,
240
+ tmp_path: Path,
241
+ ) -> Callable[..., Coroutine[Any, Any, tuple[CopilotResult, CopilotResult]]]:
242
+ """Run two agents against the same task in isolated directories.
243
+
244
+ Creates ``baseline/`` and ``treatment/`` subdirectories under
245
+ ``tmp_path``, overrides ``working_directory`` on each agent so they
246
+ never share a workspace, then runs them sequentially and stashes
247
+ both results for pytest-copilot reporting. Each side keeps its
248
+ configuration and trace; the pytest outcome describes the combined test.
249
+
250
+ Example::
251
+
252
+ async def test_docstring_instruction(ab_run):
253
+ baseline = CopilotTask(instructions="Write Python code.")
254
+ treatment = CopilotTask(
255
+ instructions="Write Python code. Add Google-style docstrings to every function."
256
+ )
257
+
258
+ b, t = await ab_run(baseline, treatment, "Create math.py with add(a, b).")
259
+
260
+ assert b.success and t.success
261
+ assert '\"\"\"' not in b.file("math.py"), "Baseline should not have docstrings"
262
+ assert '\"\"\"' in t.file("math.py"), "Treatment should add docstrings"
263
+
264
+ Args:
265
+ baseline: Control ``CopilotTask`` (the existing / unchanged config).
266
+ treatment: Treatment ``CopilotTask`` (the change you are testing).
267
+ task: Prompt to give both agents.
268
+
269
+ Returns:
270
+ ``(baseline_result, treatment_result)`` tuple.
271
+ """
272
+
273
+ async def _run(
274
+ baseline: CopilotTask,
275
+ treatment: CopilotTask,
276
+ task: str,
277
+ ) -> tuple[CopilotResult, CopilotResult]:
278
+ baseline, treatment = _prepare_ab_configs(baseline, treatment, tmp_path)
279
+
280
+ # Run sequentially — agents may write to disk, install packages, etc.
281
+ baseline_result = await run_copilot(baseline, task)
282
+ stash_on_item(request.node, baseline, baseline_result, comparison_role="baseline")
283
+ treatment_result = await run_copilot(treatment, task)
284
+
285
+ stash_on_item(request.node, treatment, treatment_result, comparison_role="treatment")
286
+
287
+ return baseline_result, treatment_result
288
+
289
+ return _run