pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Pytest fixtures for GitHub Copilot testing.
|
|
2
|
+
|
|
3
|
+
Provides the ``copilot_run`` fixture that executes prompts against Copilot
|
|
4
|
+
and stashes results for pytest-copilot reporting.
|
|
5
|
+
|
|
6
|
+
Also provides ``ab_run``, a higher-level fixture for A/B testing two agent
|
|
7
|
+
configurations against the same task in isolated directories.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import dataclasses
|
|
13
|
+
from copy import deepcopy
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
|
|
19
|
+
from pytest_copilot.copilot.api import run_copilot
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from collections.abc import Callable, Coroutine
|
|
23
|
+
|
|
24
|
+
from _pytest.nodes import Item
|
|
25
|
+
|
|
26
|
+
from pytest_copilot.copilot.result import CopilotResult
|
|
27
|
+
from pytest_copilot.copilot.task import CopilotTask
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@pytest.fixture
|
|
31
|
+
def copilot_run(
|
|
32
|
+
request: pytest.FixtureRequest,
|
|
33
|
+
) -> Callable[..., Coroutine[Any, Any, CopilotResult]]:
|
|
34
|
+
"""Execute a prompt against a CopilotTask and capture results.
|
|
35
|
+
|
|
36
|
+
Results are automatically stashed on the test node for pytest-copilot's
|
|
37
|
+
evidence collector. Native JSON preserves execution alongside ordinary pytest
|
|
38
|
+
outcomes; the coding agent interprets it together with source and test criteria.
|
|
39
|
+
|
|
40
|
+
Example:
|
|
41
|
+
async def test_file_creation(copilot_run, tmp_path):
|
|
42
|
+
agent = CopilotTask(
|
|
43
|
+
instructions="Create files as requested.",
|
|
44
|
+
working_directory=str(tmp_path),
|
|
45
|
+
)
|
|
46
|
+
result = await copilot_run(agent, "Create hello.py with print('hello')")
|
|
47
|
+
assert result.success
|
|
48
|
+
assert result.tool_was_called("create_file")
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
async def _run(agent: CopilotTask, prompt: str) -> CopilotResult:
|
|
52
|
+
result = await run_copilot(agent, prompt)
|
|
53
|
+
|
|
54
|
+
# Stash for pytest-copilot's reporting plugin.
|
|
55
|
+
# The plugin hook also does this automatically for tests that
|
|
56
|
+
# call run_copilot() directly, but explicit stashing from the
|
|
57
|
+
# fixture ensures it works even if the hook order changes.
|
|
58
|
+
stash_on_item(request.node, agent, result)
|
|
59
|
+
|
|
60
|
+
return result
|
|
61
|
+
|
|
62
|
+
return _run
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _convert_to_aitest(
|
|
66
|
+
agent: CopilotTask,
|
|
67
|
+
result: CopilotResult,
|
|
68
|
+
) -> tuple[Any, Any] | None:
|
|
69
|
+
"""Convert CopilotResult to pytest-copilot types.
|
|
70
|
+
|
|
71
|
+
Returns ``(EvalResult, agent_wrapper)`` tuple, or ``None`` if conversion
|
|
72
|
+
fails.
|
|
73
|
+
|
|
74
|
+
Since CopilotResult already uses pytest-copilot's Turn and ToolCall types,
|
|
75
|
+
the turns can be passed through directly without rebuilding.
|
|
76
|
+
|
|
77
|
+
The agent_wrapper is a simple object with the fields expected by plugin.py:
|
|
78
|
+
- name
|
|
79
|
+
- provider.model
|
|
80
|
+
- system_prompt_name (optional)
|
|
81
|
+
- mcp_servers (optional)
|
|
82
|
+
- allowed_tools (optional)
|
|
83
|
+
"""
|
|
84
|
+
from dataclasses import dataclass
|
|
85
|
+
|
|
86
|
+
from pytest_copilot.core.result import EvalResult, SubagentInvocation
|
|
87
|
+
from pytest_copilot.execution.cost import estimate_cost
|
|
88
|
+
|
|
89
|
+
# Estimate USD cost from captured token usage and pricing.toml.
|
|
90
|
+
# Models without pricing contribute 0.0 and are recorded in
|
|
91
|
+
# execution.cost.models_without_pricing (saved with report evidence).
|
|
92
|
+
cost_usd = sum(
|
|
93
|
+
estimate_cost(
|
|
94
|
+
usage.model or result.model_used or "",
|
|
95
|
+
usage.input_tokens,
|
|
96
|
+
usage.output_tokens,
|
|
97
|
+
usage.cache_read_tokens or 0,
|
|
98
|
+
)
|
|
99
|
+
for usage in result.all_usage
|
|
100
|
+
if usage.input_tokens is not None and usage.output_tokens is not None
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Turns already use aitest's Turn/ToolCall types — pass through directly
|
|
104
|
+
aitest_result = EvalResult(
|
|
105
|
+
turns=list(result.turns),
|
|
106
|
+
success=result.success,
|
|
107
|
+
error=result.error,
|
|
108
|
+
duration_ms=result.duration_ms,
|
|
109
|
+
token_usage=result.token_usage,
|
|
110
|
+
cost_usd=cost_usd,
|
|
111
|
+
effective_system_prompt=(
|
|
112
|
+
result.configuration.get("instructions") if result.configuration is not None else None
|
|
113
|
+
),
|
|
114
|
+
premium_requests=result.total_premium_requests,
|
|
115
|
+
evidence_complete=result.evidence_complete,
|
|
116
|
+
capture_errors=list(result.capture_errors),
|
|
117
|
+
request_audit=list(result.request_audit),
|
|
118
|
+
stop_reason=result.stop_reason,
|
|
119
|
+
usage=list(result.usage),
|
|
120
|
+
tool_calls_admitted=result.tool_calls_admitted,
|
|
121
|
+
skill_discovery=deepcopy(result.skill_discovery),
|
|
122
|
+
configuration=deepcopy(result.configuration),
|
|
123
|
+
model_used=result.model_used,
|
|
124
|
+
reasoning_traces=list(result.reasoning_traces),
|
|
125
|
+
permission_requested=result.permission_requested,
|
|
126
|
+
permissions=deepcopy(result.permissions),
|
|
127
|
+
)
|
|
128
|
+
for invocation in result.subagent_invocations:
|
|
129
|
+
child_result = None
|
|
130
|
+
if invocation.result is not None:
|
|
131
|
+
from pytest_copilot.copilot.task import CopilotTask
|
|
132
|
+
|
|
133
|
+
child_agent = invocation.result.agent
|
|
134
|
+
if not isinstance(child_agent, CopilotTask):
|
|
135
|
+
raise ValueError(f"Child invocation {invocation.invocation_id!r} has no eval")
|
|
136
|
+
converted_child = _convert_to_aitest(child_agent, invocation.result)
|
|
137
|
+
assert converted_child is not None
|
|
138
|
+
child_result = converted_child[0]
|
|
139
|
+
aitest_result.subagent_invocations.append(
|
|
140
|
+
SubagentInvocation(
|
|
141
|
+
invocation_id=invocation.invocation_id,
|
|
142
|
+
name=invocation.name,
|
|
143
|
+
status=invocation.status,
|
|
144
|
+
duration_ms=invocation.duration_ms,
|
|
145
|
+
result=child_result,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# Create a minimal wrapper with just the fields needed by plugin.py
|
|
150
|
+
@dataclass(slots=True)
|
|
151
|
+
class Provider:
|
|
152
|
+
model: str | None
|
|
153
|
+
|
|
154
|
+
@dataclass(slots=True)
|
|
155
|
+
class AgentWrapper:
|
|
156
|
+
name: str
|
|
157
|
+
id: str
|
|
158
|
+
provider: Provider
|
|
159
|
+
system_prompt_name: str | None = None
|
|
160
|
+
mcp_servers: list[Any] = None # type: ignore[assignment]
|
|
161
|
+
allowed_tools: list[str] | None = None
|
|
162
|
+
skill: Any = None
|
|
163
|
+
|
|
164
|
+
def __post_init__(self) -> None:
|
|
165
|
+
if self.mcp_servers is None:
|
|
166
|
+
self.mcp_servers = []
|
|
167
|
+
|
|
168
|
+
aitest_agent = AgentWrapper(
|
|
169
|
+
name=agent.name,
|
|
170
|
+
id=agent.name,
|
|
171
|
+
provider=Provider(
|
|
172
|
+
model=result.model_used
|
|
173
|
+
if result.model_used is not None
|
|
174
|
+
else (
|
|
175
|
+
result.configuration.get("model")
|
|
176
|
+
if result.configuration is not None
|
|
177
|
+
else agent.extra_config.get("model", agent.model)
|
|
178
|
+
)
|
|
179
|
+
),
|
|
180
|
+
system_prompt_name=None, # CopilotTask doesn't have named prompts
|
|
181
|
+
mcp_servers=[], # Could convert agent.mcp_servers if needed
|
|
182
|
+
allowed_tools=agent.allowed_tools,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
return aitest_result, aitest_agent
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def stash_on_item(
|
|
189
|
+
item: Item,
|
|
190
|
+
agent: CopilotTask,
|
|
191
|
+
result: CopilotResult,
|
|
192
|
+
*,
|
|
193
|
+
comparison_role: Literal["baseline", "treatment"] | None = None,
|
|
194
|
+
) -> None:
|
|
195
|
+
"""Stash result on the test node for pytest-copilot compatibility.
|
|
196
|
+
|
|
197
|
+
The plugin reads ``node._aitest_runs`` to retain every execution.
|
|
198
|
+
``node._aitest_result`` and ``node._aitest_agent`` expose the latest
|
|
199
|
+
execution for other report hooks.
|
|
200
|
+
|
|
201
|
+
``comparison_role`` labels report records only; the runtime config and
|
|
202
|
+
result's agent back-reference remain unchanged.
|
|
203
|
+
|
|
204
|
+
Called automatically by the ``copilot_run`` fixture and by the
|
|
205
|
+
``pytest_runtest_makereport`` plugin hook; consumers should rarely
|
|
206
|
+
need to call this directly.
|
|
207
|
+
"""
|
|
208
|
+
converted = _convert_to_aitest(agent, result)
|
|
209
|
+
if converted is not None:
|
|
210
|
+
if comparison_role is not None:
|
|
211
|
+
report_name = f"{agent.name} ({comparison_role})"
|
|
212
|
+
converted[1].name = report_name
|
|
213
|
+
converted[1].id = report_name
|
|
214
|
+
converted[0].comparison_role = comparison_role
|
|
215
|
+
item._aitest_result = converted[0] # type: ignore[attr-defined]
|
|
216
|
+
item._aitest_agent = converted[1] # type: ignore[attr-defined]
|
|
217
|
+
runs = getattr(item, "_aitest_runs", None)
|
|
218
|
+
if runs is None:
|
|
219
|
+
runs = []
|
|
220
|
+
item._aitest_runs = runs # type: ignore[attr-defined]
|
|
221
|
+
runs.append(converted)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _prepare_ab_configs(
|
|
225
|
+
baseline: CopilotTask, treatment: CopilotTask, tmp_path: Path
|
|
226
|
+
) -> tuple[CopilotTask, CopilotTask]:
|
|
227
|
+
baseline_dir = tmp_path / "baseline"
|
|
228
|
+
treatment_dir = tmp_path / "treatment"
|
|
229
|
+
baseline_dir.mkdir(exist_ok=True)
|
|
230
|
+
treatment_dir.mkdir(exist_ok=True)
|
|
231
|
+
return (
|
|
232
|
+
dataclasses.replace(baseline, working_directory=str(baseline_dir)),
|
|
233
|
+
dataclasses.replace(treatment, working_directory=str(treatment_dir)),
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@pytest.fixture
|
|
238
|
+
def ab_run(
|
|
239
|
+
request: pytest.FixtureRequest,
|
|
240
|
+
tmp_path: Path,
|
|
241
|
+
) -> Callable[..., Coroutine[Any, Any, tuple[CopilotResult, CopilotResult]]]:
|
|
242
|
+
"""Run two agents against the same task in isolated directories.
|
|
243
|
+
|
|
244
|
+
Creates ``baseline/`` and ``treatment/`` subdirectories under
|
|
245
|
+
``tmp_path``, overrides ``working_directory`` on each agent so they
|
|
246
|
+
never share a workspace, then runs them sequentially and stashes
|
|
247
|
+
both results for pytest-copilot reporting. Each side keeps its
|
|
248
|
+
configuration and trace; the pytest outcome describes the combined test.
|
|
249
|
+
|
|
250
|
+
Example::
|
|
251
|
+
|
|
252
|
+
async def test_docstring_instruction(ab_run):
|
|
253
|
+
baseline = CopilotTask(instructions="Write Python code.")
|
|
254
|
+
treatment = CopilotTask(
|
|
255
|
+
instructions="Write Python code. Add Google-style docstrings to every function."
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
b, t = await ab_run(baseline, treatment, "Create math.py with add(a, b).")
|
|
259
|
+
|
|
260
|
+
assert b.success and t.success
|
|
261
|
+
assert '\"\"\"' not in b.file("math.py"), "Baseline should not have docstrings"
|
|
262
|
+
assert '\"\"\"' in t.file("math.py"), "Treatment should add docstrings"
|
|
263
|
+
|
|
264
|
+
Args:
|
|
265
|
+
baseline: Control ``CopilotTask`` (the existing / unchanged config).
|
|
266
|
+
treatment: Treatment ``CopilotTask`` (the change you are testing).
|
|
267
|
+
task: Prompt to give both agents.
|
|
268
|
+
|
|
269
|
+
Returns:
|
|
270
|
+
``(baseline_result, treatment_result)`` tuple.
|
|
271
|
+
"""
|
|
272
|
+
|
|
273
|
+
async def _run(
|
|
274
|
+
baseline: CopilotTask,
|
|
275
|
+
treatment: CopilotTask,
|
|
276
|
+
task: str,
|
|
277
|
+
) -> tuple[CopilotResult, CopilotResult]:
|
|
278
|
+
baseline, treatment = _prepare_ab_configs(baseline, treatment, tmp_path)
|
|
279
|
+
|
|
280
|
+
# Run sequentially — agents may write to disk, install packages, etc.
|
|
281
|
+
baseline_result = await run_copilot(baseline, task)
|
|
282
|
+
stash_on_item(request.node, baseline, baseline_result, comparison_role="baseline")
|
|
283
|
+
treatment_result = await run_copilot(treatment, task)
|
|
284
|
+
|
|
285
|
+
stash_on_item(request.node, treatment, treatment_result, comparison_role="treatment")
|
|
286
|
+
|
|
287
|
+
return baseline_result, treatment_result
|
|
288
|
+
|
|
289
|
+
return _run
|