pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,177 @@
1
+ Metadata-Version: 2.5
2
+ Name: pytest-copilot
3
+ Version: 2.0.0
4
+ Summary: Run GitHub Copilot tasks in pytest. Check results with your own assertions.
5
+ Project-URL: Homepage, https://github.com/sbroenne/pytest-copilot
6
+ Project-URL: Repository, https://github.com/sbroenne/pytest-copilot
7
+ Author: Stefan Brunner
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: agents,ai,copilot,custom-agents,llm,mcp,pytest,skills,testing
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Framework :: Pytest
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.11
22
+ Requires-Dist: github-copilot-sdk>=1.0.28
23
+ Requires-Dist: httpx>=0.28.1
24
+ Requires-Dist: mcp>=2.3.0
25
+ Requires-Dist: pydantic>=2.14.1
26
+ Requires-Dist: pytest-asyncio>=1.4.1
27
+ Requires-Dist: pytest>=9.2.0
28
+ Requires-Dist: python-frontmatter>=1.3.0
29
+ Requires-Dist: tomlkit>=0.15.2
30
+ Requires-Dist: trio>=0.34.0
31
+ Requires-Dist: websockets>=17.3
32
+ Provides-Extra: dev
33
+ Requires-Dist: pre-commit>=4.7.0; extra == 'dev'
34
+ Requires-Dist: pyright>=1.1.414; extra == 'dev'
35
+ Requires-Dist: pytest-cov>=7.1.0; extra == 'dev'
36
+ Requires-Dist: python-dotenv>=1.2.4; extra == 'dev'
37
+ Requires-Dist: ruff>=0.17.3; extra == 'dev'
38
+ Requires-Dist: typeguard>=4.6.0; extra == 'dev'
39
+ Provides-Extra: docs
40
+ Requires-Dist: cairosvg>=2.9.1; extra == 'docs'
41
+ Requires-Dist: mkdocs-material<9.9,>=9.7.7; extra == 'docs'
42
+ Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
43
+ Requires-Dist: mkdocstrings[python]>=1.0.8; extra == 'docs'
44
+ Requires-Dist: pillow>=13.0.0; extra == 'docs'
45
+ Provides-Extra: test
46
+ Requires-Dist: syrupy<7,>=6.1.1; extra == 'test'
47
+ Description-Content-Type: text/markdown
48
+
49
+ # pytest-copilot
50
+
51
+ [![PyPI version](https://img.shields.io/pypi/v/pytest-copilot)](https://pypi.org/project/pytest-copilot/)
52
+ [![CI](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml)
53
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
54
+
55
+ **Test whether GitHub Copilot can complete tasks with your tools and instructions.**
56
+
57
+ Independently maintained; not an official GitHub product or endorsed by GitHub.
58
+
59
+ pytest-copilot runs real Copilot sessions from pytest. Give Copilot a
60
+ task, check the actual output with ordinary assertions, and save the execution
61
+ as JSON.
62
+
63
+ Use it to test MCP servers, CLI tools, skills, system prompts, plugins, and custom
64
+ agents. It answers a different question from your tool's unit tests: **can
65
+ Copilot use the interface correctly?** A working tool can still have confusing
66
+ descriptions or arguments that lead the agent to the wrong result.
67
+
68
+ You define what correctness means. The framework manages the Copilot session,
69
+ execution limits, usage, and evidence; it does not grade answers with another AI.
70
+
71
+ ## Quick start
72
+
73
+ You need Python 3.11+, [uv](https://docs.astral.sh/uv/), and a GitHub account with
74
+ Copilot access.
75
+
76
+ Run these commands in an existing Python project with a `pyproject.toml`:
77
+
78
+ ```bash
79
+ uv add pytest-copilot
80
+ gh auth login --hostname github.com
81
+ uv run pytest-copilot init
82
+ uv run pytest-copilot doctor
83
+ uv run python -m pytest tests/test_copilot_run.py -v
84
+ ```
85
+
86
+ `init` creates a test using the bundled Todo MCP server, configures JSON output,
87
+ and adds starter-model rates to `pricing.toml`. It reports conflicts rather than
88
+ overwriting existing files or settings. `doctor` checks setup and Copilot access.
89
+ Real task execution may consume Copilot premium requests.
90
+
91
+ The test checks the tool's arguments and returned task state. pytest prints the
92
+ result; `aitest-reports/results.json` preserves the execution. See the
93
+ [setup guide](docs/getting-started/index.md) or the standalone
94
+ [quickstart project](examples/quickstart/).
95
+
96
+ ## A test that checks the result
97
+
98
+ ```python
99
+ from __future__ import annotations
100
+
101
+ import json
102
+
103
+ from pytest_copilot.copilot import CopilotTask
104
+
105
+
106
+ async def test_order_total(copilot_run, tmp_path):
107
+ agent = CopilotTask(
108
+ name="order-total",
109
+ model="gpt-5.6-sol",
110
+ instructions="Save the requested output as JSON. Use integer cents.",
111
+ working_directory=str(tmp_path),
112
+ )
113
+ result = await copilot_run(
114
+ agent,
115
+ "Save order.json for two items at 1250 cents each plus 20% tax. "
116
+ "Include subtotal_cents, tax_cents, and total_cents.",
117
+ )
118
+ assert result.success, result.error
119
+ output = tmp_path / "order.json"
120
+ assert output.is_file()
121
+ assert json.loads(output.read_text(encoding="utf-8")) == {
122
+ "subtotal_cents": 2500,
123
+ "tax_cents": 500,
124
+ "total_cents": 3000,
125
+ }
126
+ ```
127
+
128
+ `CopilotTask` configures the session. The second argument to `copilot_run` is
129
+ the user task. `result.success` checks execution completion; the file assertions
130
+ check that the task produced the right result.
131
+
132
+ For your own tools, check the resulting application state, saved files, or
133
+ operation output. A tool call or a model's claim is not proof of task success.
134
+ Use approved isolation for live tests: a temporary directory is not a sandbox,
135
+ and generated code or tool calls can act with your user's access.
136
+
137
+ ## What the framework provides
138
+
139
+ | You need | Use |
140
+ | --- | --- |
141
+ | Real task execution | `CopilotTask` and the `copilot_run` pytest fixture |
142
+ | Controlled runs | Timeouts, tool budgets, request controls, and permission settings |
143
+ | Debugging evidence | Configuration, calls, arguments, outputs, errors, and completion records |
144
+ | Repeatable comparisons | `ab_run` and pytest repetitions |
145
+ | Usage visibility | Captured tokens, Copilot premium requests, and explicit price estimates |
146
+ | Saved results | Native JSON with execution evidence and ordinary pytest outcomes |
147
+
148
+ Your tests supply application fixtures, guarded tool adapters, and independent
149
+ output checks. The framework makes one execution attempt per call; it does not
150
+ retry failures automatically. Missing usage, prices, and incomplete evidence
151
+ remain visible.
152
+
153
+ ## Investigate and compare
154
+
155
+ Give your coding agent a failed test, its JSON evidence, the requirements, and
156
+ the relevant source. It can investigate the failure and help you change the
157
+ interface. Keep the success criteria fixed when rerunning a comparison.
158
+
159
+ For A/B tests, record verification separately for each side: both saved
160
+ executions share one pytest outcome. That outcome alone cannot establish
161
+ independent pass rates or a causal improvement. See
162
+ [comparing configurations](docs/getting-started/comparing.md).
163
+
164
+ ## Documentation
165
+
166
+ - [Getting started](https://sbroenne.github.io/pytest-copilot/getting-started/)
167
+ - [Task guides](docs/how-to/index.md) and [API reference](docs/reference/api.md)
168
+ - [Examples](examples/README.md), including historical skill comparisons
169
+ - [Upgrading to 2.0](docs/migration.md): new package, imports, and task names,
170
+ without compatibility aliases
171
+
172
+ No framework companion skill is required or distributed. Testing your own
173
+ domain skills is supported.
174
+
175
+ ## License
176
+
177
+ MIT. Inspired by [agent-benchmark](https://github.com/mykhaliev/agent-benchmark).
@@ -0,0 +1,55 @@
1
+ pytest_copilot/__init__.py,sha256=YLOEHIxqdMlDR3IQWnYrY8ypYaMzttM3YT-ijlnMFLE,2988
2
+ pytest_copilot/onboarding.py,sha256=1N0LowuNTlZDN0UzlazQxVGg-uE5OR586gapAR11XAM,14481
3
+ pytest_copilot/plugin.py,sha256=XQeWkx_fPsXxrEnxmEcRQY-RU2akRhhWZsS1pJCGDTY,20876
4
+ pytest_copilot/plugin_options.py,sha256=EeBmqd0T3Gobu67FqGXSxGJyAEjPf7xxi-5XIyLb9dA,932
5
+ pytest_copilot/plugin_report.py,sha256=GD1p8l53lO5oRe__PckmRTeIeqsBWbdpe8C5VekI5Qc,570
6
+ pytest_copilot/copilot/__init__.py,sha256=lMHWli5cWVanxUXTrGHyhdyHZ73e0iJfGEfH2lh9KbE,1416
7
+ pytest_copilot/copilot/_command_job.py,sha256=mA5rf9RPqKTk1PjL0GG4BGID5U9gXmsZ5OJ4ohmGlb4,3805
8
+ pytest_copilot/copilot/api.py,sha256=pRQTe4bVEJ8zDJhPGJ7ZVhtrypS9E1pWGSEGfob60TM,461
9
+ pytest_copilot/copilot/client.py,sha256=18J_XOGv3CKbH2YKVdwgmQABXWDKY3gJldlxat66ZYE,2028
10
+ pytest_copilot/copilot/commands.py,sha256=iXyTTbYb2Ekb18sdl5wZYSq0RR4yknYh0PgmscKb22k,11345
11
+ pytest_copilot/copilot/config.py,sha256=mwCFN7FqdI5DrSb6veohBRNyRj1396WiNiCkdbr3bJk,4761
12
+ pytest_copilot/copilot/contracts.py,sha256=275OXMRyrChnRq9eTQLJU3VtW9dZZWrDncOZbW8TMjc,6384
13
+ pytest_copilot/copilot/controls.py,sha256=oZrzugNmHmHGC5alemzrFIQfvepVj8PM-UKeeGvR1SA,9655
14
+ pytest_copilot/copilot/events.py,sha256=VGIltIvSjODCyw8a6M1p3ghoydMkivx3GSU2yvnie-I,34954
15
+ pytest_copilot/copilot/fixtures.py,sha256=1st3BpHhitHlky9nqwQJjau1_42xlU2rgndtAfD45Lw,10640
16
+ pytest_copilot/copilot/personas.py,sha256=ziHY41PwKXSTWdYmWHb6Dup3yCwESNEa9ZAX6JjG7Cs,22229
17
+ pytest_copilot/copilot/requests.py,sha256=oTS8PoBktnLtU83BqEcU4XSCTrNJWIJgzWKTQdbfrR0,14674
18
+ pytest_copilot/copilot/result.py,sha256=Iw8ly6mXlHHLjn7Cu1PVG8FqCZF-IEKw7AcXY2iTu_E,10600
19
+ pytest_copilot/copilot/runner.py,sha256=oh1PwuHtfAfdr7UrgH5g5GVhQ6MeOvPC1zY3cstcmpQ,11477
20
+ pytest_copilot/copilot/skills.py,sha256=5yYi2aUIWr8xu5PKDYQqtfmYfosAKe6vEFoKIxcUcs8,4096
21
+ pytest_copilot/copilot/task.py,sha256=kCwe3YrfSw7BwBHlOQJu5RP3epbPsl2GI2o1e3pgzLM,21051
22
+ pytest_copilot/core/__init__.py,sha256=iMjsGV4fYEFjNs4fHgNpfiIDer6WwDkALDHWwujd1u4,1799
23
+ pytest_copilot/core/errors.py,sha256=ZZtBJUCGK9W6i0vbccUOcMFDn_2pFnrwWToCpBk7SVE,1376
24
+ pytest_copilot/core/evals.py,sha256=TS2FCgboEq4TP0ujcI0j0Mjx_o7imyfmj0M-5Tw0ZVk,17876
25
+ pytest_copilot/core/plugin.py,sha256=dBEnR0JfhjmPUK6BNx8LbTvhHX4ZYnXPvyLp_5-oit0,14452
26
+ pytest_copilot/core/prompt.py,sha256=j0y27CLKqU6pw60rNf4MUfhqd6eNatdo08IKVTuSehk,4932
27
+ pytest_copilot/core/result.py,sha256=Avry4QvIYrl3ZP8Tjds9UDszmzvnbTwZNb0WorCCGVg,17764
28
+ pytest_copilot/core/serialization.py,sha256=VXe_1nGDK8aiG91kg9RtvNQ8coS1B5kkYZQtPfZwvSU,5164
29
+ pytest_copilot/core/skill.py,sha256=fJlgbojN_VWkuOPVymEkp0If3OpCzsPjXPa9iz1aRpg,14362
30
+ pytest_copilot/core/skill_evals.py,sha256=9_4xn1aSBeMq0gi4IOlS2ZSZitdkZdj-fAwTIziLFo4,5001
31
+ pytest_copilot/core/skill_grading.py,sha256=TEcPqdDp3sLPTOSEjRvUeWPTKrfo0dBJbKPll_DQzP0,2661
32
+ pytest_copilot/core/validation.py,sha256=SwthJNsBhOUFBzgCB0qnN4Y_W19fd2ykPmyUqrWxJEk,3674
33
+ pytest_copilot/execution/__init__.py,sha256=OfMlV2pZ9YFZnlgFFc7BzD3ksJqD1AKcSI170Y63qsA,269
34
+ pytest_copilot/execution/cost.py,sha256=CT5XjW7KbkXt5XTAWy7sH3S3seONOOouWGJ4ewd6fLY,5408
35
+ pytest_copilot/execution/rate_limiter.py,sha256=nyi9gkJ-gUsAONcd4dKyPzvCoKpg8Sp6fgGtv6FDz_E,6355
36
+ pytest_copilot/execution/servers.py,sha256=iVBA09L5NKCzkCIB8nlE8uS-3SBiDxfqOnCYde695GQ,15835
37
+ pytest_copilot/execution/skill_tools.py,sha256=yQAdSwrYznDemmosZdcPmPeKllglECTvcPtjvBXmCR8,7483
38
+ pytest_copilot/fixtures/__init__.py,sha256=NAzUdLnlqaTzQ9W20Vzp_Ylo-bOS7OkoOlM_WAJ7t_o,515
39
+ pytest_copilot/fixtures/factories.py,sha256=1e8dqkTVjvzvXt2bOUbhq9WipcINJdWvFaDnOAVThDM,1161
40
+ pytest_copilot/fixtures/iteration.py,sha256=76XTurW1gZQMzdfOH4FU3AMB-leHcNhOVFyVQ2ql-H0,500
41
+ pytest_copilot/reporting/__init__.py,sha256=JmeM2y9tC3aBYX6bjLvFYO7Dch0Lc3k9appKK_XC374,430
42
+ pytest_copilot/reporting/collector.py,sha256=0BBLv7o_kIhFo7xwhJhiSC3BkkM0K_B42LzA9QWTz8E,6534
43
+ pytest_copilot/reporting/generator.py,sha256=93vTlKxce_fyfuErXkxFjrMm1XHfXRuZdDSsft4G9n8,2039
44
+ pytest_copilot/reporting/schema.py,sha256=ppusJRExdwX2PUh7DprJkruonkk0eWD8C7cBBysXGyg,108
45
+ pytest_copilot/testing/__init__.py,sha256=-2guhlKA-WlpwRZchqCnwetCUd6k4udzcSQI31Hk5Mo,488
46
+ pytest_copilot/testing/banking.py,sha256=tY-Wtk7VjFCWBrLkxUEkmo8f77FSkLxQv2_TkgBqb1g,16050
47
+ pytest_copilot/testing/banking_mcp.py,sha256=E0Wxzb6H_nWW1zgXhwRfj8t1qeAThHfR1YBp4JcxMJQ,5042
48
+ pytest_copilot/testing/todo.py,sha256=kIB8KIiTeWt0FJIGOySaH6DB1RRpZxrAsQl83pHh2MA,12480
49
+ pytest_copilot/testing/todo_mcp.py,sha256=FtPLDDm_Klt0kfu9GugC1VK9AJXAHlR4lMFmVnY8H48,4626
50
+ pytest_copilot/testing/types.py,sha256=kxV2ZRUMdDSXYgY6p06bPzdm8BdtlOkyp_Vj7Vw2Yv0,745
51
+ pytest_copilot-2.0.0.dist-info/METADATA,sha256=1wFL5J8sJ8BF4XvfRzW7N3fmEuc_2Ptc3f_5zNL227E,7281
52
+ pytest_copilot-2.0.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
53
+ pytest_copilot-2.0.0.dist-info/entry_points.txt,sha256=J4dOGTO4LHMcVBGZAMpzWXM8HoE869rQ1snuaXfXFfU,109
54
+ pytest_copilot-2.0.0.dist-info/licenses/LICENSE,sha256=yibT3IgnO__PTA13Cd5Ce7x_MDngxfXu9ktxheiSDws,1071
55
+ pytest_copilot-2.0.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,5 @@
1
+ [console_scripts]
2
+ pytest-copilot = pytest_copilot.onboarding:main
3
+
4
+ [pytest11]
5
+ aitest = pytest_copilot.plugin
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Stefan Brunner
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.