pytest-copilot 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot-2.0.0/.gitignore +72 -0
  2. pytest_copilot-2.0.0/LICENSE +21 -0
  3. pytest_copilot-2.0.0/PKG-INFO +177 -0
  4. pytest_copilot-2.0.0/README.md +129 -0
  5. pytest_copilot-2.0.0/pyproject.toml +146 -0
  6. pytest_copilot-2.0.0/src/pytest_copilot/__init__.py +133 -0
  7. pytest_copilot-2.0.0/src/pytest_copilot/copilot/__init__.py +47 -0
  8. pytest_copilot-2.0.0/src/pytest_copilot/copilot/_command_job.py +109 -0
  9. pytest_copilot-2.0.0/src/pytest_copilot/copilot/api.py +12 -0
  10. pytest_copilot-2.0.0/src/pytest_copilot/copilot/client.py +58 -0
  11. pytest_copilot-2.0.0/src/pytest_copilot/copilot/commands.py +265 -0
  12. pytest_copilot-2.0.0/src/pytest_copilot/copilot/config.py +133 -0
  13. pytest_copilot-2.0.0/src/pytest_copilot/copilot/contracts.py +222 -0
  14. pytest_copilot-2.0.0/src/pytest_copilot/copilot/controls.py +230 -0
  15. pytest_copilot-2.0.0/src/pytest_copilot/copilot/events.py +795 -0
  16. pytest_copilot-2.0.0/src/pytest_copilot/copilot/fixtures.py +289 -0
  17. pytest_copilot-2.0.0/src/pytest_copilot/copilot/personas.py +607 -0
  18. pytest_copilot-2.0.0/src/pytest_copilot/copilot/requests.py +350 -0
  19. pytest_copilot-2.0.0/src/pytest_copilot/copilot/result.py +309 -0
  20. pytest_copilot-2.0.0/src/pytest_copilot/copilot/runner.py +266 -0
  21. pytest_copilot-2.0.0/src/pytest_copilot/copilot/skills.py +104 -0
  22. pytest_copilot-2.0.0/src/pytest_copilot/copilot/task.py +546 -0
  23. pytest_copilot-2.0.0/src/pytest_copilot/core/__init__.py +77 -0
  24. pytest_copilot-2.0.0/src/pytest_copilot/core/errors.py +44 -0
  25. pytest_copilot-2.0.0/src/pytest_copilot/core/evals.py +521 -0
  26. pytest_copilot-2.0.0/src/pytest_copilot/core/plugin.py +403 -0
  27. pytest_copilot-2.0.0/src/pytest_copilot/core/prompt.py +167 -0
  28. pytest_copilot-2.0.0/src/pytest_copilot/core/result.py +543 -0
  29. pytest_copilot-2.0.0/src/pytest_copilot/core/serialization.py +128 -0
  30. pytest_copilot-2.0.0/src/pytest_copilot/core/skill.py +406 -0
  31. pytest_copilot-2.0.0/src/pytest_copilot/core/skill_evals.py +148 -0
  32. pytest_copilot-2.0.0/src/pytest_copilot/core/skill_grading.py +78 -0
  33. pytest_copilot-2.0.0/src/pytest_copilot/core/validation.py +83 -0
  34. pytest_copilot-2.0.0/src/pytest_copilot/execution/__init__.py +15 -0
  35. pytest_copilot-2.0.0/src/pytest_copilot/execution/cost.py +159 -0
  36. pytest_copilot-2.0.0/src/pytest_copilot/execution/rate_limiter.py +177 -0
  37. pytest_copilot-2.0.0/src/pytest_copilot/execution/servers.py +442 -0
  38. pytest_copilot-2.0.0/src/pytest_copilot/execution/skill_tools.py +228 -0
  39. pytest_copilot-2.0.0/src/pytest_copilot/fixtures/__init__.py +18 -0
  40. pytest_copilot-2.0.0/src/pytest_copilot/fixtures/factories.py +44 -0
  41. pytest_copilot-2.0.0/src/pytest_copilot/fixtures/iteration.py +16 -0
  42. pytest_copilot-2.0.0/src/pytest_copilot/onboarding.py +413 -0
  43. pytest_copilot-2.0.0/src/pytest_copilot/plugin.py +562 -0
  44. pytest_copilot-2.0.0/src/pytest_copilot/plugin_options.py +32 -0
  45. pytest_copilot-2.0.0/src/pytest_copilot/plugin_report.py +18 -0
  46. pytest_copilot-2.0.0/src/pytest_copilot/reporting/__init__.py +16 -0
  47. pytest_copilot-2.0.0/src/pytest_copilot/reporting/collector.py +203 -0
  48. pytest_copilot-2.0.0/src/pytest_copilot/reporting/generator.py +55 -0
  49. pytest_copilot-2.0.0/src/pytest_copilot/reporting/schema.py +5 -0
  50. pytest_copilot-2.0.0/src/pytest_copilot/testing/__init__.py +17 -0
  51. pytest_copilot-2.0.0/src/pytest_copilot/testing/banking.py +479 -0
  52. pytest_copilot-2.0.0/src/pytest_copilot/testing/banking_mcp.py +167 -0
  53. pytest_copilot-2.0.0/src/pytest_copilot/testing/todo.py +380 -0
  54. pytest_copilot-2.0.0/src/pytest_copilot/testing/todo_mcp.py +156 -0
  55. pytest_copilot-2.0.0/src/pytest_copilot/testing/types.py +31 -0
@@ -0,0 +1,72 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ *.egg-info/
20
+ .installed.cfg
21
+ *.egg
22
+
23
+ # Virtual environments
24
+ .venv/
25
+ venv/
26
+ ENV/
27
+ env/
28
+
29
+ # IDE
30
+ .idea/
31
+ .vscode/
32
+ *.swp
33
+ *.swo
34
+ *~
35
+
36
+ # Testing
37
+ .pytest_cache/
38
+ .coverage
39
+ htmlcov/
40
+ .tox/
41
+ .nox/
42
+
43
+ # Type checking
44
+ .mypy_cache/
45
+
46
+ # Test outputs
47
+ test_results/
48
+ aitest-reports/
49
+
50
+ # Node.js
51
+ node_modules/
52
+ package-lock.json
53
+
54
+ # Environment
55
+ .env
56
+ .env.local
57
+
58
+ # OS
59
+ .DS_Store
60
+ Thumbs.db
61
+ site/
62
+ .cache/
63
+ # Squad: ignore runtime state (logs, inbox, sessions)
64
+ .squad/orchestration-log/
65
+ .squad/log/
66
+ .squad/decisions/inbox/
67
+ .squad/sessions/
68
+ # Squad: SubSquad activation file (local to this machine)
69
+ .squad-workstream
70
+
71
+ # pytest-copilot test scratch (some server tests create a cwd dir)
72
+ .pytest-execution-servers/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Stefan Brunner
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,177 @@
1
+ Metadata-Version: 2.5
2
+ Name: pytest-copilot
3
+ Version: 2.0.0
4
+ Summary: Run GitHub Copilot tasks in pytest. Check results with your own assertions.
5
+ Project-URL: Homepage, https://github.com/sbroenne/pytest-copilot
6
+ Project-URL: Repository, https://github.com/sbroenne/pytest-copilot
7
+ Author: Stefan Brunner
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: agents,ai,copilot,custom-agents,llm,mcp,pytest,skills,testing
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Framework :: Pytest
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Requires-Python: >=3.11
22
+ Requires-Dist: github-copilot-sdk>=1.0.28
23
+ Requires-Dist: httpx>=0.28.1
24
+ Requires-Dist: mcp>=2.3.0
25
+ Requires-Dist: pydantic>=2.14.1
26
+ Requires-Dist: pytest-asyncio>=1.4.1
27
+ Requires-Dist: pytest>=9.2.0
28
+ Requires-Dist: python-frontmatter>=1.3.0
29
+ Requires-Dist: tomlkit>=0.15.2
30
+ Requires-Dist: trio>=0.34.0
31
+ Requires-Dist: websockets>=17.3
32
+ Provides-Extra: dev
33
+ Requires-Dist: pre-commit>=4.7.0; extra == 'dev'
34
+ Requires-Dist: pyright>=1.1.414; extra == 'dev'
35
+ Requires-Dist: pytest-cov>=7.1.0; extra == 'dev'
36
+ Requires-Dist: python-dotenv>=1.2.4; extra == 'dev'
37
+ Requires-Dist: ruff>=0.17.3; extra == 'dev'
38
+ Requires-Dist: typeguard>=4.6.0; extra == 'dev'
39
+ Provides-Extra: docs
40
+ Requires-Dist: cairosvg>=2.9.1; extra == 'docs'
41
+ Requires-Dist: mkdocs-material<9.9,>=9.7.7; extra == 'docs'
42
+ Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
43
+ Requires-Dist: mkdocstrings[python]>=1.0.8; extra == 'docs'
44
+ Requires-Dist: pillow>=13.0.0; extra == 'docs'
45
+ Provides-Extra: test
46
+ Requires-Dist: syrupy<7,>=6.1.1; extra == 'test'
47
+ Description-Content-Type: text/markdown
48
+
49
+ # pytest-copilot
50
+
51
+ [![PyPI version](https://img.shields.io/pypi/v/pytest-copilot)](https://pypi.org/project/pytest-copilot/)
52
+ [![CI](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml)
53
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
54
+
55
+ **Test whether GitHub Copilot can complete tasks with your tools and instructions.**
56
+
57
+ Independently maintained; not an official GitHub product or endorsed by GitHub.
58
+
59
+ pytest-copilot runs real Copilot sessions from pytest. Give Copilot a
60
+ task, check the actual output with ordinary assertions, and save the execution
61
+ as JSON.
62
+
63
+ Use it to test MCP servers, CLI tools, skills, system prompts, plugins, and custom
64
+ agents. It answers a different question from your tool's unit tests: **can
65
+ Copilot use the interface correctly?** A working tool can still have confusing
66
+ descriptions or arguments that lead the agent to the wrong result.
67
+
68
+ You define what correctness means. The framework manages the Copilot session,
69
+ execution limits, usage, and evidence; it does not grade answers with another AI.
70
+
71
+ ## Quick start
72
+
73
+ You need Python 3.11+, [uv](https://docs.astral.sh/uv/), and a GitHub account with
74
+ Copilot access.
75
+
76
+ Run these commands in an existing Python project with a `pyproject.toml`:
77
+
78
+ ```bash
79
+ uv add pytest-copilot
80
+ gh auth login --hostname github.com
81
+ uv run pytest-copilot init
82
+ uv run pytest-copilot doctor
83
+ uv run python -m pytest tests/test_copilot_run.py -v
84
+ ```
85
+
86
+ `init` creates a test using the bundled Todo MCP server, configures JSON output,
87
+ and adds starter-model rates to `pricing.toml`. It reports conflicts rather than
88
+ overwriting existing files or settings. `doctor` checks setup and Copilot access.
89
+ Real task execution may consume Copilot premium requests.
90
+
91
+ The test checks the tool's arguments and returned task state. pytest prints the
92
+ result; `aitest-reports/results.json` preserves the execution. See the
93
+ [setup guide](docs/getting-started/index.md) or the standalone
94
+ [quickstart project](examples/quickstart/).
95
+
96
+ ## A test that checks the result
97
+
98
+ ```python
99
+ from __future__ import annotations
100
+
101
+ import json
102
+
103
+ from pytest_copilot.copilot import CopilotTask
104
+
105
+
106
+ async def test_order_total(copilot_run, tmp_path):
107
+ agent = CopilotTask(
108
+ name="order-total",
109
+ model="gpt-5.6-sol",
110
+ instructions="Save the requested output as JSON. Use integer cents.",
111
+ working_directory=str(tmp_path),
112
+ )
113
+ result = await copilot_run(
114
+ agent,
115
+ "Save order.json for two items at 1250 cents each plus 20% tax. "
116
+ "Include subtotal_cents, tax_cents, and total_cents.",
117
+ )
118
+ assert result.success, result.error
119
+ output = tmp_path / "order.json"
120
+ assert output.is_file()
121
+ assert json.loads(output.read_text(encoding="utf-8")) == {
122
+ "subtotal_cents": 2500,
123
+ "tax_cents": 500,
124
+ "total_cents": 3000,
125
+ }
126
+ ```
127
+
128
+ `CopilotTask` configures the session. The second argument to `copilot_run` is
129
+ the user task. `result.success` checks execution completion; the file assertions
130
+ check that the task produced the right result.
131
+
132
+ For your own tools, check the resulting application state, saved files, or
133
+ operation output. A tool call or a model's claim is not proof of task success.
134
+ Use approved isolation for live tests: a temporary directory is not a sandbox,
135
+ and generated code or tool calls can act with your user's access.
136
+
137
+ ## What the framework provides
138
+
139
+ | You need | Use |
140
+ | --- | --- |
141
+ | Real task execution | `CopilotTask` and the `copilot_run` pytest fixture |
142
+ | Controlled runs | Timeouts, tool budgets, request controls, and permission settings |
143
+ | Debugging evidence | Configuration, calls, arguments, outputs, errors, and completion records |
144
+ | Repeatable comparisons | `ab_run` and pytest repetitions |
145
+ | Usage visibility | Captured tokens, Copilot premium requests, and explicit price estimates |
146
+ | Saved results | Native JSON with execution evidence and ordinary pytest outcomes |
147
+
148
+ Your tests supply application fixtures, guarded tool adapters, and independent
149
+ output checks. The framework makes one execution attempt per call; it does not
150
+ retry failures automatically. Missing usage, prices, and incomplete evidence
151
+ remain visible.
152
+
153
+ ## Investigate and compare
154
+
155
+ Give your coding agent a failed test, its JSON evidence, the requirements, and
156
+ the relevant source. It can investigate the failure and help you change the
157
+ interface. Keep the success criteria fixed when rerunning a comparison.
158
+
159
+ For A/B tests, record verification separately for each side: both saved
160
+ executions share one pytest outcome. That outcome alone cannot establish
161
+ independent pass rates or a causal improvement. See
162
+ [comparing configurations](docs/getting-started/comparing.md).
163
+
164
+ ## Documentation
165
+
166
+ - [Getting started](https://sbroenne.github.io/pytest-copilot/getting-started/)
167
+ - [Task guides](docs/how-to/index.md) and [API reference](docs/reference/api.md)
168
+ - [Examples](examples/README.md), including historical skill comparisons
169
+ - [Upgrading to 2.0](docs/migration.md): new package, imports, and task names,
170
+ without compatibility aliases
171
+
172
+ No framework companion skill is required or distributed. Testing your own
173
+ domain skills is supported.
174
+
175
+ ## License
176
+
177
+ MIT. Inspired by [agent-benchmark](https://github.com/mykhaliev/agent-benchmark).
@@ -0,0 +1,129 @@
1
+ # pytest-copilot
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/pytest-copilot)](https://pypi.org/project/pytest-copilot/)
4
+ [![CI](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml/badge.svg)](https://github.com/sbroenne/pytest-copilot/actions/workflows/ci.yml)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
6
+
7
+ **Test whether GitHub Copilot can complete tasks with your tools and instructions.**
8
+
9
+ Independently maintained; not an official GitHub product or endorsed by GitHub.
10
+
11
+ pytest-copilot runs real Copilot sessions from pytest. Give Copilot a
12
+ task, check the actual output with ordinary assertions, and save the execution
13
+ as JSON.
14
+
15
+ Use it to test MCP servers, CLI tools, skills, system prompts, plugins, and custom
16
+ agents. It answers a different question from your tool's unit tests: **can
17
+ Copilot use the interface correctly?** A working tool can still have confusing
18
+ descriptions or arguments that lead the agent to the wrong result.
19
+
20
+ You define what correctness means. The framework manages the Copilot session,
21
+ execution limits, usage, and evidence; it does not grade answers with another AI.
22
+
23
+ ## Quick start
24
+
25
+ You need Python 3.11+, [uv](https://docs.astral.sh/uv/), and a GitHub account with
26
+ Copilot access.
27
+
28
+ Run these commands in an existing Python project with a `pyproject.toml`:
29
+
30
+ ```bash
31
+ uv add pytest-copilot
32
+ gh auth login --hostname github.com
33
+ uv run pytest-copilot init
34
+ uv run pytest-copilot doctor
35
+ uv run python -m pytest tests/test_copilot_run.py -v
36
+ ```
37
+
38
+ `init` creates a test using the bundled Todo MCP server, configures JSON output,
39
+ and adds starter-model rates to `pricing.toml`. It reports conflicts rather than
40
+ overwriting existing files or settings. `doctor` checks setup and Copilot access.
41
+ Real task execution may consume Copilot premium requests.
42
+
43
+ The test checks the tool's arguments and returned task state. pytest prints the
44
+ result; `aitest-reports/results.json` preserves the execution. See the
45
+ [setup guide](docs/getting-started/index.md) or the standalone
46
+ [quickstart project](examples/quickstart/).
47
+
48
+ ## A test that checks the result
49
+
50
+ ```python
51
+ from __future__ import annotations
52
+
53
+ import json
54
+
55
+ from pytest_copilot.copilot import CopilotTask
56
+
57
+
58
+ async def test_order_total(copilot_run, tmp_path):
59
+ agent = CopilotTask(
60
+ name="order-total",
61
+ model="gpt-5.6-sol",
62
+ instructions="Save the requested output as JSON. Use integer cents.",
63
+ working_directory=str(tmp_path),
64
+ )
65
+ result = await copilot_run(
66
+ agent,
67
+ "Save order.json for two items at 1250 cents each plus 20% tax. "
68
+ "Include subtotal_cents, tax_cents, and total_cents.",
69
+ )
70
+ assert result.success, result.error
71
+ output = tmp_path / "order.json"
72
+ assert output.is_file()
73
+ assert json.loads(output.read_text(encoding="utf-8")) == {
74
+ "subtotal_cents": 2500,
75
+ "tax_cents": 500,
76
+ "total_cents": 3000,
77
+ }
78
+ ```
79
+
80
+ `CopilotTask` configures the session. The second argument to `copilot_run` is
81
+ the user task. `result.success` checks execution completion; the file assertions
82
+ check that the task produced the right result.
83
+
84
+ For your own tools, check the resulting application state, saved files, or
85
+ operation output. A tool call or a model's claim is not proof of task success.
86
+ Use approved isolation for live tests: a temporary directory is not a sandbox,
87
+ and generated code or tool calls can act with your user's access.
88
+
89
+ ## What the framework provides
90
+
91
+ | You need | Use |
92
+ | --- | --- |
93
+ | Real task execution | `CopilotTask` and the `copilot_run` pytest fixture |
94
+ | Controlled runs | Timeouts, tool budgets, request controls, and permission settings |
95
+ | Debugging evidence | Configuration, calls, arguments, outputs, errors, and completion records |
96
+ | Repeatable comparisons | `ab_run` and pytest repetitions |
97
+ | Usage visibility | Captured tokens, Copilot premium requests, and explicit price estimates |
98
+ | Saved results | Native JSON with execution evidence and ordinary pytest outcomes |
99
+
100
+ Your tests supply application fixtures, guarded tool adapters, and independent
101
+ output checks. The framework makes one execution attempt per call; it does not
102
+ retry failures automatically. Missing usage, prices, and incomplete evidence
103
+ remain visible.
104
+
105
+ ## Investigate and compare
106
+
107
+ Give your coding agent a failed test, its JSON evidence, the requirements, and
108
+ the relevant source. It can investigate the failure and help you change the
109
+ interface. Keep the success criteria fixed when rerunning a comparison.
110
+
111
+ For A/B tests, record verification separately for each side: both saved
112
+ executions share one pytest outcome. That outcome alone cannot establish
113
+ independent pass rates or a causal improvement. See
114
+ [comparing configurations](docs/getting-started/comparing.md).
115
+
116
+ ## Documentation
117
+
118
+ - [Getting started](https://sbroenne.github.io/pytest-copilot/getting-started/)
119
+ - [Task guides](docs/how-to/index.md) and [API reference](docs/reference/api.md)
120
+ - [Examples](examples/README.md), including historical skill comparisons
121
+ - [Upgrading to 2.0](docs/migration.md): new package, imports, and task names,
122
+ without compatibility aliases
123
+
124
+ No framework companion skill is required or distributed. Testing your own
125
+ domain skills is supported.
126
+
127
+ ## License
128
+
129
+ MIT. Inspired by [agent-benchmark](https://github.com/mykhaliev/agent-benchmark).
@@ -0,0 +1,146 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.32.4"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "pytest-copilot"
7
+ version = "2.0.0"
8
+ description = "Run GitHub Copilot tasks in pytest. Check results with your own assertions."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.11"
12
+ authors = [
13
+ { name = "Stefan Brunner" }
14
+ ]
15
+ keywords = ["pytest", "copilot", "ai", "llm", "mcp", "testing", "agents", "skills", "custom-agents"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Framework :: Pytest",
19
+ "Intended Audience :: Developers",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Operating System :: OS Independent",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Topic :: Software Development :: Testing",
27
+ ]
28
+ dependencies = [
29
+ "pytest>=9.2.0",
30
+ "mcp>=2.3.0",
31
+ "pydantic>=2.14.1",
32
+ "python-frontmatter>=1.3.0",
33
+ "github-copilot-sdk>=1.0.28",
34
+ "httpx>=0.28.1",
35
+ "trio>=0.34.0",
36
+ "tomlkit>=0.15.2",
37
+ "pytest-asyncio>=1.4.1",
38
+ "websockets>=17.3",
39
+ ]
40
+
41
+ [project.optional-dependencies]
42
+ dev = [
43
+ "pytest-cov>=7.1.0",
44
+ "python-dotenv>=1.2.4",
45
+ "ruff>=0.17.3",
46
+ "pyright>=1.1.414",
47
+ "pre-commit>=4.7.0",
48
+ "typeguard>=4.6.0",
49
+ ]
50
+ test = [
51
+ "syrupy>=6.1.1,<7",
52
+ ]
53
+ docs = [
54
+ "mkdocs>=1.6.1",
55
+ "mkdocs-material>=9.7.7,<9.9",
56
+ "mkdocstrings[python]>=1.0.8",
57
+ "pillow>=13.0.0",
58
+ "cairosvg>=2.9.1",
59
+ ]
60
+
61
+ [project.scripts]
62
+ pytest-copilot = "pytest_copilot.onboarding:main"
63
+
64
+ [project.entry-points.pytest11]
65
+ aitest = "pytest_copilot.plugin"
66
+
67
+ [project.urls]
68
+ Homepage = "https://github.com/sbroenne/pytest-copilot"
69
+ Repository = "https://github.com/sbroenne/pytest-copilot"
70
+
71
+ [tool.hatch.build.targets.wheel]
72
+ packages = ["src/pytest_copilot"]
73
+
74
+ [tool.hatch.build.targets.wheel.sources]
75
+ "src" = ""
76
+
77
+ [tool.hatch.build]
78
+ include = [
79
+ "src/pytest_copilot/**/*.py",
80
+ "src/pytest_copilot/**/*.md",
81
+ ]
82
+
83
+ [tool.ruff]
84
+ line-length = 100
85
+ target-version = "py311"
86
+
87
+ [tool.ruff.lint]
88
+ select = ["E", "F", "B", "I"]
89
+
90
+ [tool.ruff.lint.per-file-ignores]
91
+ # Test harnesses have long tool descriptions for LLM readability
92
+ "src/pytest_copilot/testing/*.py" = ["E501"]
93
+ # Test files have long descriptions and variable declarations
94
+ "tests/**/*.py" = ["E402", "E501", "F841"]
95
+
96
+ [tool.ruff.format]
97
+ quote-style = "double"
98
+ indent-style = "space"
99
+
100
+ [tool.pyright]
101
+ include = ["src", "tests", "examples/skill-dogfood", "examples/excel-skill-value"]
102
+ pythonVersion = "3.11"
103
+ typeCheckingMode = "basic"
104
+
105
+ [tool.pytest.ini_options]
106
+ testpaths = ["tests"]
107
+ python_files = ["test_*.py"]
108
+ python_classes = ["Test*"]
109
+ python_functions = ["test_*"]
110
+ asyncio_mode = "auto"
111
+ # Suppress third-party deprecation warnings we can't fix
112
+ filterwarnings = [
113
+ "ignore:enable_cleanup_closed:DeprecationWarning:aiohttp",
114
+ ]
115
+ # pytest-copilot configuration for this project's own tests.
116
+ # This demonstrates the recommended setup - configure once in pyproject.toml.
117
+ # LLM auth is handled by the GitHub Copilot SDK (gh auth login or GITHUB_TOKEN)
118
+ addopts = """
119
+ --strict-markers
120
+ """
121
+ markers = [
122
+ "integration: marks tests as integration tests (require LLM credentials)",
123
+ "aitest: AI agent tests",
124
+ "basic: basic usage tests (banking and todo)",
125
+ "model: tests with multiple models",
126
+ "sysprompt: tests with multiple system prompts",
127
+ "matrix: model × prompt cross-product tests",
128
+ "skill: tests with skills",
129
+ "session_test: multi-turn session tests",
130
+ "clarification: clarification detection tests",
131
+ "iterations: iteration reliability tests",
132
+ "abtest: A/B server comparison tests",
133
+ "cli: command-line tool tests",
134
+ "copilot: marks tests as requiring GitHub Copilot SDK credentials",
135
+ "slow: marks tests as slow/expensive (skipped by default, opt-in with --run-slow)",
136
+ ]
137
+
138
+ # UV/PEP 735 dependency groups (for uv sync)
139
+ [dependency-groups]
140
+ dev = [
141
+ "pytest-copilot[dev,test,docs]",
142
+ "pytest-copilot-dogfood",
143
+ ]
144
+
145
+ [tool.uv.sources]
146
+ pytest-copilot-dogfood = { path = "examples/skill-dogfood", editable = true }
@@ -0,0 +1,133 @@
1
+ """Copilot task execution and evidence for testing AI-facing tools and skills."""
2
+
3
+ import logging
4
+
5
+ # Configure library logging per Python best practices:
6
+ # https://docs.python.org/3/howto/logging.html#configuring-logging-for-a-library
7
+ # Libraries should add NullHandler to prevent "No handler found" warnings
8
+ # when the application hasn't configured logging.
9
+ logging.getLogger(__name__).addHandler(logging.NullHandler())
10
+
11
+ # Core types # noqa: E402 - logging must be configured before submodule imports
12
+ # Copilot coding agent support (now required)
13
+ from pytest_copilot.copilot import ( # noqa: E402
14
+ ClaudeCodePersona,
15
+ CopilotCLIPersona,
16
+ CopilotResult,
17
+ CopilotTask,
18
+ HeadlessPersona,
19
+ Persona,
20
+ VSCodePersona,
21
+ run_copilot,
22
+ )
23
+ from pytest_copilot.core import ( # noqa: E402
24
+ AITestError,
25
+ EngineTimeoutError,
26
+ EvalResult,
27
+ HookDefinition,
28
+ ImageContent,
29
+ MCPPrompt,
30
+ MCPPromptArgument,
31
+ Plugin,
32
+ PluginMetadata,
33
+ Prompt,
34
+ ServerStartError,
35
+ Skill,
36
+ SkillError,
37
+ SkillEvalCase,
38
+ SkillInfo,
39
+ SkillMetadata,
40
+ SubagentInvocation,
41
+ ToolCall,
42
+ ToolInfo,
43
+ Turn,
44
+ export_grading,
45
+ has_skill_evals,
46
+ load_custom_agent,
47
+ load_custom_agents,
48
+ load_instruction_file,
49
+ load_instruction_files,
50
+ load_plugin,
51
+ load_prompt,
52
+ load_prompt_file,
53
+ load_prompt_files,
54
+ load_prompts,
55
+ load_skill,
56
+ load_skill_evals,
57
+ load_system_prompts,
58
+ )
59
+ from pytest_copilot.execution import ( # noqa: E402
60
+ MCPServer,
61
+ Wait,
62
+ WaitStrategy,
63
+ )
64
+
65
+ # Reporting
66
+ from pytest_copilot.reporting import ( # noqa: E402
67
+ SuiteReport,
68
+ TestReport,
69
+ build_suite_report,
70
+ generate_json,
71
+ load_suite_report,
72
+ )
73
+
74
+ __all__ = [ # noqa: RUF022
75
+ # Core
76
+ "AITestError",
77
+ "EngineTimeoutError",
78
+ "EvalResult",
79
+ "HookDefinition",
80
+ "ImageContent",
81
+ "MCPPrompt",
82
+ "MCPPromptArgument",
83
+ "Plugin",
84
+ "PluginMetadata",
85
+ "Prompt",
86
+ "ServerStartError",
87
+ "Skill",
88
+ "SkillError",
89
+ "SkillEvalCase",
90
+ "SkillInfo",
91
+ "SkillMetadata",
92
+ "SubagentInvocation",
93
+ "ToolCall",
94
+ "ToolInfo",
95
+ "Turn",
96
+ "export_grading",
97
+ "has_skill_evals",
98
+ "load_custom_agent",
99
+ "load_custom_agents",
100
+ "load_instruction_file",
101
+ "load_instruction_files",
102
+ "load_plugin",
103
+ "load_prompt",
104
+ "load_prompt_file",
105
+ "load_prompt_files",
106
+ "load_prompts",
107
+ "load_skill",
108
+ "load_skill_evals",
109
+ "load_system_prompts",
110
+ # Copilot (primary eval harness)
111
+ "ClaudeCodePersona",
112
+ "CopilotCLIPersona",
113
+ "CopilotTask",
114
+ "CopilotResult",
115
+ "HeadlessPersona",
116
+ "Persona",
117
+ "VSCodePersona",
118
+ "run_copilot",
119
+ # Server config types
120
+ "MCPServer",
121
+ "Wait",
122
+ "WaitStrategy",
123
+ # Reporting
124
+ "SuiteReport",
125
+ "TestReport",
126
+ "build_suite_report",
127
+ "generate_json",
128
+ "load_suite_report",
129
+ ]
130
+
131
+ from importlib.metadata import version as _get_version # noqa: E402
132
+
133
+ __version__ = _get_version("pytest-copilot")