pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
pytest_copilot/plugin.py
ADDED
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
"""pytest plugin for aitest."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from pytest_copilot.plugin_options import add_aitest_options
|
|
13
|
+
from pytest_copilot.plugin_report import (
|
|
14
|
+
log_report_path,
|
|
15
|
+
)
|
|
16
|
+
from pytest_copilot.reporting import (
|
|
17
|
+
TestReport,
|
|
18
|
+
build_suite_report,
|
|
19
|
+
generate_json,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from _pytest.config import Config
|
|
24
|
+
from _pytest.config.argparsing import Parser
|
|
25
|
+
from _pytest.nodes import Item
|
|
26
|
+
from _pytest.reports import TestReport as PytestTestReport
|
|
27
|
+
from _pytest.terminal import TerminalReporter
|
|
28
|
+
|
|
29
|
+
from pytest_copilot.core.result import EvalResult
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
_logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# Key for storing test reports in config
|
|
36
|
+
COLLECTOR_KEY = pytest.StashKey[list[TestReport]]()
|
|
37
|
+
# Key for storing session messages for @pytest.mark.session
|
|
38
|
+
SESSION_MESSAGES_KEY = pytest.StashKey[dict[str, list[dict[str, Any]]]]()
|
|
39
|
+
PHASE_REPORTS_KEY = pytest.StashKey[dict[str, "PytestTestReport"]]()
|
|
40
|
+
# Export for use in fixtures and downstream consumers
|
|
41
|
+
__all__ = [
|
|
42
|
+
"COLLECTOR_KEY",
|
|
43
|
+
"SESSION_MESSAGES_KEY",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _get_timestamped_path(
|
|
48
|
+
base_name: str, test_name: str | None = None, default_dir: Path | None = None
|
|
49
|
+
) -> Path:
|
|
50
|
+
"""Generate timestamped filename for unique report names.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
base_name: Base evidence filename with extension (e.g., 'results.json')
|
|
54
|
+
test_name: Name of the test/suite to include in filename
|
|
55
|
+
default_dir: Directory to store the file (default: 'aitest-reports')
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Path with format: {dir}/{prefix}_{test_name}_{ISO8601-timestamp}.{ext}
|
|
59
|
+
"""
|
|
60
|
+
if default_dir is None:
|
|
61
|
+
default_dir = Path("aitest-reports")
|
|
62
|
+
|
|
63
|
+
# Use ISO8601 timestamp: YYYY-MM-DDTHH-MM-SS (seconds precision, : replaced with -)
|
|
64
|
+
timestamp = datetime.now().isoformat(timespec="seconds").replace(":", "-")
|
|
65
|
+
|
|
66
|
+
# Sanitize test name (remove paths, lowercase, replace spaces/special chars)
|
|
67
|
+
if test_name:
|
|
68
|
+
# Remove file extensions and paths
|
|
69
|
+
safe_name = (
|
|
70
|
+
test_name.split("/")[-1].split(".")[0].lower().replace(" ", "-").replace("_", "-")
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
safe_name = None
|
|
74
|
+
|
|
75
|
+
# Split filename and extension
|
|
76
|
+
if "." in base_name:
|
|
77
|
+
name_part, ext = base_name.rsplit(".", 1)
|
|
78
|
+
if safe_name:
|
|
79
|
+
filename = f"{name_part}_{safe_name}_{timestamp}.{ext}"
|
|
80
|
+
else:
|
|
81
|
+
filename = f"{name_part}_{timestamp}.{ext}"
|
|
82
|
+
else:
|
|
83
|
+
if safe_name:
|
|
84
|
+
filename = f"{base_name}_{safe_name}_{timestamp}"
|
|
85
|
+
else:
|
|
86
|
+
filename = f"{base_name}_{timestamp}"
|
|
87
|
+
|
|
88
|
+
return default_dir / filename
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def pytest_addoption(parser: Parser) -> None:
|
|
92
|
+
"""Add pytest CLI options for aitest.
|
|
93
|
+
|
|
94
|
+
Note: Copilot-authenticated runs use the GitHub CLI or ``GITHUB_TOKEN``.
|
|
95
|
+
"""
|
|
96
|
+
group = parser.getgroup("aitest", "AI agent testing")
|
|
97
|
+
add_aitest_options(group)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def pytest_configure(config: Config) -> None:
|
|
101
|
+
"""Configure the aitest plugin."""
|
|
102
|
+
if not config.pluginmanager.hasplugin("pytest_copilot.fixtures"):
|
|
103
|
+
config.pluginmanager.import_plugin("pytest_copilot.fixtures")
|
|
104
|
+
|
|
105
|
+
# Register markers
|
|
106
|
+
config.addinivalue_line(
|
|
107
|
+
"markers",
|
|
108
|
+
"aitest: Mark test as an AI agent test (optional, enables filtering with -m aitest)",
|
|
109
|
+
)
|
|
110
|
+
config.addinivalue_line(
|
|
111
|
+
"markers",
|
|
112
|
+
"aitest_skip_report: Exclude this test from AI test reports",
|
|
113
|
+
)
|
|
114
|
+
config.addinivalue_line(
|
|
115
|
+
"markers",
|
|
116
|
+
"session(name): Mark tests as part of a named session for multi-turn conversations. "
|
|
117
|
+
"Tests with the same session name share conversation history automatically.",
|
|
118
|
+
)
|
|
119
|
+
config.addinivalue_line(
|
|
120
|
+
"markers",
|
|
121
|
+
"copilot: mark test as requiring GitHub Copilot SDK credentials",
|
|
122
|
+
)
|
|
123
|
+
config.addinivalue_line(
|
|
124
|
+
"markers",
|
|
125
|
+
"aitest_iteration_axis(position): internal repetition parameter-group index",
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
# Always initialize report collection - JSON is always generated
|
|
129
|
+
config.stash[COLLECTOR_KEY] = []
|
|
130
|
+
# Initialize session message storage
|
|
131
|
+
config.stash[SESSION_MESSAGES_KEY] = {}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
|
135
|
+
"""Parametrize tests with iteration index when ``--aitest-iterations`` > 1.
|
|
136
|
+
|
|
137
|
+
Follows the same pattern as *pytest-repeat*: the fixture name is
|
|
138
|
+
appended to ``metafunc.fixturenames`` so every test function
|
|
139
|
+
receives the parameter even though it does not declare the fixture
|
|
140
|
+
explicitly.
|
|
141
|
+
"""
|
|
142
|
+
count_option = metafunc.config.getoption("--aitest-iterations", default=1)
|
|
143
|
+
count = count_option if isinstance(count_option, int) else 1
|
|
144
|
+
if count <= 1:
|
|
145
|
+
return
|
|
146
|
+
metafunc.fixturenames.append("_aitest_iteration")
|
|
147
|
+
existing = getattr(metafunc, "_calls", [])
|
|
148
|
+
position = len(existing[0]._idlist) if existing else 0
|
|
149
|
+
metafunc.parametrize(
|
|
150
|
+
"_aitest_iteration",
|
|
151
|
+
[
|
|
152
|
+
pytest.param(i, marks=pytest.mark.aitest_iteration_axis(position))
|
|
153
|
+
for i in range(1, count + 1)
|
|
154
|
+
],
|
|
155
|
+
ids=[f"iter-{i}" for i in range(1, count + 1)],
|
|
156
|
+
indirect=True,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def pytest_collection_modifyitems(
|
|
161
|
+
session: pytest.Session,
|
|
162
|
+
config: Config,
|
|
163
|
+
items: list[pytest.Item],
|
|
164
|
+
) -> None:
|
|
165
|
+
"""Auto-mark tests that use aitest fixtures."""
|
|
166
|
+
for item in items:
|
|
167
|
+
# Check if test uses any aitest fixtures
|
|
168
|
+
fixturenames = getattr(item, "fixturenames", [])
|
|
169
|
+
aitest_fixtures = {"copilot_run"}
|
|
170
|
+
if (aitest_fixtures & set(fixturenames)) and not any(
|
|
171
|
+
m.name == "aitest" for m in item.iter_markers()
|
|
172
|
+
):
|
|
173
|
+
item.add_marker(pytest.mark.aitest)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _require_agent_string(agent: Any, *, field_name: str) -> str:
|
|
177
|
+
"""Read a required string from the reporting wrapper."""
|
|
178
|
+
value = getattr(agent, field_name, None)
|
|
179
|
+
if isinstance(value, str) and value:
|
|
180
|
+
return value
|
|
181
|
+
msg = f"aitest reporting agent is missing required field '{field_name}'"
|
|
182
|
+
raise ValueError(msg)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _require_provider_model(agent: Any) -> str | None:
|
|
186
|
+
"""Read the required provider model from the reporting wrapper."""
|
|
187
|
+
provider = getattr(agent, "provider", None)
|
|
188
|
+
if provider is None or not hasattr(provider, "model"):
|
|
189
|
+
raise ValueError("aitest reporting agent is missing required field 'provider.model'")
|
|
190
|
+
model = provider.model
|
|
191
|
+
if model is None or isinstance(model, str) and model:
|
|
192
|
+
return model
|
|
193
|
+
raise ValueError("aitest reporting agent is missing required field 'provider.model'")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _display_model(model: str | None) -> str | None:
|
|
197
|
+
"""Strip any provider prefix from a model name for display."""
|
|
198
|
+
return model.split("/")[-1] if model is not None and "/" in model else model
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _build_agent_identity(
|
|
202
|
+
agent: Any,
|
|
203
|
+
eval_result: EvalResult,
|
|
204
|
+
) -> tuple[str, str, str | None, str | None, str | None]:
|
|
205
|
+
"""Resolve stable and human-facing identity fields for report production."""
|
|
206
|
+
base_agent_id = _require_agent_string(agent, field_name="id")
|
|
207
|
+
eval_name = _require_agent_string(agent, field_name="name")
|
|
208
|
+
model_raw = _require_provider_model(agent)
|
|
209
|
+
system_prompt_name = getattr(agent, "system_prompt_name", None)
|
|
210
|
+
if system_prompt_name is not None and not isinstance(system_prompt_name, str):
|
|
211
|
+
raise ValueError("aitest reporting agent field 'system_prompt_name' must be a string")
|
|
212
|
+
|
|
213
|
+
skill_name = eval_result.skill_info.name if eval_result.skill_info else None
|
|
214
|
+
if skill_name is None:
|
|
215
|
+
skill = getattr(agent, "skill", None)
|
|
216
|
+
if skill is not None:
|
|
217
|
+
skill_name = _require_agent_string(skill, field_name="name")
|
|
218
|
+
|
|
219
|
+
return (
|
|
220
|
+
base_agent_id,
|
|
221
|
+
eval_name,
|
|
222
|
+
_display_model(model_raw),
|
|
223
|
+
system_prompt_name,
|
|
224
|
+
skill_name,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _build_case_name(item: Item) -> str:
|
|
229
|
+
"""Build a case identity that removes only the synthetic iteration axis."""
|
|
230
|
+
callspec = getattr(item, "callspec", None)
|
|
231
|
+
if callspec is None or "_aitest_iteration" not in callspec.params:
|
|
232
|
+
return item.nodeid
|
|
233
|
+
|
|
234
|
+
param_ids = list(getattr(callspec, "_idlist", []))
|
|
235
|
+
axis = item.get_closest_marker("aitest_iteration_axis")
|
|
236
|
+
if axis is None or len(axis.args) != 1 or type(axis.args[0]) is not int:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
"Unable to derive case identity for "
|
|
239
|
+
f"{item.nodeid!r}: repetition parameter-group identity is unavailable"
|
|
240
|
+
)
|
|
241
|
+
position = axis.args[0]
|
|
242
|
+
if not 0 <= position < len(param_ids):
|
|
243
|
+
raise ValueError(f"Invalid repetition parameter-group index for {item.nodeid!r}")
|
|
244
|
+
del param_ids[position]
|
|
245
|
+
base_nodeid = item.nodeid.removesuffix(f"[{callspec.id}]")
|
|
246
|
+
if not param_ids:
|
|
247
|
+
return base_nodeid
|
|
248
|
+
return f"{base_nodeid}[{'-'.join(param_ids)}]"
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
|
252
|
+
def pytest_runtest_makereport(item: Item, call: Any) -> Any:
|
|
253
|
+
"""Capture test results for reporting.
|
|
254
|
+
|
|
255
|
+
Also auto-stashes CopilotResult for tests that call ``run_copilot()``
|
|
256
|
+
directly instead of using the ``copilot_run`` fixture — critical for
|
|
257
|
+
module-scoped agent fixtures that cannot use the function-scoped fixture.
|
|
258
|
+
"""
|
|
259
|
+
# Auto-stash CopilotResult before processing (tryfirst ensures this runs early)
|
|
260
|
+
if call.when in ("setup", "call") and not hasattr(item, "_aitest_result"):
|
|
261
|
+
from pytest_copilot.copilot.fixtures import stash_on_item
|
|
262
|
+
from pytest_copilot.copilot.result import CopilotResult
|
|
263
|
+
from pytest_copilot.copilot.task import CopilotTask
|
|
264
|
+
|
|
265
|
+
funcargs = getattr(item, "funcargs", {})
|
|
266
|
+
for val in funcargs.values():
|
|
267
|
+
if isinstance(val, CopilotResult) and val.agent is not None:
|
|
268
|
+
stash_on_item(item, cast(CopilotTask, val.agent), val)
|
|
269
|
+
break
|
|
270
|
+
|
|
271
|
+
outcome = yield
|
|
272
|
+
report: PytestTestReport = outcome.get_result()
|
|
273
|
+
|
|
274
|
+
phases = item.stash.setdefault(PHASE_REPORTS_KEY, {})
|
|
275
|
+
phases[report.when] = report
|
|
276
|
+
|
|
277
|
+
# Check if reporting is enabled
|
|
278
|
+
tests = item.config.stash.get(COLLECTOR_KEY, None)
|
|
279
|
+
if tests is None:
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
# Skip if marked to exclude from report
|
|
283
|
+
if any(m.name == "aitest_skip_report" for m in item.iter_markers()):
|
|
284
|
+
return
|
|
285
|
+
|
|
286
|
+
runs = getattr(item, "_aitest_runs", [])
|
|
287
|
+
|
|
288
|
+
# Only collect tests that actually used aitest (have an agent result)
|
|
289
|
+
# Tests without eval execution do not create eval reports.
|
|
290
|
+
if not runs:
|
|
291
|
+
return
|
|
292
|
+
if report.when != "teardown":
|
|
293
|
+
if report.when == "call":
|
|
294
|
+
for eval_result, agent in runs:
|
|
295
|
+
_add_junit_properties(report, eval_result, agent)
|
|
296
|
+
return
|
|
297
|
+
failed_phases = [phase for phase in phases.values() if phase.failed]
|
|
298
|
+
skipped_phases = [phase for phase in phases.values() if phase.skipped]
|
|
299
|
+
final_outcome = "failed" if failed_phases else "skipped" if skipped_phases else "passed"
|
|
300
|
+
|
|
301
|
+
# Get test function docstring if available
|
|
302
|
+
docstring = None
|
|
303
|
+
func = getattr(item, "function", None)
|
|
304
|
+
if func is not None and func.__doc__:
|
|
305
|
+
docstring = func.__doc__
|
|
306
|
+
|
|
307
|
+
# Get test class docstring if available
|
|
308
|
+
class_docstring = None
|
|
309
|
+
parent = getattr(item, "parent", None)
|
|
310
|
+
if parent is not None:
|
|
311
|
+
parent_obj = getattr(parent, "obj", None)
|
|
312
|
+
if parent_obj is not None and hasattr(parent_obj, "__doc__") and parent_obj.__doc__:
|
|
313
|
+
# Only use class docstrings (not module docstrings)
|
|
314
|
+
import inspect
|
|
315
|
+
|
|
316
|
+
if inspect.isclass(parent_obj):
|
|
317
|
+
class_docstring = parent_obj.__doc__
|
|
318
|
+
|
|
319
|
+
# Capture error message — just the assertion/exception, never raw tracebacks.
|
|
320
|
+
# Tracebacks contain file paths, line numbers, and nodeids that pollute
|
|
321
|
+
# User-facing reports.
|
|
322
|
+
phase_errors = []
|
|
323
|
+
for failed_report in failed_phases:
|
|
324
|
+
error_msg = None
|
|
325
|
+
error_text = str(failed_report.longrepr)
|
|
326
|
+
error_lines = error_text.split("\n")
|
|
327
|
+
|
|
328
|
+
# Extract lines starting with "E " — pytest's assertion/exception lines
|
|
329
|
+
e_lines = [line.strip()[2:] for line in error_lines if line.strip().startswith("E ")]
|
|
330
|
+
|
|
331
|
+
if e_lines:
|
|
332
|
+
error_msg = "\n".join(e_lines)
|
|
333
|
+
else:
|
|
334
|
+
# No E-lines: grab the last non-empty line (typically "ExceptionType: message")
|
|
335
|
+
for line in reversed(error_lines):
|
|
336
|
+
stripped = line.strip()
|
|
337
|
+
if stripped:
|
|
338
|
+
error_msg = stripped
|
|
339
|
+
break
|
|
340
|
+
if error_msg is not None:
|
|
341
|
+
phase_errors.append(
|
|
342
|
+
error_msg if failed_report.when == "call" else f"{failed_report.when}: {error_msg}"
|
|
343
|
+
)
|
|
344
|
+
error_msg = "\n".join(phase_errors) or None
|
|
345
|
+
|
|
346
|
+
# Detect iteration index from _aitest_iteration fixture
|
|
347
|
+
iteration: int | None = None
|
|
348
|
+
callspec = getattr(item, "callspec", None)
|
|
349
|
+
if callspec and "_aitest_iteration" in callspec.params:
|
|
350
|
+
iteration = callspec.params["_aitest_iteration"]
|
|
351
|
+
|
|
352
|
+
properties = list(report.user_properties)
|
|
353
|
+
for eval_result, agent in runs:
|
|
354
|
+
agent_id, eval_name, model, system_prompt_name, skill_name = _build_agent_identity(
|
|
355
|
+
agent, eval_result
|
|
356
|
+
)
|
|
357
|
+
test_report = TestReport(
|
|
358
|
+
name=_build_case_name(item),
|
|
359
|
+
outcome=final_outcome,
|
|
360
|
+
duration_ms=(
|
|
361
|
+
eval_result.duration_ms
|
|
362
|
+
if len(runs) > 1
|
|
363
|
+
else sum(phase.duration for phase in phases.values()) * 1000
|
|
364
|
+
),
|
|
365
|
+
eval_result=eval_result,
|
|
366
|
+
error=error_msg,
|
|
367
|
+
properties=properties,
|
|
368
|
+
docstring=docstring,
|
|
369
|
+
class_docstring=class_docstring,
|
|
370
|
+
agent_id=agent_id,
|
|
371
|
+
eval_name=eval_name,
|
|
372
|
+
model=model,
|
|
373
|
+
system_prompt_name=system_prompt_name,
|
|
374
|
+
skill_name=skill_name,
|
|
375
|
+
iteration=iteration,
|
|
376
|
+
)
|
|
377
|
+
tests.append(test_report)
|
|
378
|
+
_add_junit_properties(report, eval_result, agent)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _add_junit_properties(
|
|
382
|
+
report: PytestTestReport,
|
|
383
|
+
eval_result: EvalResult,
|
|
384
|
+
agent: Any | None = None,
|
|
385
|
+
) -> None:
|
|
386
|
+
"""Add agent metadata to pytest report for JUnit XML output.
|
|
387
|
+
|
|
388
|
+
Properties are added to report.user_properties which pytest writes
|
|
389
|
+
as <property> elements in JUnit XML output.
|
|
390
|
+
|
|
391
|
+
Example output:
|
|
392
|
+
<testcase name="test_balance">
|
|
393
|
+
<properties>
|
|
394
|
+
<property name="aitest.agent.name" value="banking-agent"/>
|
|
395
|
+
<property name="aitest.model" value="gpt-5.6-sol"/>
|
|
396
|
+
<property name="aitest.skill" value="financial-advisor"/>
|
|
397
|
+
<property name="aitest.tools.called" value="get_balance,transfer"/>
|
|
398
|
+
</properties>
|
|
399
|
+
</testcase>
|
|
400
|
+
"""
|
|
401
|
+
if not hasattr(report, "user_properties"):
|
|
402
|
+
return
|
|
403
|
+
|
|
404
|
+
props = []
|
|
405
|
+
|
|
406
|
+
# Eval identity (from Eval object)
|
|
407
|
+
if agent:
|
|
408
|
+
agent_id, eval_name, model, system_prompt_name, _ = _build_agent_identity(
|
|
409
|
+
agent, eval_result
|
|
410
|
+
)
|
|
411
|
+
props.append(("aitest.agent.name", eval_name))
|
|
412
|
+
if model is not None:
|
|
413
|
+
props.append(("aitest.model", model))
|
|
414
|
+
if system_prompt_name:
|
|
415
|
+
props.append(("aitest.prompt", system_prompt_name))
|
|
416
|
+
|
|
417
|
+
# Skill
|
|
418
|
+
if eval_result.skill_info:
|
|
419
|
+
props.append(("aitest.skill", eval_result.skill_info.name))
|
|
420
|
+
|
|
421
|
+
# MCP servers (from agent config)
|
|
422
|
+
if agent and agent.mcp_servers:
|
|
423
|
+
server_names = []
|
|
424
|
+
for server in agent.mcp_servers:
|
|
425
|
+
# Use server.name if available, otherwise derive from command
|
|
426
|
+
name = getattr(server, "name", None)
|
|
427
|
+
if not name and hasattr(server, "command") and server.command:
|
|
428
|
+
name = server.command[-1].split("/")[-1].split(".")[0]
|
|
429
|
+
if name:
|
|
430
|
+
server_names.append(name)
|
|
431
|
+
if server_names:
|
|
432
|
+
props.append(("aitest.servers", ",".join(server_names)))
|
|
433
|
+
|
|
434
|
+
# Allowed tools filter (from agent config)
|
|
435
|
+
if agent and agent.allowed_tools:
|
|
436
|
+
props.append(("aitest.allowed_tools", ",".join(sorted(agent.allowed_tools))))
|
|
437
|
+
|
|
438
|
+
# Token usage
|
|
439
|
+
if eval_result.token_usage:
|
|
440
|
+
prompt = eval_result.token_usage.get("prompt", 0)
|
|
441
|
+
completion = eval_result.token_usage.get("completion", 0)
|
|
442
|
+
if prompt:
|
|
443
|
+
props.append(("aitest.tokens.input", str(prompt)))
|
|
444
|
+
if completion:
|
|
445
|
+
props.append(("aitest.tokens.output", str(completion)))
|
|
446
|
+
total = prompt + completion
|
|
447
|
+
if total:
|
|
448
|
+
props.append(("aitest.tokens.total", str(total)))
|
|
449
|
+
|
|
450
|
+
# Cost
|
|
451
|
+
if eval_result.cost_usd > 0:
|
|
452
|
+
props.append(("aitest.cost_usd", f"{eval_result.cost_usd:.6f}"))
|
|
453
|
+
|
|
454
|
+
# Turns
|
|
455
|
+
if eval_result.turns:
|
|
456
|
+
props.append(("aitest.turns", str(len(eval_result.turns))))
|
|
457
|
+
|
|
458
|
+
# Tools called (unique, comma-separated)
|
|
459
|
+
tools_called = set()
|
|
460
|
+
for turn in eval_result.turns:
|
|
461
|
+
for tc in turn.tool_calls:
|
|
462
|
+
tools_called.add(tc.name)
|
|
463
|
+
if tools_called:
|
|
464
|
+
props.append(("aitest.tools.called", ",".join(sorted(tools_called))))
|
|
465
|
+
|
|
466
|
+
# Success/failure
|
|
467
|
+
props.append(("aitest.success", str(eval_result.success).lower()))
|
|
468
|
+
|
|
469
|
+
# Add all properties to report
|
|
470
|
+
report.user_properties.extend(props)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
|
|
474
|
+
"""Save execution evidence and enforce the observed pytest pass-rate threshold."""
|
|
475
|
+
config = session.config
|
|
476
|
+
tests = config.stash.get(COLLECTOR_KEY, None)
|
|
477
|
+
|
|
478
|
+
if tests is None or not tests:
|
|
479
|
+
return
|
|
480
|
+
|
|
481
|
+
json_path = config.getoption("--aitest-json")
|
|
482
|
+
min_pass_rate: int | None = config.getoption("--aitest-min-pass-rate")
|
|
483
|
+
|
|
484
|
+
# Extract suite docstring from first test's parent class/module
|
|
485
|
+
suite_docstring = None
|
|
486
|
+
if session.items:
|
|
487
|
+
first_item = session.items[0]
|
|
488
|
+
# Try to get docstring from test class first
|
|
489
|
+
if hasattr(first_item, "parent") and first_item.parent:
|
|
490
|
+
parent = first_item.parent
|
|
491
|
+
# Check if parent is a class
|
|
492
|
+
parent_obj = getattr(parent, "obj", None)
|
|
493
|
+
if parent_obj and hasattr(parent_obj, "__doc__"):
|
|
494
|
+
suite_docstring = parent_obj.__doc__
|
|
495
|
+
if suite_docstring:
|
|
496
|
+
# Get first line only
|
|
497
|
+
suite_docstring = suite_docstring.strip().split("\n")[0].strip()
|
|
498
|
+
|
|
499
|
+
# Build suite report first (to get the test name for default filenames)
|
|
500
|
+
default_dir = Path("aitest-reports")
|
|
501
|
+
suite_report = build_suite_report(
|
|
502
|
+
tests,
|
|
503
|
+
name=session.name or "pytest-copilot",
|
|
504
|
+
suite_docstring=suite_docstring,
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# Generate default paths with test name included
|
|
508
|
+
default_json_path = _get_timestamped_path(
|
|
509
|
+
"results.json", test_name=suite_report.name, default_dir=default_dir
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
json_output_path = Path(json_path) if json_path else default_json_path
|
|
513
|
+
try:
|
|
514
|
+
json_output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
515
|
+
generate_json(suite_report, json_output_path)
|
|
516
|
+
except (OSError, TypeError, ValueError) as error:
|
|
517
|
+
if session.exitstatus == pytest.ExitCode.OK:
|
|
518
|
+
session.exitstatus = pytest.ExitCode.TESTS_FAILED
|
|
519
|
+
terminalreporter = config.pluginmanager.get_plugin("terminalreporter")
|
|
520
|
+
if terminalreporter:
|
|
521
|
+
terminalreporter.write_line(
|
|
522
|
+
f"\naitest: Failed to save execution evidence to {json_output_path}: {error}",
|
|
523
|
+
red=True,
|
|
524
|
+
bold=True,
|
|
525
|
+
)
|
|
526
|
+
_logger.error(
|
|
527
|
+
"Failed to save execution evidence to %s; "
|
|
528
|
+
"continuing with pass-rate enforcement and cleanup",
|
|
529
|
+
json_output_path,
|
|
530
|
+
exc_info=True,
|
|
531
|
+
)
|
|
532
|
+
else:
|
|
533
|
+
log_report_path(config, "JSON evidence", json_output_path)
|
|
534
|
+
|
|
535
|
+
# Enforce minimum pass rate threshold
|
|
536
|
+
if min_pass_rate is not None:
|
|
537
|
+
actual_rate = suite_report.pass_rate
|
|
538
|
+
terminalreporter: TerminalReporter | None = config.pluginmanager.get_plugin(
|
|
539
|
+
"terminalreporter"
|
|
540
|
+
)
|
|
541
|
+
if actual_rate < min_pass_rate:
|
|
542
|
+
if terminalreporter:
|
|
543
|
+
terminalreporter.write_line(
|
|
544
|
+
f"\naitest: FAILED - pass rate {actual_rate:.1f}% "
|
|
545
|
+
f"is below minimum threshold {min_pass_rate}% "
|
|
546
|
+
f"({suite_report.passed}/{suite_report.total} passed)",
|
|
547
|
+
red=True,
|
|
548
|
+
bold=True,
|
|
549
|
+
)
|
|
550
|
+
session.exitstatus = pytest.ExitCode.TESTS_FAILED
|
|
551
|
+
elif terminalreporter:
|
|
552
|
+
terminalreporter.write_line(
|
|
553
|
+
f"\naitest: pass rate {actual_rate:.1f}% meets minimum threshold {min_pass_rate}%",
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
# Reset global plugin/execution state without masking the main test outcome.
|
|
557
|
+
from pytest_copilot.execution.rate_limiter import reset_rate_limiters
|
|
558
|
+
|
|
559
|
+
try:
|
|
560
|
+
reset_rate_limiters()
|
|
561
|
+
except Exception:
|
|
562
|
+
_logger.warning("Rate limiter cleanup failed", exc_info=True)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Execution-evidence reporting options for the pytest plugin."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from _pytest.config.argparsing import OptionGroup
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def add_aitest_options(group: OptionGroup) -> None:
|
|
12
|
+
"""Register execution evidence, pass-rate thresholds, and repeated runs."""
|
|
13
|
+
group.addoption(
|
|
14
|
+
"--aitest-json",
|
|
15
|
+
metavar="PATH",
|
|
16
|
+
default=None,
|
|
17
|
+
help="Save structured execution evidence and pytest outcomes as JSON",
|
|
18
|
+
)
|
|
19
|
+
group.addoption(
|
|
20
|
+
"--aitest-min-pass-rate",
|
|
21
|
+
metavar="N",
|
|
22
|
+
type=int,
|
|
23
|
+
default=None,
|
|
24
|
+
help="Fail the session when the observed pytest pass rate is below N percent (0-100)",
|
|
25
|
+
)
|
|
26
|
+
group.addoption(
|
|
27
|
+
"--aitest-iterations",
|
|
28
|
+
metavar="N",
|
|
29
|
+
type=int,
|
|
30
|
+
default=1,
|
|
31
|
+
help="Run each test N times in independently created pytest fixtures",
|
|
32
|
+
)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Print the saved evidence location alongside ordinary pytest output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from _pytest.config import Config
|
|
11
|
+
from _pytest.terminal import TerminalReporter
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def log_report_path(config: Config, format_name: str, path: Path) -> None:
|
|
15
|
+
"""Print the native evidence path."""
|
|
16
|
+
terminal: TerminalReporter | None = config.pluginmanager.get_plugin("terminalreporter")
|
|
17
|
+
if terminal:
|
|
18
|
+
terminal.write_line(f"aitest {format_name}: {path}")
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Native execution evidence and ordinary pytest outcomes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pytest_copilot.reporting.collector import SuiteReport, TestReport, build_suite_report
|
|
6
|
+
from pytest_copilot.reporting.generator import generate_json, load_suite_report
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
# Core exports
|
|
10
|
+
"SuiteReport",
|
|
11
|
+
"TestReport",
|
|
12
|
+
"build_suite_report",
|
|
13
|
+
# Generation
|
|
14
|
+
"generate_json",
|
|
15
|
+
"load_suite_report",
|
|
16
|
+
]
|