pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,562 @@
1
+ """pytest plugin for aitest."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ from datetime import datetime
7
+ from pathlib import Path
8
+ from typing import TYPE_CHECKING, Any, cast
9
+
10
+ import pytest
11
+
12
+ from pytest_copilot.plugin_options import add_aitest_options
13
+ from pytest_copilot.plugin_report import (
14
+ log_report_path,
15
+ )
16
+ from pytest_copilot.reporting import (
17
+ TestReport,
18
+ build_suite_report,
19
+ generate_json,
20
+ )
21
+
22
+ if TYPE_CHECKING:
23
+ from _pytest.config import Config
24
+ from _pytest.config.argparsing import Parser
25
+ from _pytest.nodes import Item
26
+ from _pytest.reports import TestReport as PytestTestReport
27
+ from _pytest.terminal import TerminalReporter
28
+
29
+ from pytest_copilot.core.result import EvalResult
30
+
31
+
32
+ _logger = logging.getLogger(__name__)
33
+
34
+
35
+ # Key for storing test reports in config
36
+ COLLECTOR_KEY = pytest.StashKey[list[TestReport]]()
37
+ # Key for storing session messages for @pytest.mark.session
38
+ SESSION_MESSAGES_KEY = pytest.StashKey[dict[str, list[dict[str, Any]]]]()
39
+ PHASE_REPORTS_KEY = pytest.StashKey[dict[str, "PytestTestReport"]]()
40
+ # Export for use in fixtures and downstream consumers
41
+ __all__ = [
42
+ "COLLECTOR_KEY",
43
+ "SESSION_MESSAGES_KEY",
44
+ ]
45
+
46
+
47
+ def _get_timestamped_path(
48
+ base_name: str, test_name: str | None = None, default_dir: Path | None = None
49
+ ) -> Path:
50
+ """Generate timestamped filename for unique report names.
51
+
52
+ Args:
53
+ base_name: Base evidence filename with extension (e.g., 'results.json')
54
+ test_name: Name of the test/suite to include in filename
55
+ default_dir: Directory to store the file (default: 'aitest-reports')
56
+
57
+ Returns:
58
+ Path with format: {dir}/{prefix}_{test_name}_{ISO8601-timestamp}.{ext}
59
+ """
60
+ if default_dir is None:
61
+ default_dir = Path("aitest-reports")
62
+
63
+ # Use ISO8601 timestamp: YYYY-MM-DDTHH-MM-SS (seconds precision, : replaced with -)
64
+ timestamp = datetime.now().isoformat(timespec="seconds").replace(":", "-")
65
+
66
+ # Sanitize test name (remove paths, lowercase, replace spaces/special chars)
67
+ if test_name:
68
+ # Remove file extensions and paths
69
+ safe_name = (
70
+ test_name.split("/")[-1].split(".")[0].lower().replace(" ", "-").replace("_", "-")
71
+ )
72
+ else:
73
+ safe_name = None
74
+
75
+ # Split filename and extension
76
+ if "." in base_name:
77
+ name_part, ext = base_name.rsplit(".", 1)
78
+ if safe_name:
79
+ filename = f"{name_part}_{safe_name}_{timestamp}.{ext}"
80
+ else:
81
+ filename = f"{name_part}_{timestamp}.{ext}"
82
+ else:
83
+ if safe_name:
84
+ filename = f"{base_name}_{safe_name}_{timestamp}"
85
+ else:
86
+ filename = f"{base_name}_{timestamp}"
87
+
88
+ return default_dir / filename
89
+
90
+
91
+ def pytest_addoption(parser: Parser) -> None:
92
+ """Add pytest CLI options for aitest.
93
+
94
+ Note: Copilot-authenticated runs use the GitHub CLI or ``GITHUB_TOKEN``.
95
+ """
96
+ group = parser.getgroup("aitest", "AI agent testing")
97
+ add_aitest_options(group)
98
+
99
+
100
+ def pytest_configure(config: Config) -> None:
101
+ """Configure the aitest plugin."""
102
+ if not config.pluginmanager.hasplugin("pytest_copilot.fixtures"):
103
+ config.pluginmanager.import_plugin("pytest_copilot.fixtures")
104
+
105
+ # Register markers
106
+ config.addinivalue_line(
107
+ "markers",
108
+ "aitest: Mark test as an AI agent test (optional, enables filtering with -m aitest)",
109
+ )
110
+ config.addinivalue_line(
111
+ "markers",
112
+ "aitest_skip_report: Exclude this test from AI test reports",
113
+ )
114
+ config.addinivalue_line(
115
+ "markers",
116
+ "session(name): Mark tests as part of a named session for multi-turn conversations. "
117
+ "Tests with the same session name share conversation history automatically.",
118
+ )
119
+ config.addinivalue_line(
120
+ "markers",
121
+ "copilot: mark test as requiring GitHub Copilot SDK credentials",
122
+ )
123
+ config.addinivalue_line(
124
+ "markers",
125
+ "aitest_iteration_axis(position): internal repetition parameter-group index",
126
+ )
127
+
128
+ # Always initialize report collection - JSON is always generated
129
+ config.stash[COLLECTOR_KEY] = []
130
+ # Initialize session message storage
131
+ config.stash[SESSION_MESSAGES_KEY] = {}
132
+
133
+
134
+ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
135
+ """Parametrize tests with iteration index when ``--aitest-iterations`` > 1.
136
+
137
+ Follows the same pattern as *pytest-repeat*: the fixture name is
138
+ appended to ``metafunc.fixturenames`` so every test function
139
+ receives the parameter even though it does not declare the fixture
140
+ explicitly.
141
+ """
142
+ count_option = metafunc.config.getoption("--aitest-iterations", default=1)
143
+ count = count_option if isinstance(count_option, int) else 1
144
+ if count <= 1:
145
+ return
146
+ metafunc.fixturenames.append("_aitest_iteration")
147
+ existing = getattr(metafunc, "_calls", [])
148
+ position = len(existing[0]._idlist) if existing else 0
149
+ metafunc.parametrize(
150
+ "_aitest_iteration",
151
+ [
152
+ pytest.param(i, marks=pytest.mark.aitest_iteration_axis(position))
153
+ for i in range(1, count + 1)
154
+ ],
155
+ ids=[f"iter-{i}" for i in range(1, count + 1)],
156
+ indirect=True,
157
+ )
158
+
159
+
160
+ def pytest_collection_modifyitems(
161
+ session: pytest.Session,
162
+ config: Config,
163
+ items: list[pytest.Item],
164
+ ) -> None:
165
+ """Auto-mark tests that use aitest fixtures."""
166
+ for item in items:
167
+ # Check if test uses any aitest fixtures
168
+ fixturenames = getattr(item, "fixturenames", [])
169
+ aitest_fixtures = {"copilot_run"}
170
+ if (aitest_fixtures & set(fixturenames)) and not any(
171
+ m.name == "aitest" for m in item.iter_markers()
172
+ ):
173
+ item.add_marker(pytest.mark.aitest)
174
+
175
+
176
+ def _require_agent_string(agent: Any, *, field_name: str) -> str:
177
+ """Read a required string from the reporting wrapper."""
178
+ value = getattr(agent, field_name, None)
179
+ if isinstance(value, str) and value:
180
+ return value
181
+ msg = f"aitest reporting agent is missing required field '{field_name}'"
182
+ raise ValueError(msg)
183
+
184
+
185
+ def _require_provider_model(agent: Any) -> str | None:
186
+ """Read the required provider model from the reporting wrapper."""
187
+ provider = getattr(agent, "provider", None)
188
+ if provider is None or not hasattr(provider, "model"):
189
+ raise ValueError("aitest reporting agent is missing required field 'provider.model'")
190
+ model = provider.model
191
+ if model is None or isinstance(model, str) and model:
192
+ return model
193
+ raise ValueError("aitest reporting agent is missing required field 'provider.model'")
194
+
195
+
196
+ def _display_model(model: str | None) -> str | None:
197
+ """Strip any provider prefix from a model name for display."""
198
+ return model.split("/")[-1] if model is not None and "/" in model else model
199
+
200
+
201
+ def _build_agent_identity(
202
+ agent: Any,
203
+ eval_result: EvalResult,
204
+ ) -> tuple[str, str, str | None, str | None, str | None]:
205
+ """Resolve stable and human-facing identity fields for report production."""
206
+ base_agent_id = _require_agent_string(agent, field_name="id")
207
+ eval_name = _require_agent_string(agent, field_name="name")
208
+ model_raw = _require_provider_model(agent)
209
+ system_prompt_name = getattr(agent, "system_prompt_name", None)
210
+ if system_prompt_name is not None and not isinstance(system_prompt_name, str):
211
+ raise ValueError("aitest reporting agent field 'system_prompt_name' must be a string")
212
+
213
+ skill_name = eval_result.skill_info.name if eval_result.skill_info else None
214
+ if skill_name is None:
215
+ skill = getattr(agent, "skill", None)
216
+ if skill is not None:
217
+ skill_name = _require_agent_string(skill, field_name="name")
218
+
219
+ return (
220
+ base_agent_id,
221
+ eval_name,
222
+ _display_model(model_raw),
223
+ system_prompt_name,
224
+ skill_name,
225
+ )
226
+
227
+
228
+ def _build_case_name(item: Item) -> str:
229
+ """Build a case identity that removes only the synthetic iteration axis."""
230
+ callspec = getattr(item, "callspec", None)
231
+ if callspec is None or "_aitest_iteration" not in callspec.params:
232
+ return item.nodeid
233
+
234
+ param_ids = list(getattr(callspec, "_idlist", []))
235
+ axis = item.get_closest_marker("aitest_iteration_axis")
236
+ if axis is None or len(axis.args) != 1 or type(axis.args[0]) is not int:
237
+ raise ValueError(
238
+ "Unable to derive case identity for "
239
+ f"{item.nodeid!r}: repetition parameter-group identity is unavailable"
240
+ )
241
+ position = axis.args[0]
242
+ if not 0 <= position < len(param_ids):
243
+ raise ValueError(f"Invalid repetition parameter-group index for {item.nodeid!r}")
244
+ del param_ids[position]
245
+ base_nodeid = item.nodeid.removesuffix(f"[{callspec.id}]")
246
+ if not param_ids:
247
+ return base_nodeid
248
+ return f"{base_nodeid}[{'-'.join(param_ids)}]"
249
+
250
+
251
+ @pytest.hookimpl(hookwrapper=True, tryfirst=True)
252
+ def pytest_runtest_makereport(item: Item, call: Any) -> Any:
253
+ """Capture test results for reporting.
254
+
255
+ Also auto-stashes CopilotResult for tests that call ``run_copilot()``
256
+ directly instead of using the ``copilot_run`` fixture — critical for
257
+ module-scoped agent fixtures that cannot use the function-scoped fixture.
258
+ """
259
+ # Auto-stash CopilotResult before processing (tryfirst ensures this runs early)
260
+ if call.when in ("setup", "call") and not hasattr(item, "_aitest_result"):
261
+ from pytest_copilot.copilot.fixtures import stash_on_item
262
+ from pytest_copilot.copilot.result import CopilotResult
263
+ from pytest_copilot.copilot.task import CopilotTask
264
+
265
+ funcargs = getattr(item, "funcargs", {})
266
+ for val in funcargs.values():
267
+ if isinstance(val, CopilotResult) and val.agent is not None:
268
+ stash_on_item(item, cast(CopilotTask, val.agent), val)
269
+ break
270
+
271
+ outcome = yield
272
+ report: PytestTestReport = outcome.get_result()
273
+
274
+ phases = item.stash.setdefault(PHASE_REPORTS_KEY, {})
275
+ phases[report.when] = report
276
+
277
+ # Check if reporting is enabled
278
+ tests = item.config.stash.get(COLLECTOR_KEY, None)
279
+ if tests is None:
280
+ return
281
+
282
+ # Skip if marked to exclude from report
283
+ if any(m.name == "aitest_skip_report" for m in item.iter_markers()):
284
+ return
285
+
286
+ runs = getattr(item, "_aitest_runs", [])
287
+
288
+ # Only collect tests that actually used aitest (have an agent result)
289
+ # Tests without eval execution do not create eval reports.
290
+ if not runs:
291
+ return
292
+ if report.when != "teardown":
293
+ if report.when == "call":
294
+ for eval_result, agent in runs:
295
+ _add_junit_properties(report, eval_result, agent)
296
+ return
297
+ failed_phases = [phase for phase in phases.values() if phase.failed]
298
+ skipped_phases = [phase for phase in phases.values() if phase.skipped]
299
+ final_outcome = "failed" if failed_phases else "skipped" if skipped_phases else "passed"
300
+
301
+ # Get test function docstring if available
302
+ docstring = None
303
+ func = getattr(item, "function", None)
304
+ if func is not None and func.__doc__:
305
+ docstring = func.__doc__
306
+
307
+ # Get test class docstring if available
308
+ class_docstring = None
309
+ parent = getattr(item, "parent", None)
310
+ if parent is not None:
311
+ parent_obj = getattr(parent, "obj", None)
312
+ if parent_obj is not None and hasattr(parent_obj, "__doc__") and parent_obj.__doc__:
313
+ # Only use class docstrings (not module docstrings)
314
+ import inspect
315
+
316
+ if inspect.isclass(parent_obj):
317
+ class_docstring = parent_obj.__doc__
318
+
319
+ # Capture error message — just the assertion/exception, never raw tracebacks.
320
+ # Tracebacks contain file paths, line numbers, and nodeids that pollute
321
+ # User-facing reports.
322
+ phase_errors = []
323
+ for failed_report in failed_phases:
324
+ error_msg = None
325
+ error_text = str(failed_report.longrepr)
326
+ error_lines = error_text.split("\n")
327
+
328
+ # Extract lines starting with "E " — pytest's assertion/exception lines
329
+ e_lines = [line.strip()[2:] for line in error_lines if line.strip().startswith("E ")]
330
+
331
+ if e_lines:
332
+ error_msg = "\n".join(e_lines)
333
+ else:
334
+ # No E-lines: grab the last non-empty line (typically "ExceptionType: message")
335
+ for line in reversed(error_lines):
336
+ stripped = line.strip()
337
+ if stripped:
338
+ error_msg = stripped
339
+ break
340
+ if error_msg is not None:
341
+ phase_errors.append(
342
+ error_msg if failed_report.when == "call" else f"{failed_report.when}: {error_msg}"
343
+ )
344
+ error_msg = "\n".join(phase_errors) or None
345
+
346
+ # Detect iteration index from _aitest_iteration fixture
347
+ iteration: int | None = None
348
+ callspec = getattr(item, "callspec", None)
349
+ if callspec and "_aitest_iteration" in callspec.params:
350
+ iteration = callspec.params["_aitest_iteration"]
351
+
352
+ properties = list(report.user_properties)
353
+ for eval_result, agent in runs:
354
+ agent_id, eval_name, model, system_prompt_name, skill_name = _build_agent_identity(
355
+ agent, eval_result
356
+ )
357
+ test_report = TestReport(
358
+ name=_build_case_name(item),
359
+ outcome=final_outcome,
360
+ duration_ms=(
361
+ eval_result.duration_ms
362
+ if len(runs) > 1
363
+ else sum(phase.duration for phase in phases.values()) * 1000
364
+ ),
365
+ eval_result=eval_result,
366
+ error=error_msg,
367
+ properties=properties,
368
+ docstring=docstring,
369
+ class_docstring=class_docstring,
370
+ agent_id=agent_id,
371
+ eval_name=eval_name,
372
+ model=model,
373
+ system_prompt_name=system_prompt_name,
374
+ skill_name=skill_name,
375
+ iteration=iteration,
376
+ )
377
+ tests.append(test_report)
378
+ _add_junit_properties(report, eval_result, agent)
379
+
380
+
381
+ def _add_junit_properties(
382
+ report: PytestTestReport,
383
+ eval_result: EvalResult,
384
+ agent: Any | None = None,
385
+ ) -> None:
386
+ """Add agent metadata to pytest report for JUnit XML output.
387
+
388
+ Properties are added to report.user_properties which pytest writes
389
+ as <property> elements in JUnit XML output.
390
+
391
+ Example output:
392
+ <testcase name="test_balance">
393
+ <properties>
394
+ <property name="aitest.agent.name" value="banking-agent"/>
395
+ <property name="aitest.model" value="gpt-5.6-sol"/>
396
+ <property name="aitest.skill" value="financial-advisor"/>
397
+ <property name="aitest.tools.called" value="get_balance,transfer"/>
398
+ </properties>
399
+ </testcase>
400
+ """
401
+ if not hasattr(report, "user_properties"):
402
+ return
403
+
404
+ props = []
405
+
406
+ # Eval identity (from Eval object)
407
+ if agent:
408
+ agent_id, eval_name, model, system_prompt_name, _ = _build_agent_identity(
409
+ agent, eval_result
410
+ )
411
+ props.append(("aitest.agent.name", eval_name))
412
+ if model is not None:
413
+ props.append(("aitest.model", model))
414
+ if system_prompt_name:
415
+ props.append(("aitest.prompt", system_prompt_name))
416
+
417
+ # Skill
418
+ if eval_result.skill_info:
419
+ props.append(("aitest.skill", eval_result.skill_info.name))
420
+
421
+ # MCP servers (from agent config)
422
+ if agent and agent.mcp_servers:
423
+ server_names = []
424
+ for server in agent.mcp_servers:
425
+ # Use server.name if available, otherwise derive from command
426
+ name = getattr(server, "name", None)
427
+ if not name and hasattr(server, "command") and server.command:
428
+ name = server.command[-1].split("/")[-1].split(".")[0]
429
+ if name:
430
+ server_names.append(name)
431
+ if server_names:
432
+ props.append(("aitest.servers", ",".join(server_names)))
433
+
434
+ # Allowed tools filter (from agent config)
435
+ if agent and agent.allowed_tools:
436
+ props.append(("aitest.allowed_tools", ",".join(sorted(agent.allowed_tools))))
437
+
438
+ # Token usage
439
+ if eval_result.token_usage:
440
+ prompt = eval_result.token_usage.get("prompt", 0)
441
+ completion = eval_result.token_usage.get("completion", 0)
442
+ if prompt:
443
+ props.append(("aitest.tokens.input", str(prompt)))
444
+ if completion:
445
+ props.append(("aitest.tokens.output", str(completion)))
446
+ total = prompt + completion
447
+ if total:
448
+ props.append(("aitest.tokens.total", str(total)))
449
+
450
+ # Cost
451
+ if eval_result.cost_usd > 0:
452
+ props.append(("aitest.cost_usd", f"{eval_result.cost_usd:.6f}"))
453
+
454
+ # Turns
455
+ if eval_result.turns:
456
+ props.append(("aitest.turns", str(len(eval_result.turns))))
457
+
458
+ # Tools called (unique, comma-separated)
459
+ tools_called = set()
460
+ for turn in eval_result.turns:
461
+ for tc in turn.tool_calls:
462
+ tools_called.add(tc.name)
463
+ if tools_called:
464
+ props.append(("aitest.tools.called", ",".join(sorted(tools_called))))
465
+
466
+ # Success/failure
467
+ props.append(("aitest.success", str(eval_result.success).lower()))
468
+
469
+ # Add all properties to report
470
+ report.user_properties.extend(props)
471
+
472
+
473
+ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
474
+ """Save execution evidence and enforce the observed pytest pass-rate threshold."""
475
+ config = session.config
476
+ tests = config.stash.get(COLLECTOR_KEY, None)
477
+
478
+ if tests is None or not tests:
479
+ return
480
+
481
+ json_path = config.getoption("--aitest-json")
482
+ min_pass_rate: int | None = config.getoption("--aitest-min-pass-rate")
483
+
484
+ # Extract suite docstring from first test's parent class/module
485
+ suite_docstring = None
486
+ if session.items:
487
+ first_item = session.items[0]
488
+ # Try to get docstring from test class first
489
+ if hasattr(first_item, "parent") and first_item.parent:
490
+ parent = first_item.parent
491
+ # Check if parent is a class
492
+ parent_obj = getattr(parent, "obj", None)
493
+ if parent_obj and hasattr(parent_obj, "__doc__"):
494
+ suite_docstring = parent_obj.__doc__
495
+ if suite_docstring:
496
+ # Get first line only
497
+ suite_docstring = suite_docstring.strip().split("\n")[0].strip()
498
+
499
+ # Build suite report first (to get the test name for default filenames)
500
+ default_dir = Path("aitest-reports")
501
+ suite_report = build_suite_report(
502
+ tests,
503
+ name=session.name or "pytest-copilot",
504
+ suite_docstring=suite_docstring,
505
+ )
506
+
507
+ # Generate default paths with test name included
508
+ default_json_path = _get_timestamped_path(
509
+ "results.json", test_name=suite_report.name, default_dir=default_dir
510
+ )
511
+
512
+ json_output_path = Path(json_path) if json_path else default_json_path
513
+ try:
514
+ json_output_path.parent.mkdir(parents=True, exist_ok=True)
515
+ generate_json(suite_report, json_output_path)
516
+ except (OSError, TypeError, ValueError) as error:
517
+ if session.exitstatus == pytest.ExitCode.OK:
518
+ session.exitstatus = pytest.ExitCode.TESTS_FAILED
519
+ terminalreporter = config.pluginmanager.get_plugin("terminalreporter")
520
+ if terminalreporter:
521
+ terminalreporter.write_line(
522
+ f"\naitest: Failed to save execution evidence to {json_output_path}: {error}",
523
+ red=True,
524
+ bold=True,
525
+ )
526
+ _logger.error(
527
+ "Failed to save execution evidence to %s; "
528
+ "continuing with pass-rate enforcement and cleanup",
529
+ json_output_path,
530
+ exc_info=True,
531
+ )
532
+ else:
533
+ log_report_path(config, "JSON evidence", json_output_path)
534
+
535
+ # Enforce minimum pass rate threshold
536
+ if min_pass_rate is not None:
537
+ actual_rate = suite_report.pass_rate
538
+ terminalreporter: TerminalReporter | None = config.pluginmanager.get_plugin(
539
+ "terminalreporter"
540
+ )
541
+ if actual_rate < min_pass_rate:
542
+ if terminalreporter:
543
+ terminalreporter.write_line(
544
+ f"\naitest: FAILED - pass rate {actual_rate:.1f}% "
545
+ f"is below minimum threshold {min_pass_rate}% "
546
+ f"({suite_report.passed}/{suite_report.total} passed)",
547
+ red=True,
548
+ bold=True,
549
+ )
550
+ session.exitstatus = pytest.ExitCode.TESTS_FAILED
551
+ elif terminalreporter:
552
+ terminalreporter.write_line(
553
+ f"\naitest: pass rate {actual_rate:.1f}% meets minimum threshold {min_pass_rate}%",
554
+ )
555
+
556
+ # Reset global plugin/execution state without masking the main test outcome.
557
+ from pytest_copilot.execution.rate_limiter import reset_rate_limiters
558
+
559
+ try:
560
+ reset_rate_limiters()
561
+ except Exception:
562
+ _logger.warning("Rate limiter cleanup failed", exc_info=True)
@@ -0,0 +1,32 @@
1
+ """Execution-evidence reporting options for the pytest plugin."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ if TYPE_CHECKING:
8
+ from _pytest.config.argparsing import OptionGroup
9
+
10
+
11
+ def add_aitest_options(group: OptionGroup) -> None:
12
+ """Register execution evidence, pass-rate thresholds, and repeated runs."""
13
+ group.addoption(
14
+ "--aitest-json",
15
+ metavar="PATH",
16
+ default=None,
17
+ help="Save structured execution evidence and pytest outcomes as JSON",
18
+ )
19
+ group.addoption(
20
+ "--aitest-min-pass-rate",
21
+ metavar="N",
22
+ type=int,
23
+ default=None,
24
+ help="Fail the session when the observed pytest pass rate is below N percent (0-100)",
25
+ )
26
+ group.addoption(
27
+ "--aitest-iterations",
28
+ metavar="N",
29
+ type=int,
30
+ default=1,
31
+ help="Run each test N times in independently created pytest fixtures",
32
+ )
@@ -0,0 +1,18 @@
1
+ """Print the saved evidence location alongside ordinary pytest output."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ if TYPE_CHECKING:
8
+ from pathlib import Path
9
+
10
+ from _pytest.config import Config
11
+ from _pytest.terminal import TerminalReporter
12
+
13
+
14
+ def log_report_path(config: Config, format_name: str, path: Path) -> None:
15
+ """Print the native evidence path."""
16
+ terminal: TerminalReporter | None = config.pluginmanager.get_plugin("terminalreporter")
17
+ if terminal:
18
+ terminal.write_line(f"aitest {format_name}: {path}")
@@ -0,0 +1,16 @@
1
+ """Native execution evidence and ordinary pytest outcomes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pytest_copilot.reporting.collector import SuiteReport, TestReport, build_suite_report
6
+ from pytest_copilot.reporting.generator import generate_json, load_suite_report
7
+
8
+ __all__ = [
9
+ # Core exports
10
+ "SuiteReport",
11
+ "TestReport",
12
+ "build_suite_report",
13
+ # Generation
14
+ "generate_json",
15
+ "load_suite_report",
16
+ ]