pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,203 @@
1
+ """Captured execution records and ordinary pytest outcomes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field
6
+ from datetime import datetime
7
+ from typing import TYPE_CHECKING, Any
8
+
9
+ if TYPE_CHECKING:
10
+ from pytest_copilot.core.result import EvalResult
11
+
12
+
13
+ @dataclass(slots=True)
14
+ class TestReport:
15
+ """Native evidence for one captured test execution.
16
+
17
+ Attributes:
18
+ name: Full test node ID (e.g., "test_balance[gpt-4o-PROMPT_V1]")
19
+ outcome: Test outcome - "passed", "failed", or "skipped"
20
+ duration_ms: Test duration in milliseconds
21
+ eval_result: Optional EvalResult from copilot_run
22
+ error: Error message if test failed
23
+ assertions: List of assertion results
24
+ docstring: Test function's docstring (first line) for human-readable description
25
+ class_docstring: Test class docstring (first line) for human-readable group name
26
+ agent_id: Identity supplied by the execution wrapper
27
+ eval_name: Name identifying the eval configuration
28
+ model: LLM model name (without provider prefix)
29
+ system_prompt_name: Label for the system prompt variant
30
+ skill_name: Name of the skill used
31
+ """
32
+
33
+ name: str
34
+ outcome: str # "passed", "failed", "skipped"
35
+ duration_ms: float
36
+ eval_result: EvalResult | None = None
37
+ error: str | None = None
38
+ assertions: list[dict[str, Any]] = field(default_factory=list)
39
+ docstring: str | None = None
40
+ class_docstring: str | None = None
41
+
42
+ # Eval identity (populated by plugin from Eval object)
43
+ agent_id: str = ""
44
+ eval_name: str = ""
45
+ model: str | None = None
46
+ system_prompt_name: str | None = None
47
+ skill_name: str | None = None
48
+ iteration: int | None = None
49
+ properties: list[tuple[str, Any]] = field(default_factory=list)
50
+
51
+ @property
52
+ def is_passed(self) -> bool:
53
+ return self.outcome == "passed"
54
+
55
+ @property
56
+ def is_failed(self) -> bool:
57
+ return self.outcome == "failed"
58
+
59
+ @property
60
+ def short_name(self) -> str:
61
+ """Extract just the test function name from the full node ID.
62
+
63
+ 'tests/test_foo.py::TestClass::test_bar[param]' -> 'test_bar[param]'
64
+ """
65
+ return self.name.split("::")[-1]
66
+
67
+ @property
68
+ def display_name(self) -> str:
69
+ """Human-readable test name: docstring if available, else short test name."""
70
+ if self.docstring:
71
+ # Return first line of docstring, stripped
72
+ return self.docstring.split("\n")[0].strip()
73
+ return self.short_name
74
+
75
+ @property
76
+ def tokens_used(self) -> int:
77
+ """Get total tokens used from eval_result if present."""
78
+ if self.eval_result is not None:
79
+ return self.eval_result.token_usage.get("prompt", 0) + self.eval_result.token_usage.get(
80
+ "completion", 0
81
+ )
82
+ return 0
83
+
84
+ @property
85
+ def tool_calls(self) -> list[str]:
86
+ """Get tool call names from eval_result if present."""
87
+ if self.eval_result is not None:
88
+ return [tc.name for tc in self.eval_result.all_tool_calls]
89
+ return []
90
+
91
+
92
+ @dataclass(slots=True)
93
+ class SuiteReport:
94
+ """Native evidence for captured executions in a pytest session.
95
+
96
+ Counts reflect recorded pytest outcomes, not framework-created judgments.
97
+ """
98
+
99
+ name: str
100
+ timestamp: str
101
+ duration_ms: float
102
+ tests: list[TestReport] = field(default_factory=list)
103
+ passed: int = 0
104
+ failed: int = 0
105
+ skipped: int = 0
106
+ suite_docstring: str | None = None
107
+ models_without_pricing: list[str] = field(default_factory=list)
108
+
109
+ @property
110
+ def total(self) -> int:
111
+ return self.passed + self.failed + self.skipped
112
+
113
+ @property
114
+ def pass_rate(self) -> float:
115
+ if self.total == 0:
116
+ return 0.0
117
+ return self.passed / self.total * 100
118
+
119
+ @property
120
+ def total_tokens(self) -> int:
121
+ """Sum of all tokens used."""
122
+ total = 0
123
+ for t in self.tests:
124
+ if t.eval_result is not None:
125
+ total += t.eval_result.token_usage.get("prompt", 0)
126
+ total += t.eval_result.token_usage.get("completion", 0)
127
+ return total
128
+
129
+ @property
130
+ def total_cost_usd(self) -> float:
131
+ """Sum of all costs in USD."""
132
+ return sum(t.eval_result.cost_usd for t in self.tests if t.eval_result is not None)
133
+
134
+ @property
135
+ def token_stats(self) -> dict[str, int]:
136
+ """Get min/max/avg token usage."""
137
+ tokens = [
138
+ t.eval_result.token_usage.get("prompt", 0)
139
+ + t.eval_result.token_usage.get("completion", 0)
140
+ for t in self.tests
141
+ if t.eval_result is not None
142
+ ]
143
+ if not tokens:
144
+ return {"min": 0, "max": 0, "avg": 0}
145
+ return {
146
+ "min": min(tokens),
147
+ "max": max(tokens),
148
+ "avg": sum(tokens) // len(tokens),
149
+ }
150
+
151
+ @property
152
+ def test_files(self) -> list[str]:
153
+ """Unique test file paths."""
154
+ files = set()
155
+ for t in self.tests:
156
+ # Extract file path from node ID (e.g., "tests/test_basic.py::TestClass::test")
157
+ if "::" in t.name:
158
+ files.add(t.name.split("::")[0])
159
+ else:
160
+ files.add(t.name)
161
+ return sorted(files)
162
+
163
+
164
+ def build_suite_report(
165
+ tests: list[TestReport],
166
+ name: str,
167
+ suite_docstring: str | None = None,
168
+ ) -> SuiteReport:
169
+ """Build final suite report from collected tests.
170
+
171
+ Args:
172
+ tests: List of test reports to include
173
+ name: Name for the test suite
174
+ suite_docstring: Optional docstring for the suite
175
+
176
+ Example:
177
+ report = build_suite_report(tests, "my_tests")
178
+ """
179
+ passed = sum(1 for t in tests if t.outcome == "passed")
180
+ failed = sum(1 for t in tests if t.outcome == "failed")
181
+ skipped = sum(1 for t in tests if t.outcome == "skipped")
182
+ total_duration = sum(t.duration_ms for t in tests)
183
+
184
+ from pytest_copilot.execution.cost import unpriced_models
185
+
186
+ suite_models = {
187
+ usage.model
188
+ for test in tests
189
+ if test.eval_result is not None
190
+ for result in test.eval_result.execution_results
191
+ for usage in result.usage
192
+ }
193
+ return SuiteReport(
194
+ name=name,
195
+ timestamp=datetime.now().isoformat(),
196
+ duration_ms=total_duration,
197
+ tests=tests,
198
+ passed=passed,
199
+ failed=failed,
200
+ skipped=skipped,
201
+ suite_docstring=suite_docstring,
202
+ models_without_pricing=unpriced_models(suite_models),
203
+ )
@@ -0,0 +1,55 @@
1
+ """Save and load native execution evidence without interpreting it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ from pathlib import Path
8
+ from tempfile import NamedTemporaryFile
9
+ from typing import TYPE_CHECKING
10
+
11
+ from pytest_copilot.core.serialization import (
12
+ deserialize_suite_report,
13
+ serialize_dataclass,
14
+ )
15
+ from pytest_copilot.reporting.schema import REPORT_SCHEMA_VERSION
16
+
17
+ if TYPE_CHECKING:
18
+ from pytest_copilot.reporting.collector import SuiteReport
19
+
20
+
21
+ def generate_json(report: SuiteReport, output_path: str | Path) -> None:
22
+ """Atomically save recorded evidence and pytest outcomes as JSON.
23
+
24
+ Property values must be JSON-compatible. Unsupported values are errors,
25
+ not opportunities to replace evidence with string representations.
26
+ """
27
+ data = serialize_dataclass(report)
28
+ data["schema_version"] = REPORT_SCHEMA_VERSION
29
+ content = json.dumps(data, indent=2)
30
+ destination = Path(output_path)
31
+ with NamedTemporaryFile(
32
+ dir=destination.parent, prefix=f".{destination.name}.", delete=False
33
+ ) as stream:
34
+ temporary = Path(stream.name)
35
+ try:
36
+ temporary.write_text(content, encoding="utf-8")
37
+ os.replace(temporary, destination)
38
+ finally:
39
+ temporary.unlink(missing_ok=True)
40
+
41
+
42
+ def load_suite_report(path: str | Path) -> SuiteReport:
43
+ """Load current-schema native evidence; never infer results or grade answers."""
44
+ data = json.loads(Path(path).read_text(encoding="utf-8"))
45
+ if not isinstance(data, dict):
46
+ raise ValueError("Execution evidence must be a JSON object")
47
+ if data.get("schema_version") != REPORT_SCHEMA_VERSION:
48
+ raise ValueError(
49
+ f"Unsupported schema version: {data.get('schema_version')!r}. "
50
+ f"Only {REPORT_SCHEMA_VERSION} is supported. Re-run the producer "
51
+ "with the current package to save new evidence."
52
+ )
53
+ if "insights" in data:
54
+ raise ValueError("AI insights are not part of execution evidence")
55
+ return deserialize_suite_report(data)
@@ -0,0 +1,5 @@
1
+ """Shared reporting schema constants."""
2
+
3
+ from __future__ import annotations
4
+
5
+ REPORT_SCHEMA_VERSION = "4.0"
@@ -0,0 +1,17 @@
1
+ """Test harnesses for pytest-copilot integration testing.
2
+
3
+ Provides stateful backends and MCP servers for testing agent behavior.
4
+
5
+ Available Test Servers
6
+ ----------------------
7
+ BankingService
8
+ Banking operations with business rules and error handling. Stateful.
9
+
10
+ TodoStore
11
+ Task management for CRUD operation tests. Stateful.
12
+ """
13
+
14
+ from pytest_copilot.testing.banking import BankingService
15
+ from pytest_copilot.testing.todo import TodoStore
16
+
17
+ __all__ = ["BankingService", "TodoStore"]