pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Captured execution records and ordinary pytest outcomes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import TYPE_CHECKING, Any
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from pytest_copilot.core.result import EvalResult
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(slots=True)
|
|
14
|
+
class TestReport:
|
|
15
|
+
"""Native evidence for one captured test execution.
|
|
16
|
+
|
|
17
|
+
Attributes:
|
|
18
|
+
name: Full test node ID (e.g., "test_balance[gpt-4o-PROMPT_V1]")
|
|
19
|
+
outcome: Test outcome - "passed", "failed", or "skipped"
|
|
20
|
+
duration_ms: Test duration in milliseconds
|
|
21
|
+
eval_result: Optional EvalResult from copilot_run
|
|
22
|
+
error: Error message if test failed
|
|
23
|
+
assertions: List of assertion results
|
|
24
|
+
docstring: Test function's docstring (first line) for human-readable description
|
|
25
|
+
class_docstring: Test class docstring (first line) for human-readable group name
|
|
26
|
+
agent_id: Identity supplied by the execution wrapper
|
|
27
|
+
eval_name: Name identifying the eval configuration
|
|
28
|
+
model: LLM model name (without provider prefix)
|
|
29
|
+
system_prompt_name: Label for the system prompt variant
|
|
30
|
+
skill_name: Name of the skill used
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
name: str
|
|
34
|
+
outcome: str # "passed", "failed", "skipped"
|
|
35
|
+
duration_ms: float
|
|
36
|
+
eval_result: EvalResult | None = None
|
|
37
|
+
error: str | None = None
|
|
38
|
+
assertions: list[dict[str, Any]] = field(default_factory=list)
|
|
39
|
+
docstring: str | None = None
|
|
40
|
+
class_docstring: str | None = None
|
|
41
|
+
|
|
42
|
+
# Eval identity (populated by plugin from Eval object)
|
|
43
|
+
agent_id: str = ""
|
|
44
|
+
eval_name: str = ""
|
|
45
|
+
model: str | None = None
|
|
46
|
+
system_prompt_name: str | None = None
|
|
47
|
+
skill_name: str | None = None
|
|
48
|
+
iteration: int | None = None
|
|
49
|
+
properties: list[tuple[str, Any]] = field(default_factory=list)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def is_passed(self) -> bool:
|
|
53
|
+
return self.outcome == "passed"
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def is_failed(self) -> bool:
|
|
57
|
+
return self.outcome == "failed"
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def short_name(self) -> str:
|
|
61
|
+
"""Extract just the test function name from the full node ID.
|
|
62
|
+
|
|
63
|
+
'tests/test_foo.py::TestClass::test_bar[param]' -> 'test_bar[param]'
|
|
64
|
+
"""
|
|
65
|
+
return self.name.split("::")[-1]
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def display_name(self) -> str:
|
|
69
|
+
"""Human-readable test name: docstring if available, else short test name."""
|
|
70
|
+
if self.docstring:
|
|
71
|
+
# Return first line of docstring, stripped
|
|
72
|
+
return self.docstring.split("\n")[0].strip()
|
|
73
|
+
return self.short_name
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def tokens_used(self) -> int:
|
|
77
|
+
"""Get total tokens used from eval_result if present."""
|
|
78
|
+
if self.eval_result is not None:
|
|
79
|
+
return self.eval_result.token_usage.get("prompt", 0) + self.eval_result.token_usage.get(
|
|
80
|
+
"completion", 0
|
|
81
|
+
)
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def tool_calls(self) -> list[str]:
|
|
86
|
+
"""Get tool call names from eval_result if present."""
|
|
87
|
+
if self.eval_result is not None:
|
|
88
|
+
return [tc.name for tc in self.eval_result.all_tool_calls]
|
|
89
|
+
return []
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(slots=True)
|
|
93
|
+
class SuiteReport:
|
|
94
|
+
"""Native evidence for captured executions in a pytest session.
|
|
95
|
+
|
|
96
|
+
Counts reflect recorded pytest outcomes, not framework-created judgments.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
name: str
|
|
100
|
+
timestamp: str
|
|
101
|
+
duration_ms: float
|
|
102
|
+
tests: list[TestReport] = field(default_factory=list)
|
|
103
|
+
passed: int = 0
|
|
104
|
+
failed: int = 0
|
|
105
|
+
skipped: int = 0
|
|
106
|
+
suite_docstring: str | None = None
|
|
107
|
+
models_without_pricing: list[str] = field(default_factory=list)
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def total(self) -> int:
|
|
111
|
+
return self.passed + self.failed + self.skipped
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def pass_rate(self) -> float:
|
|
115
|
+
if self.total == 0:
|
|
116
|
+
return 0.0
|
|
117
|
+
return self.passed / self.total * 100
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def total_tokens(self) -> int:
|
|
121
|
+
"""Sum of all tokens used."""
|
|
122
|
+
total = 0
|
|
123
|
+
for t in self.tests:
|
|
124
|
+
if t.eval_result is not None:
|
|
125
|
+
total += t.eval_result.token_usage.get("prompt", 0)
|
|
126
|
+
total += t.eval_result.token_usage.get("completion", 0)
|
|
127
|
+
return total
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def total_cost_usd(self) -> float:
|
|
131
|
+
"""Sum of all costs in USD."""
|
|
132
|
+
return sum(t.eval_result.cost_usd for t in self.tests if t.eval_result is not None)
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def token_stats(self) -> dict[str, int]:
|
|
136
|
+
"""Get min/max/avg token usage."""
|
|
137
|
+
tokens = [
|
|
138
|
+
t.eval_result.token_usage.get("prompt", 0)
|
|
139
|
+
+ t.eval_result.token_usage.get("completion", 0)
|
|
140
|
+
for t in self.tests
|
|
141
|
+
if t.eval_result is not None
|
|
142
|
+
]
|
|
143
|
+
if not tokens:
|
|
144
|
+
return {"min": 0, "max": 0, "avg": 0}
|
|
145
|
+
return {
|
|
146
|
+
"min": min(tokens),
|
|
147
|
+
"max": max(tokens),
|
|
148
|
+
"avg": sum(tokens) // len(tokens),
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def test_files(self) -> list[str]:
|
|
153
|
+
"""Unique test file paths."""
|
|
154
|
+
files = set()
|
|
155
|
+
for t in self.tests:
|
|
156
|
+
# Extract file path from node ID (e.g., "tests/test_basic.py::TestClass::test")
|
|
157
|
+
if "::" in t.name:
|
|
158
|
+
files.add(t.name.split("::")[0])
|
|
159
|
+
else:
|
|
160
|
+
files.add(t.name)
|
|
161
|
+
return sorted(files)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def build_suite_report(
|
|
165
|
+
tests: list[TestReport],
|
|
166
|
+
name: str,
|
|
167
|
+
suite_docstring: str | None = None,
|
|
168
|
+
) -> SuiteReport:
|
|
169
|
+
"""Build final suite report from collected tests.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
tests: List of test reports to include
|
|
173
|
+
name: Name for the test suite
|
|
174
|
+
suite_docstring: Optional docstring for the suite
|
|
175
|
+
|
|
176
|
+
Example:
|
|
177
|
+
report = build_suite_report(tests, "my_tests")
|
|
178
|
+
"""
|
|
179
|
+
passed = sum(1 for t in tests if t.outcome == "passed")
|
|
180
|
+
failed = sum(1 for t in tests if t.outcome == "failed")
|
|
181
|
+
skipped = sum(1 for t in tests if t.outcome == "skipped")
|
|
182
|
+
total_duration = sum(t.duration_ms for t in tests)
|
|
183
|
+
|
|
184
|
+
from pytest_copilot.execution.cost import unpriced_models
|
|
185
|
+
|
|
186
|
+
suite_models = {
|
|
187
|
+
usage.model
|
|
188
|
+
for test in tests
|
|
189
|
+
if test.eval_result is not None
|
|
190
|
+
for result in test.eval_result.execution_results
|
|
191
|
+
for usage in result.usage
|
|
192
|
+
}
|
|
193
|
+
return SuiteReport(
|
|
194
|
+
name=name,
|
|
195
|
+
timestamp=datetime.now().isoformat(),
|
|
196
|
+
duration_ms=total_duration,
|
|
197
|
+
tests=tests,
|
|
198
|
+
passed=passed,
|
|
199
|
+
failed=failed,
|
|
200
|
+
skipped=skipped,
|
|
201
|
+
suite_docstring=suite_docstring,
|
|
202
|
+
models_without_pricing=unpriced_models(suite_models),
|
|
203
|
+
)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Save and load native execution evidence without interpreting it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from tempfile import NamedTemporaryFile
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
from pytest_copilot.core.serialization import (
|
|
12
|
+
deserialize_suite_report,
|
|
13
|
+
serialize_dataclass,
|
|
14
|
+
)
|
|
15
|
+
from pytest_copilot.reporting.schema import REPORT_SCHEMA_VERSION
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from pytest_copilot.reporting.collector import SuiteReport
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def generate_json(report: SuiteReport, output_path: str | Path) -> None:
|
|
22
|
+
"""Atomically save recorded evidence and pytest outcomes as JSON.
|
|
23
|
+
|
|
24
|
+
Property values must be JSON-compatible. Unsupported values are errors,
|
|
25
|
+
not opportunities to replace evidence with string representations.
|
|
26
|
+
"""
|
|
27
|
+
data = serialize_dataclass(report)
|
|
28
|
+
data["schema_version"] = REPORT_SCHEMA_VERSION
|
|
29
|
+
content = json.dumps(data, indent=2)
|
|
30
|
+
destination = Path(output_path)
|
|
31
|
+
with NamedTemporaryFile(
|
|
32
|
+
dir=destination.parent, prefix=f".{destination.name}.", delete=False
|
|
33
|
+
) as stream:
|
|
34
|
+
temporary = Path(stream.name)
|
|
35
|
+
try:
|
|
36
|
+
temporary.write_text(content, encoding="utf-8")
|
|
37
|
+
os.replace(temporary, destination)
|
|
38
|
+
finally:
|
|
39
|
+
temporary.unlink(missing_ok=True)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def load_suite_report(path: str | Path) -> SuiteReport:
|
|
43
|
+
"""Load current-schema native evidence; never infer results or grade answers."""
|
|
44
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
45
|
+
if not isinstance(data, dict):
|
|
46
|
+
raise ValueError("Execution evidence must be a JSON object")
|
|
47
|
+
if data.get("schema_version") != REPORT_SCHEMA_VERSION:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"Unsupported schema version: {data.get('schema_version')!r}. "
|
|
50
|
+
f"Only {REPORT_SCHEMA_VERSION} is supported. Re-run the producer "
|
|
51
|
+
"with the current package to save new evidence."
|
|
52
|
+
)
|
|
53
|
+
if "insights" in data:
|
|
54
|
+
raise ValueError("AI insights are not part of execution evidence")
|
|
55
|
+
return deserialize_suite_report(data)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Test harnesses for pytest-copilot integration testing.
|
|
2
|
+
|
|
3
|
+
Provides stateful backends and MCP servers for testing agent behavior.
|
|
4
|
+
|
|
5
|
+
Available Test Servers
|
|
6
|
+
----------------------
|
|
7
|
+
BankingService
|
|
8
|
+
Banking operations with business rules and error handling. Stateful.
|
|
9
|
+
|
|
10
|
+
TodoStore
|
|
11
|
+
Task management for CRUD operation tests. Stateful.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from pytest_copilot.testing.banking import BankingService
|
|
15
|
+
from pytest_copilot.testing.todo import TodoStore
|
|
16
|
+
|
|
17
|
+
__all__ = ["BankingService", "TodoStore"]
|