pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,479 @@
1
+ """Banking service for testing - stateful account management.
2
+
3
+ Provides a realistic banking service for testing multi-turn conversations
4
+ and session-based workflows.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass, field
10
+ from datetime import datetime
11
+ from typing import Any
12
+
13
+ from pytest_copilot.testing.types import ToolResult
14
+
15
+
16
+ @dataclass(slots=True)
17
+ class Transaction:
18
+ """A banking transaction record."""
19
+
20
+ id: str
21
+ timestamp: str
22
+ type: str # "transfer", "deposit", "withdrawal"
23
+ from_account: str | None
24
+ to_account: str | None
25
+ amount: float
26
+ description: str
27
+
28
+
29
+ @dataclass(slots=True)
30
+ class BankingService:
31
+ """In-memory banking service with account management.
32
+
33
+ Maintains state across calls, enabling tests for:
34
+ - Multi-turn conversations with state changes
35
+ - Session-based workflows (check balance → transfer → verify)
36
+ - Complex prompts requiring multiple tool calls
37
+ """
38
+
39
+ accounts: dict[str, float] = field(
40
+ default_factory=lambda: {
41
+ "checking": 1500.00,
42
+ "savings": 3000.00,
43
+ }
44
+ )
45
+ transactions: list[Transaction] = field(default_factory=list)
46
+ _tx_counter: int = 0
47
+
48
+ def _next_tx_id(self) -> str:
49
+ """Generate next transaction ID."""
50
+ self._tx_counter += 1
51
+ return f"TX{self._tx_counter:04d}"
52
+
53
+ def _now(self) -> str:
54
+ """Get current timestamp."""
55
+ return datetime.now().isoformat()
56
+
57
+ def get_balance(self, account: str) -> ToolResult:
58
+ """Get the balance of an account.
59
+
60
+ Args:
61
+ account: Account name ("checking" or "savings")
62
+
63
+ Returns:
64
+ ToolResult with balance or error
65
+ """
66
+ account_lower = account.lower()
67
+ if account_lower not in self.accounts:
68
+ return ToolResult(
69
+ success=False,
70
+ value=None,
71
+ error=f"Account '{account}' not found. Available: {list(self.accounts.keys())}",
72
+ )
73
+
74
+ balance = self.accounts[account_lower]
75
+ return ToolResult(
76
+ success=True,
77
+ value={
78
+ "account": account_lower,
79
+ "balance": balance,
80
+ "formatted": f"${balance:,.2f}",
81
+ },
82
+ )
83
+
84
+ def get_all_balances(self) -> ToolResult:
85
+ """Get balances for all accounts.
86
+
87
+ Returns:
88
+ ToolResult with all account balances
89
+ """
90
+ balances = {}
91
+ total = 0.0
92
+ for name, balance in self.accounts.items():
93
+ balances[name] = {
94
+ "balance": balance,
95
+ "formatted": f"${balance:,.2f}",
96
+ }
97
+ total += balance
98
+
99
+ return ToolResult(
100
+ success=True,
101
+ value={
102
+ "accounts": balances,
103
+ "total": total,
104
+ "total_formatted": f"${total:,.2f}",
105
+ },
106
+ )
107
+
108
+ def transfer(self, from_account: str, to_account: str, amount: float) -> ToolResult:
109
+ """Transfer money between accounts.
110
+
111
+ Args:
112
+ from_account: Source account name
113
+ to_account: Destination account name
114
+ amount: Amount to transfer (positive number)
115
+
116
+ Returns:
117
+ ToolResult with transfer confirmation or error
118
+ """
119
+ from_lower = from_account.lower()
120
+ to_lower = to_account.lower()
121
+
122
+ if from_lower not in self.accounts:
123
+ return ToolResult(
124
+ success=False,
125
+ value=None,
126
+ error=f"Source account '{from_account}' not found.",
127
+ )
128
+
129
+ if to_lower not in self.accounts:
130
+ return ToolResult(
131
+ success=False,
132
+ value=None,
133
+ error=f"Destination account '{to_account}' not found.",
134
+ )
135
+
136
+ if amount <= 0:
137
+ return ToolResult(
138
+ success=False,
139
+ value=None,
140
+ error="Transfer amount must be positive.",
141
+ )
142
+
143
+ if self.accounts[from_lower] < amount:
144
+ return ToolResult(
145
+ success=False,
146
+ value=None,
147
+ error=f"Insufficient funds. {from_account} has ${self.accounts[from_lower]:,.2f}, "
148
+ f"but transfer requires ${amount:,.2f}.",
149
+ )
150
+
151
+ # Execute transfer
152
+ self.accounts[from_lower] -= amount
153
+ self.accounts[to_lower] += amount
154
+
155
+ # Record transaction
156
+ tx = Transaction(
157
+ id=self._next_tx_id(),
158
+ timestamp=self._now(),
159
+ type="transfer",
160
+ from_account=from_lower,
161
+ to_account=to_lower,
162
+ amount=amount,
163
+ description=f"Transfer from {from_lower} to {to_lower}",
164
+ )
165
+ self.transactions.append(tx)
166
+
167
+ return ToolResult(
168
+ success=True,
169
+ value={
170
+ "transaction_id": tx.id,
171
+ "type": "transfer",
172
+ "from_account": from_lower,
173
+ "to_account": to_lower,
174
+ "amount": amount,
175
+ "amount_formatted": f"${amount:,.2f}",
176
+ "new_balance_from": self.accounts[from_lower],
177
+ "new_balance_to": self.accounts[to_lower],
178
+ "message": f"Successfully transferred ${amount:,.2f} from {from_lower} to {to_lower}.",
179
+ },
180
+ )
181
+
182
+ def deposit(self, account: str, amount: float) -> ToolResult:
183
+ """Deposit money into an account.
184
+
185
+ Args:
186
+ account: Account name
187
+ amount: Amount to deposit (positive number)
188
+
189
+ Returns:
190
+ ToolResult with deposit confirmation or error
191
+ """
192
+ account_lower = account.lower()
193
+
194
+ if account_lower not in self.accounts:
195
+ return ToolResult(
196
+ success=False,
197
+ value=None,
198
+ error=f"Account '{account}' not found.",
199
+ )
200
+
201
+ if amount <= 0:
202
+ return ToolResult(
203
+ success=False,
204
+ value=None,
205
+ error="Deposit amount must be positive.",
206
+ )
207
+
208
+ self.accounts[account_lower] += amount
209
+
210
+ tx = Transaction(
211
+ id=self._next_tx_id(),
212
+ timestamp=self._now(),
213
+ type="deposit",
214
+ from_account=None,
215
+ to_account=account_lower,
216
+ amount=amount,
217
+ description=f"Deposit to {account_lower}",
218
+ )
219
+ self.transactions.append(tx)
220
+
221
+ return ToolResult(
222
+ success=True,
223
+ value={
224
+ "transaction_id": tx.id,
225
+ "type": "deposit",
226
+ "account": account_lower,
227
+ "amount": amount,
228
+ "amount_formatted": f"${amount:,.2f}",
229
+ "new_balance": self.accounts[account_lower],
230
+ "message": f"Successfully deposited ${amount:,.2f} to {account_lower}.",
231
+ },
232
+ )
233
+
234
+ def withdraw(self, account: str, amount: float) -> ToolResult:
235
+ """Withdraw money from an account.
236
+
237
+ Args:
238
+ account: Account name
239
+ amount: Amount to withdraw (positive number)
240
+
241
+ Returns:
242
+ ToolResult with withdrawal confirmation or error
243
+ """
244
+ account_lower = account.lower()
245
+
246
+ if account_lower not in self.accounts:
247
+ return ToolResult(
248
+ success=False,
249
+ value=None,
250
+ error=f"Account '{account}' not found.",
251
+ )
252
+
253
+ if amount <= 0:
254
+ return ToolResult(
255
+ success=False,
256
+ value=None,
257
+ error="Withdrawal amount must be positive.",
258
+ )
259
+
260
+ if self.accounts[account_lower] < amount:
261
+ return ToolResult(
262
+ success=False,
263
+ value=None,
264
+ error=f"Insufficient funds. {account} has ${self.accounts[account_lower]:,.2f}.",
265
+ )
266
+
267
+ self.accounts[account_lower] -= amount
268
+
269
+ tx = Transaction(
270
+ id=self._next_tx_id(),
271
+ timestamp=self._now(),
272
+ type="withdrawal",
273
+ from_account=account_lower,
274
+ to_account=None,
275
+ amount=amount,
276
+ description=f"Withdrawal from {account_lower}",
277
+ )
278
+ self.transactions.append(tx)
279
+
280
+ return ToolResult(
281
+ success=True,
282
+ value={
283
+ "transaction_id": tx.id,
284
+ "type": "withdrawal",
285
+ "account": account_lower,
286
+ "amount": amount,
287
+ "amount_formatted": f"${amount:,.2f}",
288
+ "new_balance": self.accounts[account_lower],
289
+ "message": f"Successfully withdrew ${amount:,.2f} from {account_lower}.",
290
+ },
291
+ )
292
+
293
+ def get_transactions(self, account: str | None = None, limit: int = 10) -> ToolResult:
294
+ """Get recent transactions.
295
+
296
+ Args:
297
+ account: Optional account to filter by
298
+ limit: Maximum number of transactions to return
299
+
300
+ Returns:
301
+ ToolResult with transaction list
302
+ """
303
+ txs = self.transactions
304
+
305
+ if account:
306
+ account_lower = account.lower()
307
+ txs = [
308
+ t for t in txs if t.from_account == account_lower or t.to_account == account_lower
309
+ ]
310
+
311
+ # Most recent first
312
+ txs = list(reversed(txs[-limit:]))
313
+
314
+ return ToolResult(
315
+ success=True,
316
+ value={
317
+ "transactions": [
318
+ {
319
+ "id": t.id,
320
+ "timestamp": t.timestamp,
321
+ "type": t.type,
322
+ "from_account": t.from_account,
323
+ "to_account": t.to_account,
324
+ "amount": t.amount,
325
+ "amount_formatted": f"${t.amount:,.2f}",
326
+ "description": t.description,
327
+ }
328
+ for t in txs
329
+ ],
330
+ "count": len(txs),
331
+ "filter": account,
332
+ },
333
+ )
334
+
335
+ def get_tool_schemas(self) -> list[dict[str, Any]]:
336
+ """Get MCP tool schemas for all operations."""
337
+ return [
338
+ {
339
+ "name": "get_balance",
340
+ "description": "Get the current balance of a specific account (checking or savings).",
341
+ "inputSchema": {
342
+ "type": "object",
343
+ "properties": {
344
+ "account": {
345
+ "type": "string",
346
+ "description": "Account name: 'checking' or 'savings'",
347
+ "enum": ["checking", "savings"],
348
+ },
349
+ },
350
+ "required": ["account"],
351
+ },
352
+ },
353
+ {
354
+ "name": "get_all_balances",
355
+ "description": "Get balances for all accounts at once, including total.",
356
+ "inputSchema": {
357
+ "type": "object",
358
+ "properties": {},
359
+ },
360
+ },
361
+ {
362
+ "name": "transfer",
363
+ "description": "Transfer money between accounts (checking and savings).",
364
+ "inputSchema": {
365
+ "type": "object",
366
+ "properties": {
367
+ "from_account": {
368
+ "type": "string",
369
+ "description": "Source account name",
370
+ "enum": ["checking", "savings"],
371
+ },
372
+ "to_account": {
373
+ "type": "string",
374
+ "description": "Destination account name",
375
+ "enum": ["checking", "savings"],
376
+ },
377
+ "amount": {
378
+ "type": "number",
379
+ "description": "Amount to transfer in dollars",
380
+ },
381
+ },
382
+ "required": ["from_account", "to_account", "amount"],
383
+ },
384
+ },
385
+ {
386
+ "name": "deposit",
387
+ "description": "Deposit money into an account.",
388
+ "inputSchema": {
389
+ "type": "object",
390
+ "properties": {
391
+ "account": {
392
+ "type": "string",
393
+ "description": "Account to deposit into",
394
+ "enum": ["checking", "savings"],
395
+ },
396
+ "amount": {
397
+ "type": "number",
398
+ "description": "Amount to deposit in dollars",
399
+ },
400
+ },
401
+ "required": ["account", "amount"],
402
+ },
403
+ },
404
+ {
405
+ "name": "withdraw",
406
+ "description": "Withdraw money from an account.",
407
+ "inputSchema": {
408
+ "type": "object",
409
+ "properties": {
410
+ "account": {
411
+ "type": "string",
412
+ "description": "Account to withdraw from",
413
+ "enum": ["checking", "savings"],
414
+ },
415
+ "amount": {
416
+ "type": "number",
417
+ "description": "Amount to withdraw in dollars",
418
+ },
419
+ },
420
+ "required": ["account", "amount"],
421
+ },
422
+ },
423
+ {
424
+ "name": "get_transactions",
425
+ "description": "Get recent transaction history, optionally filtered by account.",
426
+ "inputSchema": {
427
+ "type": "object",
428
+ "properties": {
429
+ "account": {
430
+ "type": "string",
431
+ "description": "Optional: filter by account name",
432
+ "enum": ["checking", "savings"],
433
+ },
434
+ "limit": {
435
+ "type": "integer",
436
+ "description": "Maximum number of transactions to return (default: 10)",
437
+ "default": 10,
438
+ },
439
+ },
440
+ },
441
+ },
442
+ ]
443
+
444
+ async def call_tool_async(self, name: str, arguments: dict[str, Any]) -> ToolResult:
445
+ """Call a tool by name with arguments."""
446
+ match name:
447
+ case "get_balance":
448
+ return self.get_balance(arguments.get("account", ""))
449
+ case "get_all_balances":
450
+ return self.get_all_balances()
451
+ case "transfer":
452
+ return self.transfer(
453
+ arguments.get("from_account", ""),
454
+ arguments.get("to_account", ""),
455
+ arguments.get("amount", 0),
456
+ )
457
+ case "deposit":
458
+ return self.deposit(
459
+ arguments.get("account", ""),
460
+ arguments.get("amount", 0),
461
+ )
462
+ case "withdraw":
463
+ return self.withdraw(
464
+ arguments.get("account", ""),
465
+ arguments.get("amount", 0),
466
+ )
467
+ case "get_transactions":
468
+ return self.get_transactions(
469
+ arguments.get("account"),
470
+ arguments.get("limit", 10),
471
+ )
472
+ case _:
473
+ return ToolResult(
474
+ success=False,
475
+ value=None,
476
+ error=f"Unknown tool: {name}",
477
+ )
478
+ # Unreachable: match is exhaustive, but satisfies static analyzers
479
+ return ToolResult(success=False, value=None, error=f"Unhandled tool: {name}")
@@ -0,0 +1,167 @@
1
+ """Banking MCP server for integration testing.
2
+
3
+ Run as: python -m pytest_copilot.testing.banking_mcp
4
+
5
+ Supports all three transports:
6
+ - stdio (default): ``python -m pytest_copilot.testing.banking_mcp``
7
+ - streamable-http: ``python -m pytest_copilot.testing.banking_mcp --transport streamable-http --port 8080``
8
+ - sse: ``python -m pytest_copilot.testing.banking_mcp --transport sse --port 8080``
9
+
10
+ This server maintains state across calls, enabling tests for:
11
+ - Multi-turn conversations where account balances change
12
+ - Session-based workflows (check → transfer → verify)
13
+ - Complex prompts requiring multiple tool calls
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import argparse
19
+ import json
20
+
21
+ from mcp.server import MCPServer
22
+
23
+ from pytest_copilot.testing.banking import BankingService
24
+
25
+ # ---------------------------------------------------------------------------
26
+ # Server & service
27
+ # ---------------------------------------------------------------------------
28
+
29
+ mcp = MCPServer("pytest-copilot-banking-server")
30
+ _service = BankingService()
31
+
32
+ # ---------------------------------------------------------------------------
33
+ # Tools – thin wrappers that delegate to BankingService
34
+ # ---------------------------------------------------------------------------
35
+
36
+
37
+ @mcp.tool()
38
+ def get_balance(account: str) -> str:
39
+ """Get the current balance for a specific account.
40
+
41
+ Args:
42
+ account: Account name (e.g. "checking" or "savings").
43
+ """
44
+ result = _service.get_balance(account)
45
+ if result.success:
46
+ return json.dumps(result.value)
47
+ return f"Error: {result.error}"
48
+
49
+
50
+ @mcp.tool()
51
+ def get_all_balances() -> str:
52
+ """Get balances for all accounts at once."""
53
+ result = _service.get_all_balances()
54
+ if result.success:
55
+ return json.dumps(result.value)
56
+ return f"Error: {result.error}"
57
+
58
+
59
+ @mcp.tool()
60
+ def transfer(from_account: str, to_account: str, amount: float) -> str:
61
+ """Transfer money between accounts.
62
+
63
+ Args:
64
+ from_account: Source account name.
65
+ to_account: Destination account name.
66
+ amount: Amount to transfer (positive number).
67
+ """
68
+ result = _service.transfer(from_account, to_account, amount)
69
+ if result.success:
70
+ return json.dumps(result.value)
71
+ return f"Error: {result.error}"
72
+
73
+
74
+ @mcp.tool()
75
+ def deposit(account: str, amount: float) -> str:
76
+ """Deposit money into an account.
77
+
78
+ Args:
79
+ account: Account name.
80
+ amount: Amount to deposit (positive number).
81
+ """
82
+ result = _service.deposit(account, amount)
83
+ if result.success:
84
+ return json.dumps(result.value)
85
+ return f"Error: {result.error}"
86
+
87
+
88
+ @mcp.tool()
89
+ def withdraw(account: str, amount: float) -> str:
90
+ """Withdraw money from an account.
91
+
92
+ Args:
93
+ account: Account name.
94
+ amount: Amount to withdraw (positive number).
95
+ """
96
+ result = _service.withdraw(account, amount)
97
+ if result.success:
98
+ return json.dumps(result.value)
99
+ return f"Error: {result.error}"
100
+
101
+
102
+ @mcp.tool()
103
+ def get_transactions(account: str | None = None, limit: int = 10) -> str:
104
+ """View transaction history.
105
+
106
+ Args:
107
+ account: Optional account name to filter by.
108
+ limit: Maximum number of transactions to return (default 10).
109
+ """
110
+ result = _service.get_transactions(account=account, limit=limit)
111
+ if result.success:
112
+ return json.dumps(result.value)
113
+ return f"Error: {result.error}"
114
+
115
+
116
+ # ---------------------------------------------------------------------------
117
+ # Prompts – reusable prompt templates
118
+ # ---------------------------------------------------------------------------
119
+
120
+
121
+ @mcp.prompt()
122
+ def account_summary(account: str = "checking") -> str:
123
+ """Get a summary of account activity.
124
+
125
+ Args:
126
+ account: Account name to summarize (default: "checking").
127
+ """
128
+ return (
129
+ f"Please provide a summary of the {account} account. "
130
+ f"Include the current balance and recent transactions."
131
+ )
132
+
133
+
134
+ # ---------------------------------------------------------------------------
135
+ # Entry point
136
+ # ---------------------------------------------------------------------------
137
+
138
+
139
+ def main() -> None:
140
+ """Parse args and run the server with the chosen transport."""
141
+ parser = argparse.ArgumentParser(description="Banking MCP server")
142
+ parser.add_argument(
143
+ "--transport",
144
+ choices=["stdio", "sse", "streamable-http"],
145
+ default="stdio",
146
+ )
147
+ parser.add_argument("--port", type=int, default=8080)
148
+ parser.add_argument("--host", default="127.0.0.1")
149
+ args = parser.parse_args()
150
+
151
+ # Configure host/port via FastMCP settings
152
+ if args.transport == "stdio":
153
+ mcp.run(transport="stdio")
154
+ elif args.transport == "sse":
155
+ mcp.run(transport="sse", host=args.host, port=args.port)
156
+ elif args.transport == "streamable-http":
157
+ mcp.run(
158
+ transport="streamable-http",
159
+ host=args.host,
160
+ port=args.port,
161
+ stateless_http=True,
162
+ json_response=True,
163
+ )
164
+
165
+
166
+ if __name__ == "__main__":
167
+ main()