pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,380 @@
1
+ """Todo store for testing - task management.
2
+
3
+ Provides a stateful todo list for testing natural language → CRUD operations.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from dataclasses import dataclass, field
9
+ from typing import Any
10
+ from uuid import uuid4
11
+
12
+ from pytest_copilot.testing.types import ToolResult
13
+
14
+
15
+ @dataclass(slots=True)
16
+ class Task:
17
+ """A todo task."""
18
+
19
+ id: str
20
+ title: str
21
+ list_name: str = "default"
22
+ completed: bool = False
23
+ priority: str = "normal" # low, normal, high
24
+
25
+
26
+ @dataclass(slots=True)
27
+ class TodoStore:
28
+ """In-memory todo list store for testing."""
29
+
30
+ tasks: dict[str, Task] = field(default_factory=dict)
31
+
32
+ def add_task(
33
+ self, title: str, list_name: str = "default", priority: str = "normal"
34
+ ) -> ToolResult:
35
+ """Add a new task.
36
+
37
+ Args:
38
+ title: Task description
39
+ list_name: Which list to add to (default: "default")
40
+ priority: Task priority (low, normal, high)
41
+
42
+ Returns:
43
+ ToolResult with the created task
44
+ """
45
+ if priority not in ("low", "normal", "high"):
46
+ return ToolResult(
47
+ success=False,
48
+ value=None,
49
+ error=f"Invalid priority '{priority}'. Must be: low, normal, high",
50
+ )
51
+
52
+ task_id = str(uuid4())[:8]
53
+ task = Task(
54
+ id=task_id,
55
+ title=title,
56
+ list_name=list_name.lower().strip(),
57
+ priority=priority,
58
+ )
59
+ self.tasks[task_id] = task
60
+
61
+ return ToolResult(
62
+ success=True,
63
+ value={
64
+ "id": task.id,
65
+ "title": task.title,
66
+ "list": task.list_name,
67
+ "priority": task.priority,
68
+ "completed": task.completed,
69
+ "message": f"Added task '{title}' to {list_name} list",
70
+ },
71
+ )
72
+
73
+ def complete_task(self, task_id: str) -> ToolResult:
74
+ """Mark a task as completed.
75
+
76
+ Args:
77
+ task_id: The task ID to complete
78
+
79
+ Returns:
80
+ ToolResult with updated task
81
+ """
82
+ if task_id not in self.tasks:
83
+ return ToolResult(
84
+ success=False,
85
+ value=None,
86
+ error=f"Task '{task_id}' not found",
87
+ )
88
+
89
+ task = self.tasks[task_id]
90
+ task.completed = True
91
+
92
+ return ToolResult(
93
+ success=True,
94
+ value={
95
+ "id": task.id,
96
+ "title": task.title,
97
+ "completed": True,
98
+ "message": f"Completed task: {task.title}",
99
+ },
100
+ )
101
+
102
+ def uncomplete_task(self, task_id: str) -> ToolResult:
103
+ """Mark a task as not completed.
104
+
105
+ Args:
106
+ task_id: The task ID to uncomplete
107
+
108
+ Returns:
109
+ ToolResult with updated task
110
+ """
111
+ if task_id not in self.tasks:
112
+ return ToolResult(
113
+ success=False,
114
+ value=None,
115
+ error=f"Task '{task_id}' not found",
116
+ )
117
+
118
+ task = self.tasks[task_id]
119
+ task.completed = False
120
+
121
+ return ToolResult(
122
+ success=True,
123
+ value={
124
+ "id": task.id,
125
+ "title": task.title,
126
+ "completed": False,
127
+ "message": f"Reopened task: {task.title}",
128
+ },
129
+ )
130
+
131
+ def delete_task(self, task_id: str) -> ToolResult:
132
+ """Delete a task.
133
+
134
+ Args:
135
+ task_id: The task ID to delete
136
+
137
+ Returns:
138
+ ToolResult confirming deletion
139
+ """
140
+ if task_id not in self.tasks:
141
+ return ToolResult(
142
+ success=False,
143
+ value=None,
144
+ error=f"Task '{task_id}' not found",
145
+ )
146
+
147
+ task = self.tasks.pop(task_id)
148
+ return ToolResult(
149
+ success=True,
150
+ value={"message": f"Deleted task: {task.title}"},
151
+ )
152
+
153
+ def list_tasks(self, list_name: str | None = None, show_completed: bool = True) -> ToolResult:
154
+ """List tasks, optionally filtered.
155
+
156
+ Args:
157
+ list_name: Filter by list name (optional)
158
+ show_completed: Include completed tasks (default True)
159
+
160
+ Returns:
161
+ ToolResult with list of tasks
162
+ """
163
+ tasks = list(self.tasks.values())
164
+
165
+ if list_name:
166
+ tasks = [t for t in tasks if t.list_name == list_name.lower().strip()]
167
+
168
+ if not show_completed:
169
+ tasks = [t for t in tasks if not t.completed]
170
+
171
+ # Sort by priority (high first), then by title
172
+ priority_order = {"high": 0, "normal": 1, "low": 2}
173
+ tasks.sort(key=lambda t: (priority_order.get(t.priority, 1), t.title))
174
+
175
+ return ToolResult(
176
+ success=True,
177
+ value=[
178
+ {
179
+ "id": t.id,
180
+ "title": t.title,
181
+ "list": t.list_name,
182
+ "priority": t.priority,
183
+ "completed": t.completed,
184
+ }
185
+ for t in tasks
186
+ ],
187
+ )
188
+
189
+ def get_lists(self) -> ToolResult:
190
+ """Get all list names.
191
+
192
+ Returns:
193
+ ToolResult with list of unique list names
194
+ """
195
+ lists = sorted(set(t.list_name for t in self.tasks.values()))
196
+ if not lists:
197
+ lists = ["default"]
198
+ return ToolResult(success=True, value=lists)
199
+
200
+ def set_priority(self, task_id: str, priority: str) -> ToolResult:
201
+ """Set task priority.
202
+
203
+ Args:
204
+ task_id: The task ID
205
+ priority: New priority (low, normal, high)
206
+
207
+ Returns:
208
+ ToolResult with updated task
209
+ """
210
+ if task_id not in self.tasks:
211
+ return ToolResult(
212
+ success=False,
213
+ value=None,
214
+ error=f"Task '{task_id}' not found",
215
+ )
216
+
217
+ if priority not in ("low", "normal", "high"):
218
+ return ToolResult(
219
+ success=False,
220
+ value=None,
221
+ error=f"Invalid priority '{priority}'. Must be: low, normal, high",
222
+ )
223
+
224
+ task = self.tasks[task_id]
225
+ task.priority = priority
226
+
227
+ return ToolResult(
228
+ success=True,
229
+ value={
230
+ "id": task.id,
231
+ "title": task.title,
232
+ "priority": priority,
233
+ "message": f"Set priority of '{task.title}' to {priority}",
234
+ },
235
+ )
236
+
237
+ def call_tool(self, name: str, arguments: dict[str, Any]) -> ToolResult:
238
+ """Dispatch a tool call by name."""
239
+ tools = {
240
+ "add_task": lambda args: self.add_task(
241
+ args["title"],
242
+ args.get("list", "default"),
243
+ args.get("priority", "normal"),
244
+ ),
245
+ "complete_task": lambda args: self.complete_task(args["task_id"]),
246
+ "uncomplete_task": lambda args: self.uncomplete_task(args["task_id"]),
247
+ "delete_task": lambda args: self.delete_task(args["task_id"]),
248
+ "list_tasks": lambda args: self.list_tasks(
249
+ args.get("list"),
250
+ args.get("show_completed", True),
251
+ ),
252
+ "get_lists": lambda _: self.get_lists(),
253
+ "set_priority": lambda args: self.set_priority(args["task_id"], args["priority"]),
254
+ }
255
+
256
+ if name not in tools:
257
+ return ToolResult(success=False, value=None, error=f"Unknown tool: {name}")
258
+
259
+ try:
260
+ return tools[name](arguments)
261
+ except KeyError as e:
262
+ return ToolResult(success=False, value=None, error=f"Missing argument: {e}")
263
+ except Exception as e:
264
+ return ToolResult(success=False, value=None, error=str(e))
265
+
266
+ async def call_tool_async(self, name: str, arguments: dict[str, Any]) -> ToolResult:
267
+ """Async dispatch (all tools are sync for todo)."""
268
+ return self.call_tool(name, arguments)
269
+
270
+ @classmethod
271
+ def get_tool_schemas(cls) -> list[dict[str, Any]]:
272
+ """Return JSON schemas for all tools (for MCP)."""
273
+ return [
274
+ {
275
+ "name": "add_task",
276
+ "description": "Add a new task to a todo list. Use for adding items like 'buy milk' or 'call mom'.",
277
+ "inputSchema": {
278
+ "type": "object",
279
+ "properties": {
280
+ "title": {
281
+ "type": "string",
282
+ "description": "The task description (e.g., 'Buy groceries', 'Call dentist')",
283
+ },
284
+ "list": {
285
+ "type": "string",
286
+ "description": "Which list to add to (e.g., 'shopping', 'work', 'personal'). Default: 'default'",
287
+ },
288
+ "priority": {
289
+ "type": "string",
290
+ "enum": ["low", "normal", "high"],
291
+ "description": "Task priority. Default: 'normal'",
292
+ },
293
+ },
294
+ "required": ["title"],
295
+ },
296
+ },
297
+ {
298
+ "name": "complete_task",
299
+ "description": "Mark a task as completed/done.",
300
+ "inputSchema": {
301
+ "type": "object",
302
+ "properties": {
303
+ "task_id": {
304
+ "type": "string",
305
+ "description": "The ID of the task to complete",
306
+ },
307
+ },
308
+ "required": ["task_id"],
309
+ },
310
+ },
311
+ {
312
+ "name": "uncomplete_task",
313
+ "description": "Mark a completed task as not done (reopen it).",
314
+ "inputSchema": {
315
+ "type": "object",
316
+ "properties": {
317
+ "task_id": {
318
+ "type": "string",
319
+ "description": "The ID of the task to reopen",
320
+ },
321
+ },
322
+ "required": ["task_id"],
323
+ },
324
+ },
325
+ {
326
+ "name": "delete_task",
327
+ "description": "Delete/remove a task permanently.",
328
+ "inputSchema": {
329
+ "type": "object",
330
+ "properties": {
331
+ "task_id": {
332
+ "type": "string",
333
+ "description": "The ID of the task to delete",
334
+ },
335
+ },
336
+ "required": ["task_id"],
337
+ },
338
+ },
339
+ {
340
+ "name": "list_tasks",
341
+ "description": "List all tasks, optionally filtered by list name or completion status.",
342
+ "inputSchema": {
343
+ "type": "object",
344
+ "properties": {
345
+ "list": {
346
+ "type": "string",
347
+ "description": "Filter by list name (e.g., 'shopping'). Omit for all lists.",
348
+ },
349
+ "show_completed": {
350
+ "type": "boolean",
351
+ "description": "Include completed tasks? Default: true",
352
+ },
353
+ },
354
+ },
355
+ },
356
+ {
357
+ "name": "get_lists",
358
+ "description": "Get all available todo list names.",
359
+ "inputSchema": {"type": "object", "properties": {}},
360
+ },
361
+ {
362
+ "name": "set_priority",
363
+ "description": "Change the priority of a task.",
364
+ "inputSchema": {
365
+ "type": "object",
366
+ "properties": {
367
+ "task_id": {
368
+ "type": "string",
369
+ "description": "The ID of the task",
370
+ },
371
+ "priority": {
372
+ "type": "string",
373
+ "enum": ["low", "normal", "high"],
374
+ "description": "New priority level",
375
+ },
376
+ },
377
+ "required": ["task_id", "priority"],
378
+ },
379
+ },
380
+ ]
@@ -0,0 +1,156 @@
1
+ """Todo MCP server for integration testing.
2
+
3
+ Run as: python -m pytest_copilot.testing.todo_mcp
4
+
5
+ Supports all three transports:
6
+ - stdio (default): ``python -m pytest_copilot.testing.todo_mcp``
7
+ - streamable-http: ``python -m pytest_copilot.testing.todo_mcp --transport streamable-http --port 8080``
8
+ - sse: ``python -m pytest_copilot.testing.todo_mcp --transport sse --port 8080``
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import json
15
+
16
+ from mcp.server import MCPServer
17
+
18
+ from pytest_copilot.testing.todo import TodoStore
19
+
20
+ # ---------------------------------------------------------------------------
21
+ # Server & service
22
+ # ---------------------------------------------------------------------------
23
+
24
+ mcp = MCPServer("pytest-copilot-todo-server")
25
+ _store = TodoStore()
26
+
27
+ # ---------------------------------------------------------------------------
28
+ # Tools – thin wrappers that delegate to TodoStore
29
+ # ---------------------------------------------------------------------------
30
+
31
+
32
+ @mcp.tool()
33
+ def add_task(title: str, list: str = "default", priority: str = "normal") -> str:
34
+ """Add a new task to a todo list.
35
+
36
+ Args:
37
+ title: The task description (e.g., 'Buy groceries', 'Call dentist').
38
+ list: Which list to add to (e.g., 'shopping', 'work', 'personal'). Default: 'default'.
39
+ priority: Task priority ('low', 'normal', 'high'). Default: 'normal'.
40
+ """
41
+ result = _store.add_task(title, list, priority)
42
+ if result.success:
43
+ return json.dumps(result.value)
44
+ return f"Error: {result.error}"
45
+
46
+
47
+ @mcp.tool()
48
+ def complete_task(task_id: str) -> str:
49
+ """Mark a task as completed/done.
50
+
51
+ Args:
52
+ task_id: The ID of the task to complete.
53
+ """
54
+ result = _store.complete_task(task_id)
55
+ if result.success:
56
+ return json.dumps(result.value)
57
+ return f"Error: {result.error}"
58
+
59
+
60
+ @mcp.tool()
61
+ def uncomplete_task(task_id: str) -> str:
62
+ """Mark a completed task as not done (reopen it).
63
+
64
+ Args:
65
+ task_id: The ID of the task to reopen.
66
+ """
67
+ result = _store.uncomplete_task(task_id)
68
+ if result.success:
69
+ return json.dumps(result.value)
70
+ return f"Error: {result.error}"
71
+
72
+
73
+ @mcp.tool()
74
+ def delete_task(task_id: str) -> str:
75
+ """Delete/remove a task permanently.
76
+
77
+ Args:
78
+ task_id: The ID of the task to delete.
79
+ """
80
+ result = _store.delete_task(task_id)
81
+ if result.success:
82
+ return json.dumps(result.value)
83
+ return f"Error: {result.error}"
84
+
85
+
86
+ @mcp.tool()
87
+ def list_tasks(list: str | None = None, show_completed: bool = True) -> str:
88
+ """List all tasks, optionally filtered by list name or completion status.
89
+
90
+ Args:
91
+ list: Filter by list name (e.g., 'shopping'). Omit for all lists.
92
+ show_completed: Include completed tasks? Default: true.
93
+ """
94
+ result = _store.list_tasks(list, show_completed)
95
+ if result.success:
96
+ return json.dumps(result.value)
97
+ return f"Error: {result.error}"
98
+
99
+
100
+ @mcp.tool()
101
+ def get_lists() -> str:
102
+ """Get all available todo list names."""
103
+ result = _store.get_lists()
104
+ if result.success:
105
+ return json.dumps(result.value)
106
+ return f"Error: {result.error}"
107
+
108
+
109
+ @mcp.tool()
110
+ def set_priority(task_id: str, priority: str) -> str:
111
+ """Change the priority of a task.
112
+
113
+ Args:
114
+ task_id: The ID of the task.
115
+ priority: New priority level ('low', 'normal', 'high').
116
+ """
117
+ result = _store.set_priority(task_id, priority)
118
+ if result.success:
119
+ return json.dumps(result.value)
120
+ return f"Error: {result.error}"
121
+
122
+
123
+ # ---------------------------------------------------------------------------
124
+ # Entry point
125
+ # ---------------------------------------------------------------------------
126
+
127
+
128
+ def main() -> None:
129
+ """Parse args and run the server with the chosen transport."""
130
+ parser = argparse.ArgumentParser(description="Todo MCP server")
131
+ parser.add_argument(
132
+ "--transport",
133
+ choices=["stdio", "sse", "streamable-http"],
134
+ default="stdio",
135
+ )
136
+ parser.add_argument("--port", type=int, default=8080)
137
+ parser.add_argument("--host", default="127.0.0.1")
138
+ args = parser.parse_args()
139
+
140
+ # Configure host/port via FastMCP settings
141
+ if args.transport == "stdio":
142
+ mcp.run(transport="stdio")
143
+ elif args.transport == "sse":
144
+ mcp.run(transport="sse", host=args.host, port=args.port)
145
+ elif args.transport == "streamable-http":
146
+ mcp.run(
147
+ transport="streamable-http",
148
+ host=args.host,
149
+ port=args.port,
150
+ stateless_http=True,
151
+ json_response=True,
152
+ )
153
+
154
+
155
+ if __name__ == "__main__":
156
+ main()
@@ -0,0 +1,31 @@
1
+ """Common types for test harnesses."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Any
7
+
8
+
9
+ @dataclass(slots=True)
10
+ class ToolResult:
11
+ """Result from a tool call.
12
+
13
+ All test harness methods return this type for consistent handling.
14
+
15
+ Attributes:
16
+ success: Whether the operation succeeded
17
+ value: The result value (if successful)
18
+ error: Error message (if failed)
19
+ """
20
+
21
+ success: bool
22
+ value: Any = None
23
+ error: str | None = None
24
+
25
+ def to_dict(self) -> dict[str, Any]:
26
+ """Convert to dictionary for JSON serialization."""
27
+ return {
28
+ "success": self.success,
29
+ "value": self.value,
30
+ "error": self.error,
31
+ }