pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,521 @@
1
+ """Custom agent file loader for ``.agent.md`` and ``.md`` files.
2
+
3
+ Loads VS Code (``.github/agents/*.agent.md``) and Claude Code
4
+ (``.claude/agents/*.md``) custom agent definitions into dicts
5
+ compatible with :attr:`CopilotTask.custom_agents`.
6
+
7
+ Eval files use YAML frontmatter for metadata and a markdown body for
8
+ the agent's prompt/instructions.
9
+
10
+ Example ``.agent.md`` file::
11
+
12
+ ---
13
+ description: 'Research specialist for codebase analysis'
14
+ maturity: stable
15
+ tools:
16
+ - read_file
17
+ - list_directory
18
+ ---
19
+
20
+ # Task Researcher
21
+
22
+ Research-only specialist. Produces findings in ``.copilot-tracking/research/``.
23
+
24
+ Example usage::
25
+
26
+ from pytest_copilot.core.evals import load_custom_agent, load_custom_agents
27
+
28
+ # Single agent
29
+ researcher = load_custom_agent("agents/task-researcher.agent.md")
30
+ # → {"name": "task-researcher", "prompt": "# Task Researcher\\n...",
31
+ # "description": "...", "metadata": {...}}
32
+
33
+ # All agents from a directory
34
+ agents = load_custom_agents("agents/")
35
+
36
+ # Use with CopilotTask
37
+ from pytest_copilot.copilot import CopilotTask
38
+ agent = CopilotTask(
39
+ name="orchestrator",
40
+ instructions="Dispatch tasks to specialized agents.",
41
+ custom_agents=load_custom_agents("agents/", exclude={"orchestrator"}),
42
+ )
43
+
44
+ Also provides prompt file loaders for VS Code prompt files
45
+ (``.github/prompts/*.prompt.md``) and Claude Code commands
46
+ (``.claude/commands/*.md``) — reusable slash-command prompts::
47
+
48
+ from pytest_copilot.core.evals import load_prompt_file, load_prompt_files
49
+
50
+ # Single prompt file
51
+ prompt = load_prompt_file(".github/prompts/review.prompt.md")
52
+ # → {"name": "review", "body": "Review this code for...",
53
+ # "description": "...", "metadata": {...}}
54
+
55
+ # Use the body as the test input
56
+ result = await copilot_run(agent, prompt["body"])
57
+
58
+ # Load all from directory
59
+ prompts = load_prompt_files(".github/prompts/")
60
+ """
61
+
62
+ from __future__ import annotations
63
+
64
+ import re
65
+ from pathlib import Path
66
+ from typing import Any
67
+
68
+ import yaml
69
+
70
+ _FRONTMATTER_RE = re.compile(
71
+ r"\A---[ \t]*\r?\n(.*?)^---[ \t]*(?:\r?\n|\Z)", re.DOTALL | re.MULTILINE
72
+ )
73
+
74
+
75
+ def _extract_frontmatter(content: str, *, path: Path | None = None) -> tuple[dict[str, Any], str]:
76
+ """Split content into parsed frontmatter dict and body.
77
+
78
+ Returns:
79
+ Tuple of (frontmatter_dict, body). Frontmatter dict is empty
80
+ if no frontmatter block is present. Malformed frontmatter raises
81
+ ValueError rather than being treated as system prompt content.
82
+ """
83
+ match = _FRONTMATTER_RE.match(content)
84
+ if not match:
85
+ if re.match(r"\A---[ \t]*(?:\r?\n|\Z)", content):
86
+ raise ValueError(f"{path}: YAML frontmatter is missing its closing --- delimiter")
87
+ return {}, content
88
+
89
+ raw = match.group(1)
90
+ body = content[match.end() :]
91
+
92
+ try:
93
+ parsed = yaml.safe_load(raw)
94
+ except yaml.YAMLError as exc:
95
+ raise ValueError(f"{path}: Invalid YAML frontmatter: {exc}") from exc
96
+
97
+ if not isinstance(parsed, dict):
98
+ raise ValueError(f"{path}: YAML frontmatter must be a mapping")
99
+ if any(not isinstance(key, str) for key in parsed):
100
+ raise ValueError(f"{path}: YAML frontmatter keys must be strings")
101
+
102
+ return parsed, body
103
+
104
+
105
+ def _validate_metadata(metadata: dict[str, Any], path: Path) -> None:
106
+ """Validate supported common metadata without coercing malformed values."""
107
+ for field in ("name", "description", "applyTo", "model"):
108
+ if field in metadata and (
109
+ not isinstance(metadata[field], str) or not metadata[field].strip()
110
+ ):
111
+ raise ValueError(f"{path}: '{field}' must be a non-empty string")
112
+ if "tools" in metadata:
113
+ tools = metadata["tools"]
114
+ if not (
115
+ isinstance(tools, str)
116
+ and tools.strip()
117
+ or isinstance(tools, list)
118
+ and all(isinstance(tool, str) and tool.strip() for tool in tools)
119
+ ):
120
+ raise ValueError(f"{path}: 'tools' must be a string or a list of non-empty strings")
121
+
122
+
123
+ def _read_markdown(path: Path) -> tuple[dict[str, Any], str]:
124
+ """Read configuration as UTF-8 and retain the source path on parse errors."""
125
+ try:
126
+ content = path.read_text(encoding="utf-8")
127
+ except UnicodeError as exc:
128
+ raise ValueError(f"{path}: configuration must be valid UTF-8: {exc}") from exc
129
+ metadata, body = _extract_frontmatter(content, path=path)
130
+ _validate_metadata(metadata, path)
131
+ return metadata, body.strip()
132
+
133
+
134
+ def _name_from_path(path: Path) -> str:
135
+ """Derive agent name from filename.
136
+
137
+ ``task-researcher.agent.md`` → ``task-researcher``
138
+ """
139
+ name = path.name
140
+ if name.endswith(".agent.md"):
141
+ return name[: -len(".agent.md")]
142
+ return path.stem
143
+
144
+
145
+ def load_custom_agent(
146
+ path: Path | str,
147
+ *,
148
+ overrides: dict[str, Any] | None = None,
149
+ ) -> dict[str, Any]:
150
+ """Load a ``.agent.md`` or ``.md`` file into a custom agent dict.
151
+
152
+ Parses YAML frontmatter with PyYAML for structured metadata
153
+ (``description``, ``tools``, ``handoffs``, ``maturity``, etc.) and uses
154
+ the markdown body as the agent's ``prompt``.
155
+
156
+ Args:
157
+ path: Path to the ``.agent.md`` or ``.md`` file.
158
+ overrides: Additional fields to merge into the result. Use this
159
+ to set ``tools``, ``mcp_servers``, or ``infer`` fields that
160
+ aren't in the agent file itself.
161
+
162
+ Returns:
163
+ Dict with keys:
164
+ - ``name`` (str): Declared name, or derived from filename when absent.
165
+ - ``prompt`` (str): Markdown body after frontmatter.
166
+ - ``description`` (str): From frontmatter, empty if absent.
167
+ - ``metadata`` (dict): Full parsed frontmatter dict.
168
+ Compatible with :attr:`CopilotTask.custom_agents`.
169
+
170
+ Raises:
171
+ FileNotFoundError: If the file does not exist.
172
+ ValueError: If the file has no content after frontmatter stripping.
173
+ """
174
+ path = Path(path)
175
+ if not path.exists():
176
+ msg = f"Eval file not found: {path}"
177
+ raise FileNotFoundError(msg)
178
+
179
+ metadata, body = _read_markdown(path)
180
+
181
+ if not body:
182
+ msg = f"Eval file has no content after frontmatter: {path}"
183
+ raise ValueError(msg)
184
+
185
+ config: dict[str, Any] = {
186
+ "name": metadata.get("name", _name_from_path(path)),
187
+ "prompt": body,
188
+ "description": metadata.get("description", ""),
189
+ "metadata": metadata,
190
+ }
191
+ for key in ("display_name", "infer", "skills", "model", "reasoning_effort", "tools"):
192
+ if key in metadata:
193
+ config[key] = metadata[key]
194
+ if "mcp-servers" in metadata:
195
+ from pytest_copilot.core.validation import _validate_mcp_servers
196
+
197
+ config["mcp_servers"] = _validate_mcp_servers(
198
+ metadata["mcp-servers"], path, field="mcp-servers"
199
+ )
200
+
201
+ if overrides:
202
+ config.update(overrides)
203
+
204
+ for field in ("name", "prompt"):
205
+ if not isinstance(config[field], str) or not config[field].strip():
206
+ raise ValueError(f"{path}: '{field}' must be a non-empty string")
207
+ _validate_metadata({**metadata, **(overrides or {})}, path)
208
+ if isinstance(config.get("tools"), str):
209
+ config["tools"] = config["tools"].replace(",", " ").split()
210
+ if "infer" in config and type(config["infer"]) is not bool:
211
+ raise ValueError(f"{path}: 'infer' must be a boolean")
212
+ if "skills" in config and not (
213
+ isinstance(config["skills"], list)
214
+ and all(isinstance(skill, str) and skill.strip() for skill in config["skills"])
215
+ ):
216
+ raise ValueError(f"{path}: 'skills' must be a list of non-empty strings")
217
+ if "reasoning_effort" in config and config["reasoning_effort"] not in (
218
+ "low",
219
+ "medium",
220
+ "high",
221
+ "xhigh",
222
+ ):
223
+ raise ValueError(f"{path}: invalid 'reasoning_effort'")
224
+ return config
225
+
226
+
227
+ def load_custom_agents(
228
+ directory: Path | str,
229
+ *,
230
+ include: set[str] | None = None,
231
+ exclude: set[str] | None = None,
232
+ overrides: dict[str, dict[str, Any]] | None = None,
233
+ ) -> list[dict[str, Any]]:
234
+ """Load all ``.agent.md`` files from a directory.
235
+
236
+ Args:
237
+ directory: Path to directory containing ``.agent.md`` files.
238
+ include: If set, only load agents with these names. Names are
239
+ derived from filenames (e.g. ``task-researcher.agent.md``
240
+ → ``task-researcher``).
241
+ exclude: Eval names to skip.
242
+ overrides: Per-agent override dicts keyed by agent name. Merged
243
+ into each matching agent's config.
244
+
245
+ Returns:
246
+ List of custom agent dicts, sorted by name.
247
+
248
+ Raises:
249
+ FileNotFoundError: If the directory does not exist.
250
+ """
251
+ directory = Path(directory)
252
+ if not directory.is_dir():
253
+ msg = f"Eval directory not found: {directory}"
254
+ raise FileNotFoundError(msg)
255
+
256
+ agents: list[dict[str, Any]] = []
257
+ for path in sorted(directory.glob("*.agent.md")):
258
+ loaded = load_custom_agent(path)
259
+ name = loaded["name"]
260
+
261
+ if include is not None and name not in include:
262
+ continue
263
+ if exclude is not None and name in exclude:
264
+ continue
265
+
266
+ agent_overrides = (overrides or {}).get(name)
267
+ agent = load_custom_agent(path, overrides=agent_overrides) if agent_overrides else loaded
268
+ if any(existing["name"] == agent["name"] for existing in agents):
269
+ raise ValueError(f"{path}: Duplicate custom agent name '{agent['name']}'")
270
+ agents.append(agent)
271
+
272
+ return sorted(agents, key=lambda agent: agent["name"])
273
+
274
+
275
+ def _prompt_name_from_path(path: Path) -> str:
276
+ """Derive prompt name from filename.
277
+
278
+ ``review.prompt.md`` → ``review``
279
+ ``code-review.md`` → ``code-review``
280
+ """
281
+ name = path.name
282
+ if name.endswith(".prompt.md"):
283
+ return name[: -len(".prompt.md")]
284
+ return path.stem
285
+
286
+
287
+ def load_prompt_file(path: Path | str) -> dict[str, Any]:
288
+ """Load a VS Code prompt file or Claude Code command file.
289
+
290
+ Reads a ``.prompt.md`` file (VS Code: ``.github/prompts/``) or a plain
291
+ ``.md`` command file (Claude Code: ``.claude/commands/``). Strips YAML
292
+ frontmatter and returns the body — the text that is sent to the agent
293
+ when a user invokes the slash command.
294
+
295
+ Args:
296
+ path: Path to the ``.prompt.md`` or ``.md`` file.
297
+
298
+ Returns:
299
+ Dict with keys ``name`` (str, derived from filename),
300
+ ``body`` (str, the prompt text to pass to :func:`copilot_run`),
301
+ ``description`` (str, from frontmatter or empty), and
302
+ ``metadata`` (dict, full frontmatter).
303
+
304
+ Raises:
305
+ FileNotFoundError: If the file does not exist.
306
+ ValueError: If frontmatter is invalid or the file has no body.
307
+
308
+ Example::
309
+
310
+ from pytest_copilot import load_prompt_file
311
+
312
+ # VS Code prompt file
313
+ prompt = load_prompt_file(".github/prompts/review.prompt.md")
314
+ result = await copilot_run(agent, prompt["body"])
315
+
316
+ # Claude Code command
317
+ prompt = load_prompt_file(".claude/commands/review.md")
318
+ result = await copilot_run(agent, prompt["body"])
319
+ """
320
+ path = Path(path)
321
+ if not path.exists():
322
+ msg = f"Prompt file not found: {path}"
323
+ raise FileNotFoundError(msg)
324
+
325
+ metadata, body = _read_markdown(path)
326
+
327
+ if not body:
328
+ msg = f"Prompt file has no body after frontmatter: {path}"
329
+ raise ValueError(msg)
330
+
331
+ return {
332
+ "name": _prompt_name_from_path(path),
333
+ "body": body,
334
+ "description": metadata.get("description", ""),
335
+ "metadata": metadata,
336
+ }
337
+
338
+
339
+ def load_prompt_files(
340
+ directory: Path | str,
341
+ *,
342
+ include: set[str] | None = None,
343
+ exclude: set[str] | None = None,
344
+ ) -> list[dict[str, Any]]:
345
+ """Load all prompt files from a directory.
346
+
347
+ Loads ``.prompt.md`` files (VS Code convention) and plain ``.md`` files
348
+ (Claude Code convention). Sorted by name.
349
+
350
+ Args:
351
+ directory: Path to directory containing prompt files.
352
+ Typically ``.github/prompts/`` (VS Code) or
353
+ ``.claude/commands/`` (Claude Code).
354
+ include: If set, only load prompts with these names.
355
+ exclude: Prompt names to skip.
356
+
357
+ Returns:
358
+ List of prompt dicts (see :func:`load_prompt_file`), sorted by name.
359
+
360
+ Raises:
361
+ FileNotFoundError: If the directory does not exist.
362
+
363
+ Example::
364
+
365
+ from pytest_copilot import load_prompt_files
366
+
367
+ prompts = load_prompt_files(".github/prompts/")
368
+
369
+ @pytest.mark.parametrize("prompt", prompts, ids=lambda p: p["name"])
370
+ async def test_prompt_files(copilot_run, agent, prompt):
371
+ result = await copilot_run(agent, prompt["body"])
372
+ assert result.success
373
+ """
374
+ directory = Path(directory)
375
+ if not directory.is_dir():
376
+ msg = f"Prompt directory not found: {directory}"
377
+ raise FileNotFoundError(msg)
378
+
379
+ prompts: list[dict[str, Any]] = []
380
+ seen: set[str] = set()
381
+
382
+ # Load .prompt.md files first (VS Code convention)
383
+ for path in sorted(directory.glob("*.prompt.md")):
384
+ name = _prompt_name_from_path(path)
385
+ if include is not None and name not in include:
386
+ continue
387
+ if exclude is not None and name in exclude:
388
+ continue
389
+ prompts.append(load_prompt_file(path))
390
+ seen.add(name)
391
+
392
+ # Also load plain .md files (Claude Code convention) if not already loaded
393
+ for path in sorted(directory.glob("*.md")):
394
+ if path.name.endswith(".prompt.md"):
395
+ continue # already handled above
396
+ name = _prompt_name_from_path(path)
397
+ if include is not None and name not in include:
398
+ continue
399
+ if exclude is not None and name in exclude:
400
+ continue
401
+ prompt = load_prompt_file(path)
402
+ if name in seen:
403
+ raise ValueError(f"{path}: Duplicate prompt name '{name}'")
404
+ prompts.append(prompt)
405
+ seen.add(name)
406
+
407
+ return sorted(prompts, key=lambda p: p["name"])
408
+
409
+
410
+ def _instruction_name_from_path(path: Path) -> str:
411
+ """Derive instruction name from filename.
412
+
413
+ ``coding-standards.instructions.md`` → ``coding-standards``
414
+ ``copilot-instructions.md`` → ``copilot-instructions``
415
+ ``AGENTS.md`` → ``AGENTS``
416
+ """
417
+ name = path.name
418
+ if name.endswith(".instructions.md"):
419
+ return name[: -len(".instructions.md")]
420
+ return path.stem
421
+
422
+
423
+ def load_instruction_file(path: Path | str) -> dict[str, Any]:
424
+ """Load a custom instruction file.
425
+
426
+ Supports VS Code instruction files (``.github/copilot-instructions.md``,
427
+ ``*.instructions.md``), AGENTS.md (Codex), and CLAUDE.md (Claude Code).
428
+ YAML frontmatter is stripped; ``applyTo`` field maps to the ``apply_to`` key.
429
+
430
+ Args:
431
+ path: Path to the instruction file.
432
+
433
+ Returns:
434
+ Dict with keys ``name`` (str), ``content`` (str, the instruction text),
435
+ ``apply_to`` (str, glob pattern or empty), ``description`` (str),
436
+ and ``metadata`` (dict, full frontmatter).
437
+
438
+ Raises:
439
+ FileNotFoundError: If the file does not exist.
440
+ ValueError: If the file has no content after frontmatter stripping.
441
+ """
442
+ path = Path(path)
443
+ if not path.exists():
444
+ msg = f"Instruction file not found: {path}"
445
+ raise FileNotFoundError(msg)
446
+
447
+ metadata, body = _read_markdown(path)
448
+
449
+ if not body:
450
+ msg = f"Instruction file has no content after frontmatter: {path}"
451
+ raise ValueError(msg)
452
+
453
+ return {
454
+ "name": _instruction_name_from_path(path),
455
+ "content": body,
456
+ "apply_to": metadata.get("applyTo", ""),
457
+ "description": metadata.get("description", ""),
458
+ "metadata": metadata,
459
+ "path": str(path),
460
+ }
461
+
462
+
463
+ def load_instruction_files(
464
+ directory: Path | str,
465
+ *,
466
+ include: set[str] | None = None,
467
+ exclude: set[str] | None = None,
468
+ ) -> list[dict[str, Any]]:
469
+ """Load all instruction files from a directory.
470
+
471
+ Loads ``*.instructions.md`` files and well-known files:
472
+ ``copilot-instructions.md``, ``AGENTS.md``, ``CLAUDE.md``.
473
+ Sorted by name.
474
+
475
+ Args:
476
+ directory: Path to directory (or parent) containing instruction files.
477
+ include: If set, only load files with these names.
478
+ exclude: File names to skip.
479
+
480
+ Returns:
481
+ List of instruction file dicts (see :func:`load_instruction_file`), sorted by name.
482
+
483
+ Raises:
484
+ FileNotFoundError: If the directory does not exist.
485
+ """
486
+ directory = Path(directory)
487
+ if not directory.is_dir():
488
+ msg = f"Instruction directory not found: {directory}"
489
+ raise FileNotFoundError(msg)
490
+
491
+ instructions: list[dict[str, Any]] = []
492
+ seen: set[str] = set()
493
+
494
+ # Load *.instructions.md files
495
+ for path in sorted(directory.glob("*.instructions.md")):
496
+ name = _instruction_name_from_path(path)
497
+ if include is not None and name not in include:
498
+ continue
499
+ if exclude is not None and name in exclude:
500
+ continue
501
+ instructions.append(load_instruction_file(path))
502
+ seen.add(name)
503
+
504
+ # Also check for well-known files: copilot-instructions.md, AGENTS.md, CLAUDE.md
505
+ well_known = ["copilot-instructions.md", "AGENTS.md", "CLAUDE.md"]
506
+ for filename in well_known:
507
+ path = directory / filename
508
+ if not path.exists():
509
+ continue
510
+ name = _instruction_name_from_path(path)
511
+ if include is not None and name not in include:
512
+ continue
513
+ if exclude is not None and name in exclude:
514
+ continue
515
+ instruction = load_instruction_file(path)
516
+ if name in seen:
517
+ raise ValueError(f"{path}: Duplicate instruction name '{name}'")
518
+ instructions.append(instruction)
519
+ seen.add(name)
520
+
521
+ return sorted(instructions, key=lambda p: p["name"])