pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,546 @@
1
+ """CopilotTask configuration for testing GitHub Copilot."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, field, replace
6
+ from pathlib import Path
7
+ from typing import Any, Literal, cast
8
+
9
+ from pytest_copilot.copilot.contracts import (
10
+ CopilotCustomAgentConfig,
11
+ CopilotMCPServerConfig,
12
+ CopilotPersona,
13
+ CopilotReasoningEffort,
14
+ CopilotSessionHooks,
15
+ require_custom_agent_name,
16
+ )
17
+ from pytest_copilot.copilot.personas import ClaudeCodePersona, VSCodePersona
18
+ from pytest_copilot.core.evals import load_custom_agent, load_instruction_file
19
+ from pytest_copilot.core.plugin import (
20
+ _discover_skills,
21
+ _validate_skill_reference_names,
22
+ )
23
+
24
+
25
+ def _empty_session_hooks() -> CopilotSessionHooks:
26
+ return cast(CopilotSessionHooks, {})
27
+
28
+
29
+ def _parse_agent_file(path: Path) -> CopilotCustomAgentConfig:
30
+ """Parse a ``.agent.md`` file into a ``CustomAgentConfig`` dict.
31
+
32
+ Handles optional YAML frontmatter for the current SDK-backed custom-agent
33
+ fields and uses the Markdown body as the custom agent's system prompt.
34
+ Parsing and metadata errors retain the definition's file path.
35
+ """
36
+ return cast(CopilotCustomAgentConfig, load_custom_agent(path))
37
+
38
+
39
+ def _parse_agent_files(paths: list[Path]) -> list[CopilotCustomAgentConfig]:
40
+ agents: list[CopilotCustomAgentConfig] = []
41
+ names: set[str] = set()
42
+ for path in paths:
43
+ agent = _parse_agent_file(path)
44
+ name = require_custom_agent_name(agent)
45
+ if name in names:
46
+ raise ValueError(f"{path}: Duplicate custom agent name '{name}'")
47
+ names.add(name)
48
+ agents.append(agent)
49
+ return agents
50
+
51
+
52
+ def _default_persona() -> CopilotPersona:
53
+ """Return the default persona (VSCodePersona)."""
54
+ return VSCodePersona()
55
+
56
+
57
+ @dataclass(slots=True, frozen=True)
58
+ class CopilotTask:
59
+ """Configuration for a GitHub Copilot agent test.
60
+
61
+ Maps to the Copilot SDK's ``SessionConfig``. User-facing field names
62
+ are kept intuitive (e.g. ``instructions``), while
63
+ ``build_session_config()`` maps them to the SDK's actual
64
+ ``system_message`` TypedDict.
65
+
66
+ The SDK's ``SessionConfig`` has no ``maxTurns`` field. The runner
67
+ enforces a hard wall-clock limit via ``timeout_s``; ``max_turns`` is
68
+ advisory for the top-level session (it is *not* hard-enforced mid-run)
69
+ and is used when configuring subagent turn caps.
70
+
71
+ Example:
72
+ # Minimal
73
+ CopilotTask()
74
+
75
+ # With instructions and model
76
+ CopilotTask(
77
+ name="security-reviewer",
78
+ model="claude-sonnet-4",
79
+ instructions="Review code for security vulnerabilities.",
80
+ )
81
+
82
+ # With custom tools and references
83
+ CopilotTask(
84
+ name="file-creator",
85
+ instructions="Create files as requested.",
86
+ working_directory="/tmp/workspace",
87
+ allowed_tools=["create_file", "read_file"],
88
+ )
89
+ """
90
+
91
+ # Identity
92
+ name: str = "copilot"
93
+
94
+ # Model selection (None = Copilot's default)
95
+ model: str | None = None
96
+ reasoning_effort: CopilotReasoningEffort | None = None
97
+ client_mode: Literal["copilot-cli", "empty"] = "copilot-cli"
98
+ image_detail: Literal["auto", "low", "high"] | None = None
99
+ audit_requests: bool = False
100
+
101
+ # System message content — maps to SDK's system_message.content
102
+ # In the Copilot SDK, this is NOT a "system prompt" — it's instructions
103
+ # appended to (or replacing) the CLI's built-in system message.
104
+ instructions: str | None = None
105
+ system_message_mode: Literal["append", "replace"] = "append"
106
+
107
+ # Context
108
+ working_directory: str | None = None
109
+
110
+ # Tool control
111
+ allowed_tools: list[str] | None = None # Allowlist (None = all)
112
+ excluded_tools: list[str] | None = None # Blocklist
113
+
114
+ # Limits — the runner enforces timeout_s (hard wall-clock limit).
115
+ # max_turns is advisory for the top-level session (not hard-enforced
116
+ # mid-run) and is used to cap subagent turns.
117
+ max_turns: int = 25
118
+ timeout_s: float = 300.0
119
+ max_tool_calls: int | None = None
120
+
121
+ # Permissions — auto-approve by default for deterministic testing
122
+ auto_confirm: bool = True
123
+
124
+ # MCP servers to attach to the session
125
+ mcp_servers: dict[str, CopilotMCPServerConfig] = field(default_factory=dict)
126
+
127
+ # Custom agents (SDK CustomAgentConfig: name, prompt, description,
128
+ # display_name, tools, mcp_servers, infer)
129
+ custom_agents: list[CopilotCustomAgentConfig] = field(default_factory=list)
130
+
131
+ # Explicit skills are enabled and checked before model execution.
132
+ skill_directories: list[str] = field(default_factory=list)
133
+ disabled_skills: list[str] = field(default_factory=list)
134
+
135
+ # SDK passthrough for unmapped fields
136
+ extra_config: dict[str, Any] = field(default_factory=dict)
137
+
138
+ # SDK passthrough: activate a specific custom agent at session start.
139
+ # Maps to the SDK's ``agent`` parameter on ``create_session()``.
140
+ active_agent: str | None = None
141
+
142
+ # SDK passthrough: lifecycle hooks (SessionHooks).
143
+ # Maps to the SDK's ``hooks`` parameter on ``create_session()``.
144
+ hooks: CopilotSessionHooks = field(default_factory=_empty_session_hooks)
145
+
146
+ # IDE persona — controls which polyfill tools are injected to simulate
147
+ # the target runtime environment (VS Code, Claude Code, Copilot CLI, etc.)
148
+ # VSCodePersona is the default: it polyfills runSubagent when custom_agents
149
+ # are present, matching VS Code's native behaviour.
150
+ persona: CopilotPersona = field(default_factory=_default_persona)
151
+
152
+ def __post_init__(self) -> None:
153
+ if self.client_mode not in ("copilot-cli", "empty"):
154
+ raise ValueError("client_mode must be 'copilot-cli' or 'empty'")
155
+ if self.image_detail not in (None, "auto", "low", "high"):
156
+ raise ValueError("image_detail must be 'auto', 'low', 'high', or None")
157
+ if self.max_tool_calls is not None and (
158
+ type(self.max_tool_calls) is not int or self.max_tool_calls < 0
159
+ ):
160
+ raise ValueError("max_tool_calls must be a nonnegative integer or None")
161
+
162
+ def build_session_config(self) -> dict[str, Any]:
163
+ """Build a SessionConfig dict for the Copilot SDK.
164
+
165
+ Returns a dict compatible with ``CopilotClient.create_session()``.
166
+ Only includes non-None/non-default fields to avoid overriding
167
+ SDK defaults.
168
+
169
+ SDK field mapping (Python snake_case TypedDict keys):
170
+ instructions → system_message: {mode, content}
171
+ allowed_tools → available_tools
172
+ excluded_tools → excluded_tools
173
+ reasoning_effort → reasoning_effort
174
+ working_directory → working_directory
175
+ mcp_servers → mcp_servers
176
+ custom_agents → custom_agents
177
+ skill_directories → skill_directories
178
+ disabled_skills → disabled_skills
179
+
180
+ Note: ``max_turns`` is NOT part of ``SessionConfig``. The runner
181
+ enforces ``timeout_s`` as the hard limit; ``max_turns`` is advisory
182
+ (not hard-enforced mid-run) and used only to cap subagent turns.
183
+ """
184
+ config: dict[str, Any] = {"enable_config_discovery": False}
185
+
186
+ if self.model is not None:
187
+ config["model"] = self.model
188
+
189
+ if self.reasoning_effort is not None:
190
+ config["reasoning_effort"] = self.reasoning_effort
191
+
192
+ # Map instructions + system_message_mode → SDK's system_message
193
+ if self.instructions:
194
+ config["system_message"] = {
195
+ "mode": self.system_message_mode,
196
+ "content": self.instructions,
197
+ }
198
+
199
+ if self.working_directory is not None:
200
+ config["working_directory"] = self.working_directory
201
+
202
+ if self.allowed_tools is not None:
203
+ config["available_tools"] = self.allowed_tools
204
+
205
+ if self.excluded_tools is not None:
206
+ config["excluded_tools"] = self.excluded_tools
207
+
208
+ if self.mcp_servers:
209
+ config["mcp_servers"] = self.mcp_servers
210
+
211
+ if self.custom_agents:
212
+ config["custom_agents"] = self.custom_agents
213
+
214
+ if self.skill_directories:
215
+ config["skill_directories"] = self.skill_directories
216
+
217
+ if self.disabled_skills:
218
+ config["disabled_skills"] = self.disabled_skills
219
+
220
+ if self.active_agent is not None:
221
+ config["agent"] = self.active_agent
222
+
223
+ if self.hooks:
224
+ config["hooks"] = self.hooks
225
+
226
+ # Apply extra_config passthrough
227
+ if (
228
+ self.skill_directories
229
+ and "skill_directories" in self.extra_config
230
+ and self.extra_config["skill_directories"] != self.skill_directories
231
+ ):
232
+ raise ValueError("extra_config must not override explicitly supplied skill_directories")
233
+ config.update(self.extra_config)
234
+
235
+ if config.get("skill_directories"):
236
+ if config.get("enable_skills") is False:
237
+ raise ValueError(
238
+ "Explicit skill_directories conflict with enable_skills=False; "
239
+ "remove the directories for a no-skill baseline, or enable skills"
240
+ )
241
+ if config.get("enable_skills") is not None and config["enable_skills"] is not True:
242
+ raise ValueError("enable_skills must be a boolean or None")
243
+ config["enable_skills"] = True
244
+
245
+ if self.client_mode == "empty":
246
+ isolated = {
247
+ "enable_config_discovery": False,
248
+ "enable_file_hooks": False,
249
+ "enable_on_demand_instruction_discovery": False,
250
+ "skip_custom_instructions": True,
251
+ "enable_managed_settings": False,
252
+ }
253
+ for key, value in isolated.items():
254
+ if key in config and config[key] != value:
255
+ raise ValueError(f"client_mode='empty' requires {key}={value}")
256
+ config.update(isolated)
257
+ if config.get("available_tools") is None:
258
+ config["available_tools"] = []
259
+
260
+ return config
261
+
262
+ def create_subagent(
263
+ self,
264
+ *,
265
+ name: str,
266
+ model: str | None,
267
+ reasoning_effort: CopilotReasoningEffort | None,
268
+ instructions: str,
269
+ timeout_s: float,
270
+ max_turns: int,
271
+ allowed_tools: list[str] | None,
272
+ mcp_servers: dict[str, CopilotMCPServerConfig],
273
+ ) -> CopilotTask:
274
+ """Create the exact nested eval configuration for a custom agent."""
275
+ return replace(
276
+ self,
277
+ name=name,
278
+ model=model,
279
+ reasoning_effort=reasoning_effort,
280
+ instructions=instructions,
281
+ timeout_s=timeout_s,
282
+ max_turns=max_turns,
283
+ allowed_tools=allowed_tools,
284
+ excluded_tools=None,
285
+ mcp_servers=mcp_servers,
286
+ active_agent=None,
287
+ custom_agents=[
288
+ agent for agent in self.custom_agents if require_custom_agent_name(agent) != name
289
+ ],
290
+ )
291
+
292
+ @classmethod
293
+ def from_copilot_config(
294
+ cls,
295
+ path: str | Path = ".",
296
+ **overrides: Any,
297
+ ) -> "CopilotTask":
298
+ """Load a ``CopilotTask`` from a directory containing Copilot config files.
299
+
300
+ Looks for the following files under ``path``:
301
+
302
+ * ``.github/copilot-instructions.md`` → ``instructions``
303
+ * ``.github/agents/*.agent.md`` → ``custom_agents``
304
+
305
+ Point ``path`` at any directory — your production project, a dedicated
306
+ test fixture project, or a shared team config repo. Any keyword
307
+ argument overrides the loaded value.
308
+
309
+ Args:
310
+ path: Root directory to load config from. Defaults to the current
311
+ working directory.
312
+ **overrides: Override any ``CopilotTask`` field after loading,
313
+ e.g. ``model="claude-opus-4.5"``.
314
+
315
+ Returns:
316
+ A ``CopilotTask`` initialised from the discovered config files.
317
+
318
+ Example::
319
+
320
+ # Load from the current project (production config as baseline)
321
+ baseline = CopilotTask.from_copilot_config()
322
+
323
+ # A/B test: same config, one instruction changed
324
+ treatment = CopilotTask.from_copilot_config(
325
+ instructions="Always add type hints.",
326
+ )
327
+
328
+ # Load from a dedicated test-fixture project
329
+ agent = CopilotTask.from_copilot_config("tests/fixtures/strict-agent")
330
+
331
+ # Load from a shared team agent library
332
+ agent = CopilotTask.from_copilot_config("/shared/team/copilot-config")
333
+ """
334
+ root = Path(path).resolve()
335
+ github_dir = root / ".github"
336
+ if not root.is_dir():
337
+ raise FileNotFoundError(f"Copilot config root is not a directory: {root}")
338
+ if github_dir.exists() and not github_dir.is_dir():
339
+ raise ValueError(f"{github_dir}: Copilot config path must be a directory")
340
+
341
+ # Load repository-wide instructions
342
+ instructions: str | None = None
343
+ instructions_file = github_dir / "copilot-instructions.md"
344
+ if instructions_file.exists():
345
+ instructions = load_instruction_file(instructions_file)["content"]
346
+
347
+ # Load custom agents — recursive so subagents/ subdirectories are included
348
+ agents: list[CopilotCustomAgentConfig] = []
349
+ agents_dir = github_dir / "agents"
350
+ if agents_dir.exists():
351
+ if not agents_dir.is_dir():
352
+ raise ValueError(f"{agents_dir}: custom agents path must be a directory")
353
+ agents = _parse_agent_files(sorted(agents_dir.rglob("*.agent.md")))
354
+
355
+ config: dict[str, Any] = {
356
+ "instructions": instructions,
357
+ "custom_agents": agents,
358
+ }
359
+ config.update(overrides)
360
+ return cls(**config)
361
+
362
+ @classmethod
363
+ def from_plugin(
364
+ cls,
365
+ path: str | Path,
366
+ *,
367
+ model: str = "",
368
+ persona: CopilotPersona | None = None,
369
+ instructions: str = "",
370
+ working_directory: str = "",
371
+ name: str = "",
372
+ **overrides: Any,
373
+ ) -> "CopilotTask":
374
+ """Create a CopilotTask from a plugin directory.
375
+
376
+ Loads the plugin's agents, skills, MCP servers, and instructions,
377
+ then constructs a CopilotTask with all components wired together.
378
+
379
+ Uses :func:`~pytest_copilot.core.plugin.load_plugin` to
380
+ discover the plugin structure. The persona is auto-detected from the
381
+ plugin path (``ClaudeCodePersona`` for ``.claude/`` paths,
382
+ ``VSCodePersona`` otherwise) unless explicitly overridden.
383
+
384
+ Args:
385
+ path: Path to the plugin directory (may contain ``plugin.json``,
386
+ or be a ``.github/`` / ``.claude/`` project config directory).
387
+ model: Model override for the eval.
388
+ persona: IDE persona override. Auto-detected when ``None``.
389
+ instructions: Additional instructions to **append** to the
390
+ plugin's discovered instructions.
391
+ working_directory: Override the working directory.
392
+ name: Override the eval name (defaults to plugin metadata name).
393
+ **overrides: Override any additional ``CopilotTask`` fields such as
394
+ ``max_turns`` or ``timeout_s``.
395
+
396
+ Returns:
397
+ A ``CopilotTask`` initialised from the plugin.
398
+
399
+ Example::
400
+
401
+ agent = CopilotTask.from_plugin("my-plugin/")
402
+
403
+ agent = CopilotTask.from_plugin(
404
+ ".claude/",
405
+ model="claude-sonnet-4",
406
+ instructions="Focus on security reviews.",
407
+ )
408
+ """
409
+ from pytest_copilot.core.plugin import load_plugin # noqa: PLC0415
410
+
411
+ plugin = load_plugin(path)
412
+
413
+ # Build combined instructions: plugin base + caller override
414
+ combined_instructions = plugin.instructions
415
+ if instructions:
416
+ if combined_instructions:
417
+ combined_instructions = combined_instructions + "\n\n" + instructions
418
+ else:
419
+ combined_instructions = instructions
420
+
421
+ # Map plugin.skills to skill directory paths
422
+ skill_dirs = [str(s.path) for s in plugin.skills]
423
+
424
+ # Auto-detect persona from plugin path
425
+ if persona is None:
426
+ resolved = Path(path).resolve()
427
+ if resolved.name == ".claude" or (resolved / ".claude").is_dir():
428
+ persona = ClaudeCodePersona()
429
+ else:
430
+ persona = _default_persona()
431
+
432
+ config: dict[str, Any] = {
433
+ "name": name or plugin.metadata.name,
434
+ "model": model or None,
435
+ "instructions": combined_instructions or None,
436
+ "custom_agents": plugin.agents,
437
+ "skill_directories": skill_dirs,
438
+ "mcp_servers": plugin.mcp_servers,
439
+ "working_directory": working_directory or None,
440
+ "persona": persona,
441
+ }
442
+ config.update(overrides)
443
+ return cls(
444
+ **config,
445
+ )
446
+
447
+ @classmethod
448
+ def from_claude_config(
449
+ cls,
450
+ path: str | Path = ".",
451
+ *,
452
+ model: str = "",
453
+ persona: CopilotPersona | None = None,
454
+ instructions: str = "",
455
+ working_directory: str = "",
456
+ name: str = "claude-code-eval",
457
+ **overrides: Any,
458
+ ) -> "CopilotTask":
459
+ """Create a CopilotTask from a Claude Code project directory.
460
+
461
+ Scans for:
462
+
463
+ * ``CLAUDE.md`` (project root) and ``.claude/CLAUDE.md`` → instructions
464
+ * ``.claude/agents/*.md`` → custom agents
465
+ * ``.claude/skills/`` → skill directories (subdirs with ``SKILL.md``)
466
+ * ``.mcp.json`` → MCP server configs
467
+
468
+ Args:
469
+ path: Root of the Claude Code project (default: current directory).
470
+ model: Model override for the eval.
471
+ persona: IDE persona override. Defaults to ``ClaudeCodePersona``.
472
+ instructions: Additional instructions to **append** to the
473
+ discovered ``CLAUDE.md`` content.
474
+ working_directory: Override the working directory.
475
+ name: Override the eval name.
476
+ **overrides: Override any additional ``CopilotTask`` fields such as
477
+ ``max_turns`` or ``timeout_s``.
478
+
479
+ Returns:
480
+ A ``CopilotTask`` initialised from the discovered config files.
481
+
482
+ Example::
483
+
484
+ # Load from the current project
485
+ agent = CopilotTask.from_claude_config()
486
+
487
+ # Load from a specific directory with model override
488
+ agent = CopilotTask.from_claude_config(
489
+ "tests/fixtures/claude-project",
490
+ model="claude-sonnet-4",
491
+ )
492
+ """
493
+ from pytest_copilot.copilot.config import load_mcp_config # noqa: PLC0415
494
+
495
+ root = Path(path).resolve()
496
+ claude_dir = root / ".claude"
497
+ if not root.is_dir():
498
+ raise FileNotFoundError(f"Claude config root is not a directory: {root}")
499
+ if claude_dir.exists() and not claude_dir.is_dir():
500
+ raise ValueError(f"{claude_dir}: Claude config path must be a directory")
501
+
502
+ # 1. Concatenate instructions from CLAUDE.md files
503
+ instruction_parts: list[str] = []
504
+ for md_path in [root / "CLAUDE.md", claude_dir / "CLAUDE.md"]:
505
+ if md_path.exists():
506
+ instruction_parts.append(load_instruction_file(md_path)["content"])
507
+ if instructions:
508
+ instruction_parts.append(instructions)
509
+ combined_instructions = "\n\n".join(instruction_parts) or None
510
+
511
+ # 2. Load custom agents from .claude/agents/
512
+ agents: list[CopilotCustomAgentConfig] = []
513
+ agents_dir = claude_dir / "agents"
514
+ if agents_dir.exists():
515
+ if not agents_dir.is_dir():
516
+ raise ValueError(f"{agents_dir}: custom agents path must be a directory")
517
+ # Claude Code uses plain .md files for agents
518
+ agents = _parse_agent_files(sorted(agents_dir.glob("*.md")))
519
+
520
+ # 3. Discover skill directories from .claude/skills/
521
+ skills = _discover_skills(claude_dir)
522
+ _validate_skill_reference_names(skills)
523
+ skill_dirs = [str(skill.path) for skill in skills]
524
+
525
+ # 4. Parse .mcp.json if it exists
526
+ mcp_servers: dict[str, dict[str, Any]] = {}
527
+ mcp_config_path = root / ".mcp.json"
528
+ if mcp_config_path.exists():
529
+ mcp_servers = load_mcp_config(mcp_config_path)
530
+
531
+ # 5. Default persona to ClaudeCodePersona
532
+ if persona is None:
533
+ persona = ClaudeCodePersona()
534
+
535
+ config: dict[str, Any] = {
536
+ "name": name,
537
+ "model": model or None,
538
+ "instructions": combined_instructions,
539
+ "custom_agents": agents,
540
+ "skill_directories": skill_dirs,
541
+ "mcp_servers": mcp_servers,
542
+ "working_directory": working_directory or None,
543
+ "persona": persona,
544
+ }
545
+ config.update(overrides)
546
+ return cls(**config)
@@ -0,0 +1,77 @@
1
+ """Core module - result types and skill management."""
2
+
3
+ from pytest_copilot.core.errors import AITestError, EngineTimeoutError, ServerStartError
4
+ from pytest_copilot.core.evals import (
5
+ load_custom_agent,
6
+ load_custom_agents,
7
+ load_instruction_file,
8
+ load_instruction_files,
9
+ load_prompt_file,
10
+ load_prompt_files,
11
+ )
12
+ from pytest_copilot.core.plugin import HookDefinition, Plugin, PluginMetadata, load_plugin
13
+ from pytest_copilot.core.prompt import (
14
+ Prompt,
15
+ load_prompt,
16
+ load_prompts,
17
+ load_system_prompts,
18
+ )
19
+ from pytest_copilot.core.result import (
20
+ DiscoveredSkill,
21
+ EvalResult,
22
+ ImageContent,
23
+ MCPPrompt,
24
+ MCPPromptArgument,
25
+ SkillDiscovery,
26
+ SkillInfo,
27
+ SubagentInvocation,
28
+ ToolCall,
29
+ ToolInfo,
30
+ Turn,
31
+ )
32
+ from pytest_copilot.core.skill import Skill, SkillError, SkillMetadata, load_skill
33
+ from pytest_copilot.core.skill_evals import (
34
+ SkillEvalCase,
35
+ has_skill_evals,
36
+ load_skill_evals,
37
+ )
38
+ from pytest_copilot.core.skill_grading import export_grading
39
+
40
+ __all__ = [
41
+ "AITestError",
42
+ "EvalResult",
43
+ "DiscoveredSkill",
44
+ "EngineTimeoutError",
45
+ "HookDefinition",
46
+ "ImageContent",
47
+ "MCPPrompt",
48
+ "MCPPromptArgument",
49
+ "Plugin",
50
+ "PluginMetadata",
51
+ "Prompt",
52
+ "ServerStartError",
53
+ "Skill",
54
+ "SkillDiscovery",
55
+ "SkillError",
56
+ "SkillInfo",
57
+ "SkillMetadata",
58
+ "SubagentInvocation",
59
+ "ToolCall",
60
+ "ToolInfo",
61
+ "Turn",
62
+ "load_custom_agent",
63
+ "load_custom_agents",
64
+ "load_instruction_file",
65
+ "load_instruction_files",
66
+ "load_plugin",
67
+ "load_prompt_file",
68
+ "load_prompt_files",
69
+ "load_prompt",
70
+ "load_prompts",
71
+ "load_skill",
72
+ "load_system_prompts",
73
+ "SkillEvalCase",
74
+ "has_skill_evals",
75
+ "load_skill_evals",
76
+ "export_grading",
77
+ ]
@@ -0,0 +1,44 @@
1
+ """Custom exceptions for pytest-copilot."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class AITestError(Exception):
7
+ """Base exception for pytest-copilot errors."""
8
+
9
+
10
+ class ServerStartError(AITestError):
11
+ """Error starting an MCP server."""
12
+
13
+ def __init__(self, server_type: str, command: list[str], message: str) -> None:
14
+ self.server_type = server_type
15
+ self.command = command
16
+ super().__init__(f"Failed to start {server_type} server ({' '.join(command)}): {message}")
17
+
18
+
19
+ class EngineTimeoutError(AITestError):
20
+ """Eval engine timed out."""
21
+
22
+ def __init__(self, timeout_ms: int, turns_completed: int) -> None:
23
+ self.timeout_ms = timeout_ms
24
+ self.turns_completed = turns_completed
25
+ super().__init__(f"Eval timed out after {timeout_ms}ms ({turns_completed} turns completed)")
26
+
27
+
28
+ class ToolCallError(AITestError):
29
+ """Error calling a tool."""
30
+
31
+ def __init__(self, tool_name: str, message: str) -> None:
32
+ self.tool_name = tool_name
33
+ super().__init__(f"Tool '{tool_name}' failed: {message}")
34
+
35
+
36
+ class RateLimitError(AITestError):
37
+ """Rate limit exceeded."""
38
+
39
+ def __init__(self, retry_after: float | None = None) -> None:
40
+ self.retry_after = retry_after
41
+ msg = "Rate limit exceeded"
42
+ if retry_after:
43
+ msg += f" (retry after {retry_after}s)"
44
+ super().__init__(msg)