pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
"""CopilotTask configuration for testing GitHub Copilot."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field, replace
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Literal, cast
|
|
8
|
+
|
|
9
|
+
from pytest_copilot.copilot.contracts import (
|
|
10
|
+
CopilotCustomAgentConfig,
|
|
11
|
+
CopilotMCPServerConfig,
|
|
12
|
+
CopilotPersona,
|
|
13
|
+
CopilotReasoningEffort,
|
|
14
|
+
CopilotSessionHooks,
|
|
15
|
+
require_custom_agent_name,
|
|
16
|
+
)
|
|
17
|
+
from pytest_copilot.copilot.personas import ClaudeCodePersona, VSCodePersona
|
|
18
|
+
from pytest_copilot.core.evals import load_custom_agent, load_instruction_file
|
|
19
|
+
from pytest_copilot.core.plugin import (
|
|
20
|
+
_discover_skills,
|
|
21
|
+
_validate_skill_reference_names,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _empty_session_hooks() -> CopilotSessionHooks:
|
|
26
|
+
return cast(CopilotSessionHooks, {})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _parse_agent_file(path: Path) -> CopilotCustomAgentConfig:
|
|
30
|
+
"""Parse a ``.agent.md`` file into a ``CustomAgentConfig`` dict.
|
|
31
|
+
|
|
32
|
+
Handles optional YAML frontmatter for the current SDK-backed custom-agent
|
|
33
|
+
fields and uses the Markdown body as the custom agent's system prompt.
|
|
34
|
+
Parsing and metadata errors retain the definition's file path.
|
|
35
|
+
"""
|
|
36
|
+
return cast(CopilotCustomAgentConfig, load_custom_agent(path))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parse_agent_files(paths: list[Path]) -> list[CopilotCustomAgentConfig]:
|
|
40
|
+
agents: list[CopilotCustomAgentConfig] = []
|
|
41
|
+
names: set[str] = set()
|
|
42
|
+
for path in paths:
|
|
43
|
+
agent = _parse_agent_file(path)
|
|
44
|
+
name = require_custom_agent_name(agent)
|
|
45
|
+
if name in names:
|
|
46
|
+
raise ValueError(f"{path}: Duplicate custom agent name '{name}'")
|
|
47
|
+
names.add(name)
|
|
48
|
+
agents.append(agent)
|
|
49
|
+
return agents
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _default_persona() -> CopilotPersona:
|
|
53
|
+
"""Return the default persona (VSCodePersona)."""
|
|
54
|
+
return VSCodePersona()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(slots=True, frozen=True)
|
|
58
|
+
class CopilotTask:
|
|
59
|
+
"""Configuration for a GitHub Copilot agent test.
|
|
60
|
+
|
|
61
|
+
Maps to the Copilot SDK's ``SessionConfig``. User-facing field names
|
|
62
|
+
are kept intuitive (e.g. ``instructions``), while
|
|
63
|
+
``build_session_config()`` maps them to the SDK's actual
|
|
64
|
+
``system_message`` TypedDict.
|
|
65
|
+
|
|
66
|
+
The SDK's ``SessionConfig`` has no ``maxTurns`` field. The runner
|
|
67
|
+
enforces a hard wall-clock limit via ``timeout_s``; ``max_turns`` is
|
|
68
|
+
advisory for the top-level session (it is *not* hard-enforced mid-run)
|
|
69
|
+
and is used when configuring subagent turn caps.
|
|
70
|
+
|
|
71
|
+
Example:
|
|
72
|
+
# Minimal
|
|
73
|
+
CopilotTask()
|
|
74
|
+
|
|
75
|
+
# With instructions and model
|
|
76
|
+
CopilotTask(
|
|
77
|
+
name="security-reviewer",
|
|
78
|
+
model="claude-sonnet-4",
|
|
79
|
+
instructions="Review code for security vulnerabilities.",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# With custom tools and references
|
|
83
|
+
CopilotTask(
|
|
84
|
+
name="file-creator",
|
|
85
|
+
instructions="Create files as requested.",
|
|
86
|
+
working_directory="/tmp/workspace",
|
|
87
|
+
allowed_tools=["create_file", "read_file"],
|
|
88
|
+
)
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
# Identity
|
|
92
|
+
name: str = "copilot"
|
|
93
|
+
|
|
94
|
+
# Model selection (None = Copilot's default)
|
|
95
|
+
model: str | None = None
|
|
96
|
+
reasoning_effort: CopilotReasoningEffort | None = None
|
|
97
|
+
client_mode: Literal["copilot-cli", "empty"] = "copilot-cli"
|
|
98
|
+
image_detail: Literal["auto", "low", "high"] | None = None
|
|
99
|
+
audit_requests: bool = False
|
|
100
|
+
|
|
101
|
+
# System message content — maps to SDK's system_message.content
|
|
102
|
+
# In the Copilot SDK, this is NOT a "system prompt" — it's instructions
|
|
103
|
+
# appended to (or replacing) the CLI's built-in system message.
|
|
104
|
+
instructions: str | None = None
|
|
105
|
+
system_message_mode: Literal["append", "replace"] = "append"
|
|
106
|
+
|
|
107
|
+
# Context
|
|
108
|
+
working_directory: str | None = None
|
|
109
|
+
|
|
110
|
+
# Tool control
|
|
111
|
+
allowed_tools: list[str] | None = None # Allowlist (None = all)
|
|
112
|
+
excluded_tools: list[str] | None = None # Blocklist
|
|
113
|
+
|
|
114
|
+
# Limits — the runner enforces timeout_s (hard wall-clock limit).
|
|
115
|
+
# max_turns is advisory for the top-level session (not hard-enforced
|
|
116
|
+
# mid-run) and is used to cap subagent turns.
|
|
117
|
+
max_turns: int = 25
|
|
118
|
+
timeout_s: float = 300.0
|
|
119
|
+
max_tool_calls: int | None = None
|
|
120
|
+
|
|
121
|
+
# Permissions — auto-approve by default for deterministic testing
|
|
122
|
+
auto_confirm: bool = True
|
|
123
|
+
|
|
124
|
+
# MCP servers to attach to the session
|
|
125
|
+
mcp_servers: dict[str, CopilotMCPServerConfig] = field(default_factory=dict)
|
|
126
|
+
|
|
127
|
+
# Custom agents (SDK CustomAgentConfig: name, prompt, description,
|
|
128
|
+
# display_name, tools, mcp_servers, infer)
|
|
129
|
+
custom_agents: list[CopilotCustomAgentConfig] = field(default_factory=list)
|
|
130
|
+
|
|
131
|
+
# Explicit skills are enabled and checked before model execution.
|
|
132
|
+
skill_directories: list[str] = field(default_factory=list)
|
|
133
|
+
disabled_skills: list[str] = field(default_factory=list)
|
|
134
|
+
|
|
135
|
+
# SDK passthrough for unmapped fields
|
|
136
|
+
extra_config: dict[str, Any] = field(default_factory=dict)
|
|
137
|
+
|
|
138
|
+
# SDK passthrough: activate a specific custom agent at session start.
|
|
139
|
+
# Maps to the SDK's ``agent`` parameter on ``create_session()``.
|
|
140
|
+
active_agent: str | None = None
|
|
141
|
+
|
|
142
|
+
# SDK passthrough: lifecycle hooks (SessionHooks).
|
|
143
|
+
# Maps to the SDK's ``hooks`` parameter on ``create_session()``.
|
|
144
|
+
hooks: CopilotSessionHooks = field(default_factory=_empty_session_hooks)
|
|
145
|
+
|
|
146
|
+
# IDE persona — controls which polyfill tools are injected to simulate
|
|
147
|
+
# the target runtime environment (VS Code, Claude Code, Copilot CLI, etc.)
|
|
148
|
+
# VSCodePersona is the default: it polyfills runSubagent when custom_agents
|
|
149
|
+
# are present, matching VS Code's native behaviour.
|
|
150
|
+
persona: CopilotPersona = field(default_factory=_default_persona)
|
|
151
|
+
|
|
152
|
+
def __post_init__(self) -> None:
|
|
153
|
+
if self.client_mode not in ("copilot-cli", "empty"):
|
|
154
|
+
raise ValueError("client_mode must be 'copilot-cli' or 'empty'")
|
|
155
|
+
if self.image_detail not in (None, "auto", "low", "high"):
|
|
156
|
+
raise ValueError("image_detail must be 'auto', 'low', 'high', or None")
|
|
157
|
+
if self.max_tool_calls is not None and (
|
|
158
|
+
type(self.max_tool_calls) is not int or self.max_tool_calls < 0
|
|
159
|
+
):
|
|
160
|
+
raise ValueError("max_tool_calls must be a nonnegative integer or None")
|
|
161
|
+
|
|
162
|
+
def build_session_config(self) -> dict[str, Any]:
|
|
163
|
+
"""Build a SessionConfig dict for the Copilot SDK.
|
|
164
|
+
|
|
165
|
+
Returns a dict compatible with ``CopilotClient.create_session()``.
|
|
166
|
+
Only includes non-None/non-default fields to avoid overriding
|
|
167
|
+
SDK defaults.
|
|
168
|
+
|
|
169
|
+
SDK field mapping (Python snake_case TypedDict keys):
|
|
170
|
+
instructions → system_message: {mode, content}
|
|
171
|
+
allowed_tools → available_tools
|
|
172
|
+
excluded_tools → excluded_tools
|
|
173
|
+
reasoning_effort → reasoning_effort
|
|
174
|
+
working_directory → working_directory
|
|
175
|
+
mcp_servers → mcp_servers
|
|
176
|
+
custom_agents → custom_agents
|
|
177
|
+
skill_directories → skill_directories
|
|
178
|
+
disabled_skills → disabled_skills
|
|
179
|
+
|
|
180
|
+
Note: ``max_turns`` is NOT part of ``SessionConfig``. The runner
|
|
181
|
+
enforces ``timeout_s`` as the hard limit; ``max_turns`` is advisory
|
|
182
|
+
(not hard-enforced mid-run) and used only to cap subagent turns.
|
|
183
|
+
"""
|
|
184
|
+
config: dict[str, Any] = {"enable_config_discovery": False}
|
|
185
|
+
|
|
186
|
+
if self.model is not None:
|
|
187
|
+
config["model"] = self.model
|
|
188
|
+
|
|
189
|
+
if self.reasoning_effort is not None:
|
|
190
|
+
config["reasoning_effort"] = self.reasoning_effort
|
|
191
|
+
|
|
192
|
+
# Map instructions + system_message_mode → SDK's system_message
|
|
193
|
+
if self.instructions:
|
|
194
|
+
config["system_message"] = {
|
|
195
|
+
"mode": self.system_message_mode,
|
|
196
|
+
"content": self.instructions,
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if self.working_directory is not None:
|
|
200
|
+
config["working_directory"] = self.working_directory
|
|
201
|
+
|
|
202
|
+
if self.allowed_tools is not None:
|
|
203
|
+
config["available_tools"] = self.allowed_tools
|
|
204
|
+
|
|
205
|
+
if self.excluded_tools is not None:
|
|
206
|
+
config["excluded_tools"] = self.excluded_tools
|
|
207
|
+
|
|
208
|
+
if self.mcp_servers:
|
|
209
|
+
config["mcp_servers"] = self.mcp_servers
|
|
210
|
+
|
|
211
|
+
if self.custom_agents:
|
|
212
|
+
config["custom_agents"] = self.custom_agents
|
|
213
|
+
|
|
214
|
+
if self.skill_directories:
|
|
215
|
+
config["skill_directories"] = self.skill_directories
|
|
216
|
+
|
|
217
|
+
if self.disabled_skills:
|
|
218
|
+
config["disabled_skills"] = self.disabled_skills
|
|
219
|
+
|
|
220
|
+
if self.active_agent is not None:
|
|
221
|
+
config["agent"] = self.active_agent
|
|
222
|
+
|
|
223
|
+
if self.hooks:
|
|
224
|
+
config["hooks"] = self.hooks
|
|
225
|
+
|
|
226
|
+
# Apply extra_config passthrough
|
|
227
|
+
if (
|
|
228
|
+
self.skill_directories
|
|
229
|
+
and "skill_directories" in self.extra_config
|
|
230
|
+
and self.extra_config["skill_directories"] != self.skill_directories
|
|
231
|
+
):
|
|
232
|
+
raise ValueError("extra_config must not override explicitly supplied skill_directories")
|
|
233
|
+
config.update(self.extra_config)
|
|
234
|
+
|
|
235
|
+
if config.get("skill_directories"):
|
|
236
|
+
if config.get("enable_skills") is False:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
"Explicit skill_directories conflict with enable_skills=False; "
|
|
239
|
+
"remove the directories for a no-skill baseline, or enable skills"
|
|
240
|
+
)
|
|
241
|
+
if config.get("enable_skills") is not None and config["enable_skills"] is not True:
|
|
242
|
+
raise ValueError("enable_skills must be a boolean or None")
|
|
243
|
+
config["enable_skills"] = True
|
|
244
|
+
|
|
245
|
+
if self.client_mode == "empty":
|
|
246
|
+
isolated = {
|
|
247
|
+
"enable_config_discovery": False,
|
|
248
|
+
"enable_file_hooks": False,
|
|
249
|
+
"enable_on_demand_instruction_discovery": False,
|
|
250
|
+
"skip_custom_instructions": True,
|
|
251
|
+
"enable_managed_settings": False,
|
|
252
|
+
}
|
|
253
|
+
for key, value in isolated.items():
|
|
254
|
+
if key in config and config[key] != value:
|
|
255
|
+
raise ValueError(f"client_mode='empty' requires {key}={value}")
|
|
256
|
+
config.update(isolated)
|
|
257
|
+
if config.get("available_tools") is None:
|
|
258
|
+
config["available_tools"] = []
|
|
259
|
+
|
|
260
|
+
return config
|
|
261
|
+
|
|
262
|
+
def create_subagent(
|
|
263
|
+
self,
|
|
264
|
+
*,
|
|
265
|
+
name: str,
|
|
266
|
+
model: str | None,
|
|
267
|
+
reasoning_effort: CopilotReasoningEffort | None,
|
|
268
|
+
instructions: str,
|
|
269
|
+
timeout_s: float,
|
|
270
|
+
max_turns: int,
|
|
271
|
+
allowed_tools: list[str] | None,
|
|
272
|
+
mcp_servers: dict[str, CopilotMCPServerConfig],
|
|
273
|
+
) -> CopilotTask:
|
|
274
|
+
"""Create the exact nested eval configuration for a custom agent."""
|
|
275
|
+
return replace(
|
|
276
|
+
self,
|
|
277
|
+
name=name,
|
|
278
|
+
model=model,
|
|
279
|
+
reasoning_effort=reasoning_effort,
|
|
280
|
+
instructions=instructions,
|
|
281
|
+
timeout_s=timeout_s,
|
|
282
|
+
max_turns=max_turns,
|
|
283
|
+
allowed_tools=allowed_tools,
|
|
284
|
+
excluded_tools=None,
|
|
285
|
+
mcp_servers=mcp_servers,
|
|
286
|
+
active_agent=None,
|
|
287
|
+
custom_agents=[
|
|
288
|
+
agent for agent in self.custom_agents if require_custom_agent_name(agent) != name
|
|
289
|
+
],
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
@classmethod
|
|
293
|
+
def from_copilot_config(
|
|
294
|
+
cls,
|
|
295
|
+
path: str | Path = ".",
|
|
296
|
+
**overrides: Any,
|
|
297
|
+
) -> "CopilotTask":
|
|
298
|
+
"""Load a ``CopilotTask`` from a directory containing Copilot config files.
|
|
299
|
+
|
|
300
|
+
Looks for the following files under ``path``:
|
|
301
|
+
|
|
302
|
+
* ``.github/copilot-instructions.md`` → ``instructions``
|
|
303
|
+
* ``.github/agents/*.agent.md`` → ``custom_agents``
|
|
304
|
+
|
|
305
|
+
Point ``path`` at any directory — your production project, a dedicated
|
|
306
|
+
test fixture project, or a shared team config repo. Any keyword
|
|
307
|
+
argument overrides the loaded value.
|
|
308
|
+
|
|
309
|
+
Args:
|
|
310
|
+
path: Root directory to load config from. Defaults to the current
|
|
311
|
+
working directory.
|
|
312
|
+
**overrides: Override any ``CopilotTask`` field after loading,
|
|
313
|
+
e.g. ``model="claude-opus-4.5"``.
|
|
314
|
+
|
|
315
|
+
Returns:
|
|
316
|
+
A ``CopilotTask`` initialised from the discovered config files.
|
|
317
|
+
|
|
318
|
+
Example::
|
|
319
|
+
|
|
320
|
+
# Load from the current project (production config as baseline)
|
|
321
|
+
baseline = CopilotTask.from_copilot_config()
|
|
322
|
+
|
|
323
|
+
# A/B test: same config, one instruction changed
|
|
324
|
+
treatment = CopilotTask.from_copilot_config(
|
|
325
|
+
instructions="Always add type hints.",
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
# Load from a dedicated test-fixture project
|
|
329
|
+
agent = CopilotTask.from_copilot_config("tests/fixtures/strict-agent")
|
|
330
|
+
|
|
331
|
+
# Load from a shared team agent library
|
|
332
|
+
agent = CopilotTask.from_copilot_config("/shared/team/copilot-config")
|
|
333
|
+
"""
|
|
334
|
+
root = Path(path).resolve()
|
|
335
|
+
github_dir = root / ".github"
|
|
336
|
+
if not root.is_dir():
|
|
337
|
+
raise FileNotFoundError(f"Copilot config root is not a directory: {root}")
|
|
338
|
+
if github_dir.exists() and not github_dir.is_dir():
|
|
339
|
+
raise ValueError(f"{github_dir}: Copilot config path must be a directory")
|
|
340
|
+
|
|
341
|
+
# Load repository-wide instructions
|
|
342
|
+
instructions: str | None = None
|
|
343
|
+
instructions_file = github_dir / "copilot-instructions.md"
|
|
344
|
+
if instructions_file.exists():
|
|
345
|
+
instructions = load_instruction_file(instructions_file)["content"]
|
|
346
|
+
|
|
347
|
+
# Load custom agents — recursive so subagents/ subdirectories are included
|
|
348
|
+
agents: list[CopilotCustomAgentConfig] = []
|
|
349
|
+
agents_dir = github_dir / "agents"
|
|
350
|
+
if agents_dir.exists():
|
|
351
|
+
if not agents_dir.is_dir():
|
|
352
|
+
raise ValueError(f"{agents_dir}: custom agents path must be a directory")
|
|
353
|
+
agents = _parse_agent_files(sorted(agents_dir.rglob("*.agent.md")))
|
|
354
|
+
|
|
355
|
+
config: dict[str, Any] = {
|
|
356
|
+
"instructions": instructions,
|
|
357
|
+
"custom_agents": agents,
|
|
358
|
+
}
|
|
359
|
+
config.update(overrides)
|
|
360
|
+
return cls(**config)
|
|
361
|
+
|
|
362
|
+
@classmethod
|
|
363
|
+
def from_plugin(
|
|
364
|
+
cls,
|
|
365
|
+
path: str | Path,
|
|
366
|
+
*,
|
|
367
|
+
model: str = "",
|
|
368
|
+
persona: CopilotPersona | None = None,
|
|
369
|
+
instructions: str = "",
|
|
370
|
+
working_directory: str = "",
|
|
371
|
+
name: str = "",
|
|
372
|
+
**overrides: Any,
|
|
373
|
+
) -> "CopilotTask":
|
|
374
|
+
"""Create a CopilotTask from a plugin directory.
|
|
375
|
+
|
|
376
|
+
Loads the plugin's agents, skills, MCP servers, and instructions,
|
|
377
|
+
then constructs a CopilotTask with all components wired together.
|
|
378
|
+
|
|
379
|
+
Uses :func:`~pytest_copilot.core.plugin.load_plugin` to
|
|
380
|
+
discover the plugin structure. The persona is auto-detected from the
|
|
381
|
+
plugin path (``ClaudeCodePersona`` for ``.claude/`` paths,
|
|
382
|
+
``VSCodePersona`` otherwise) unless explicitly overridden.
|
|
383
|
+
|
|
384
|
+
Args:
|
|
385
|
+
path: Path to the plugin directory (may contain ``plugin.json``,
|
|
386
|
+
or be a ``.github/`` / ``.claude/`` project config directory).
|
|
387
|
+
model: Model override for the eval.
|
|
388
|
+
persona: IDE persona override. Auto-detected when ``None``.
|
|
389
|
+
instructions: Additional instructions to **append** to the
|
|
390
|
+
plugin's discovered instructions.
|
|
391
|
+
working_directory: Override the working directory.
|
|
392
|
+
name: Override the eval name (defaults to plugin metadata name).
|
|
393
|
+
**overrides: Override any additional ``CopilotTask`` fields such as
|
|
394
|
+
``max_turns`` or ``timeout_s``.
|
|
395
|
+
|
|
396
|
+
Returns:
|
|
397
|
+
A ``CopilotTask`` initialised from the plugin.
|
|
398
|
+
|
|
399
|
+
Example::
|
|
400
|
+
|
|
401
|
+
agent = CopilotTask.from_plugin("my-plugin/")
|
|
402
|
+
|
|
403
|
+
agent = CopilotTask.from_plugin(
|
|
404
|
+
".claude/",
|
|
405
|
+
model="claude-sonnet-4",
|
|
406
|
+
instructions="Focus on security reviews.",
|
|
407
|
+
)
|
|
408
|
+
"""
|
|
409
|
+
from pytest_copilot.core.plugin import load_plugin # noqa: PLC0415
|
|
410
|
+
|
|
411
|
+
plugin = load_plugin(path)
|
|
412
|
+
|
|
413
|
+
# Build combined instructions: plugin base + caller override
|
|
414
|
+
combined_instructions = plugin.instructions
|
|
415
|
+
if instructions:
|
|
416
|
+
if combined_instructions:
|
|
417
|
+
combined_instructions = combined_instructions + "\n\n" + instructions
|
|
418
|
+
else:
|
|
419
|
+
combined_instructions = instructions
|
|
420
|
+
|
|
421
|
+
# Map plugin.skills to skill directory paths
|
|
422
|
+
skill_dirs = [str(s.path) for s in plugin.skills]
|
|
423
|
+
|
|
424
|
+
# Auto-detect persona from plugin path
|
|
425
|
+
if persona is None:
|
|
426
|
+
resolved = Path(path).resolve()
|
|
427
|
+
if resolved.name == ".claude" or (resolved / ".claude").is_dir():
|
|
428
|
+
persona = ClaudeCodePersona()
|
|
429
|
+
else:
|
|
430
|
+
persona = _default_persona()
|
|
431
|
+
|
|
432
|
+
config: dict[str, Any] = {
|
|
433
|
+
"name": name or plugin.metadata.name,
|
|
434
|
+
"model": model or None,
|
|
435
|
+
"instructions": combined_instructions or None,
|
|
436
|
+
"custom_agents": plugin.agents,
|
|
437
|
+
"skill_directories": skill_dirs,
|
|
438
|
+
"mcp_servers": plugin.mcp_servers,
|
|
439
|
+
"working_directory": working_directory or None,
|
|
440
|
+
"persona": persona,
|
|
441
|
+
}
|
|
442
|
+
config.update(overrides)
|
|
443
|
+
return cls(
|
|
444
|
+
**config,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
@classmethod
|
|
448
|
+
def from_claude_config(
|
|
449
|
+
cls,
|
|
450
|
+
path: str | Path = ".",
|
|
451
|
+
*,
|
|
452
|
+
model: str = "",
|
|
453
|
+
persona: CopilotPersona | None = None,
|
|
454
|
+
instructions: str = "",
|
|
455
|
+
working_directory: str = "",
|
|
456
|
+
name: str = "claude-code-eval",
|
|
457
|
+
**overrides: Any,
|
|
458
|
+
) -> "CopilotTask":
|
|
459
|
+
"""Create a CopilotTask from a Claude Code project directory.
|
|
460
|
+
|
|
461
|
+
Scans for:
|
|
462
|
+
|
|
463
|
+
* ``CLAUDE.md`` (project root) and ``.claude/CLAUDE.md`` → instructions
|
|
464
|
+
* ``.claude/agents/*.md`` → custom agents
|
|
465
|
+
* ``.claude/skills/`` → skill directories (subdirs with ``SKILL.md``)
|
|
466
|
+
* ``.mcp.json`` → MCP server configs
|
|
467
|
+
|
|
468
|
+
Args:
|
|
469
|
+
path: Root of the Claude Code project (default: current directory).
|
|
470
|
+
model: Model override for the eval.
|
|
471
|
+
persona: IDE persona override. Defaults to ``ClaudeCodePersona``.
|
|
472
|
+
instructions: Additional instructions to **append** to the
|
|
473
|
+
discovered ``CLAUDE.md`` content.
|
|
474
|
+
working_directory: Override the working directory.
|
|
475
|
+
name: Override the eval name.
|
|
476
|
+
**overrides: Override any additional ``CopilotTask`` fields such as
|
|
477
|
+
``max_turns`` or ``timeout_s``.
|
|
478
|
+
|
|
479
|
+
Returns:
|
|
480
|
+
A ``CopilotTask`` initialised from the discovered config files.
|
|
481
|
+
|
|
482
|
+
Example::
|
|
483
|
+
|
|
484
|
+
# Load from the current project
|
|
485
|
+
agent = CopilotTask.from_claude_config()
|
|
486
|
+
|
|
487
|
+
# Load from a specific directory with model override
|
|
488
|
+
agent = CopilotTask.from_claude_config(
|
|
489
|
+
"tests/fixtures/claude-project",
|
|
490
|
+
model="claude-sonnet-4",
|
|
491
|
+
)
|
|
492
|
+
"""
|
|
493
|
+
from pytest_copilot.copilot.config import load_mcp_config # noqa: PLC0415
|
|
494
|
+
|
|
495
|
+
root = Path(path).resolve()
|
|
496
|
+
claude_dir = root / ".claude"
|
|
497
|
+
if not root.is_dir():
|
|
498
|
+
raise FileNotFoundError(f"Claude config root is not a directory: {root}")
|
|
499
|
+
if claude_dir.exists() and not claude_dir.is_dir():
|
|
500
|
+
raise ValueError(f"{claude_dir}: Claude config path must be a directory")
|
|
501
|
+
|
|
502
|
+
# 1. Concatenate instructions from CLAUDE.md files
|
|
503
|
+
instruction_parts: list[str] = []
|
|
504
|
+
for md_path in [root / "CLAUDE.md", claude_dir / "CLAUDE.md"]:
|
|
505
|
+
if md_path.exists():
|
|
506
|
+
instruction_parts.append(load_instruction_file(md_path)["content"])
|
|
507
|
+
if instructions:
|
|
508
|
+
instruction_parts.append(instructions)
|
|
509
|
+
combined_instructions = "\n\n".join(instruction_parts) or None
|
|
510
|
+
|
|
511
|
+
# 2. Load custom agents from .claude/agents/
|
|
512
|
+
agents: list[CopilotCustomAgentConfig] = []
|
|
513
|
+
agents_dir = claude_dir / "agents"
|
|
514
|
+
if agents_dir.exists():
|
|
515
|
+
if not agents_dir.is_dir():
|
|
516
|
+
raise ValueError(f"{agents_dir}: custom agents path must be a directory")
|
|
517
|
+
# Claude Code uses plain .md files for agents
|
|
518
|
+
agents = _parse_agent_files(sorted(agents_dir.glob("*.md")))
|
|
519
|
+
|
|
520
|
+
# 3. Discover skill directories from .claude/skills/
|
|
521
|
+
skills = _discover_skills(claude_dir)
|
|
522
|
+
_validate_skill_reference_names(skills)
|
|
523
|
+
skill_dirs = [str(skill.path) for skill in skills]
|
|
524
|
+
|
|
525
|
+
# 4. Parse .mcp.json if it exists
|
|
526
|
+
mcp_servers: dict[str, dict[str, Any]] = {}
|
|
527
|
+
mcp_config_path = root / ".mcp.json"
|
|
528
|
+
if mcp_config_path.exists():
|
|
529
|
+
mcp_servers = load_mcp_config(mcp_config_path)
|
|
530
|
+
|
|
531
|
+
# 5. Default persona to ClaudeCodePersona
|
|
532
|
+
if persona is None:
|
|
533
|
+
persona = ClaudeCodePersona()
|
|
534
|
+
|
|
535
|
+
config: dict[str, Any] = {
|
|
536
|
+
"name": name,
|
|
537
|
+
"model": model or None,
|
|
538
|
+
"instructions": combined_instructions,
|
|
539
|
+
"custom_agents": agents,
|
|
540
|
+
"skill_directories": skill_dirs,
|
|
541
|
+
"mcp_servers": mcp_servers,
|
|
542
|
+
"working_directory": working_directory or None,
|
|
543
|
+
"persona": persona,
|
|
544
|
+
}
|
|
545
|
+
config.update(overrides)
|
|
546
|
+
return cls(**config)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Core module - result types and skill management."""
|
|
2
|
+
|
|
3
|
+
from pytest_copilot.core.errors import AITestError, EngineTimeoutError, ServerStartError
|
|
4
|
+
from pytest_copilot.core.evals import (
|
|
5
|
+
load_custom_agent,
|
|
6
|
+
load_custom_agents,
|
|
7
|
+
load_instruction_file,
|
|
8
|
+
load_instruction_files,
|
|
9
|
+
load_prompt_file,
|
|
10
|
+
load_prompt_files,
|
|
11
|
+
)
|
|
12
|
+
from pytest_copilot.core.plugin import HookDefinition, Plugin, PluginMetadata, load_plugin
|
|
13
|
+
from pytest_copilot.core.prompt import (
|
|
14
|
+
Prompt,
|
|
15
|
+
load_prompt,
|
|
16
|
+
load_prompts,
|
|
17
|
+
load_system_prompts,
|
|
18
|
+
)
|
|
19
|
+
from pytest_copilot.core.result import (
|
|
20
|
+
DiscoveredSkill,
|
|
21
|
+
EvalResult,
|
|
22
|
+
ImageContent,
|
|
23
|
+
MCPPrompt,
|
|
24
|
+
MCPPromptArgument,
|
|
25
|
+
SkillDiscovery,
|
|
26
|
+
SkillInfo,
|
|
27
|
+
SubagentInvocation,
|
|
28
|
+
ToolCall,
|
|
29
|
+
ToolInfo,
|
|
30
|
+
Turn,
|
|
31
|
+
)
|
|
32
|
+
from pytest_copilot.core.skill import Skill, SkillError, SkillMetadata, load_skill
|
|
33
|
+
from pytest_copilot.core.skill_evals import (
|
|
34
|
+
SkillEvalCase,
|
|
35
|
+
has_skill_evals,
|
|
36
|
+
load_skill_evals,
|
|
37
|
+
)
|
|
38
|
+
from pytest_copilot.core.skill_grading import export_grading
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"AITestError",
|
|
42
|
+
"EvalResult",
|
|
43
|
+
"DiscoveredSkill",
|
|
44
|
+
"EngineTimeoutError",
|
|
45
|
+
"HookDefinition",
|
|
46
|
+
"ImageContent",
|
|
47
|
+
"MCPPrompt",
|
|
48
|
+
"MCPPromptArgument",
|
|
49
|
+
"Plugin",
|
|
50
|
+
"PluginMetadata",
|
|
51
|
+
"Prompt",
|
|
52
|
+
"ServerStartError",
|
|
53
|
+
"Skill",
|
|
54
|
+
"SkillDiscovery",
|
|
55
|
+
"SkillError",
|
|
56
|
+
"SkillInfo",
|
|
57
|
+
"SkillMetadata",
|
|
58
|
+
"SubagentInvocation",
|
|
59
|
+
"ToolCall",
|
|
60
|
+
"ToolInfo",
|
|
61
|
+
"Turn",
|
|
62
|
+
"load_custom_agent",
|
|
63
|
+
"load_custom_agents",
|
|
64
|
+
"load_instruction_file",
|
|
65
|
+
"load_instruction_files",
|
|
66
|
+
"load_plugin",
|
|
67
|
+
"load_prompt_file",
|
|
68
|
+
"load_prompt_files",
|
|
69
|
+
"load_prompt",
|
|
70
|
+
"load_prompts",
|
|
71
|
+
"load_skill",
|
|
72
|
+
"load_system_prompts",
|
|
73
|
+
"SkillEvalCase",
|
|
74
|
+
"has_skill_evals",
|
|
75
|
+
"load_skill_evals",
|
|
76
|
+
"export_grading",
|
|
77
|
+
]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Custom exceptions for pytest-copilot."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class AITestError(Exception):
|
|
7
|
+
"""Base exception for pytest-copilot errors."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ServerStartError(AITestError):
|
|
11
|
+
"""Error starting an MCP server."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, server_type: str, command: list[str], message: str) -> None:
|
|
14
|
+
self.server_type = server_type
|
|
15
|
+
self.command = command
|
|
16
|
+
super().__init__(f"Failed to start {server_type} server ({' '.join(command)}): {message}")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class EngineTimeoutError(AITestError):
|
|
20
|
+
"""Eval engine timed out."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, timeout_ms: int, turns_completed: int) -> None:
|
|
23
|
+
self.timeout_ms = timeout_ms
|
|
24
|
+
self.turns_completed = turns_completed
|
|
25
|
+
super().__init__(f"Eval timed out after {timeout_ms}ms ({turns_completed} turns completed)")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ToolCallError(AITestError):
|
|
29
|
+
"""Error calling a tool."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, tool_name: str, message: str) -> None:
|
|
32
|
+
self.tool_name = tool_name
|
|
33
|
+
super().__init__(f"Tool '{tool_name}' failed: {message}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RateLimitError(AITestError):
|
|
37
|
+
"""Rate limit exceeded."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, retry_after: float | None = None) -> None:
|
|
40
|
+
self.retry_after = retry_after
|
|
41
|
+
msg = "Rate limit exceeded"
|
|
42
|
+
if retry_after:
|
|
43
|
+
msg += f" (retry after {retry_after}s)"
|
|
44
|
+
super().__init__(msg)
|