pytest-copilot 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_copilot/__init__.py +133 -0
- pytest_copilot/copilot/__init__.py +47 -0
- pytest_copilot/copilot/_command_job.py +109 -0
- pytest_copilot/copilot/api.py +12 -0
- pytest_copilot/copilot/client.py +58 -0
- pytest_copilot/copilot/commands.py +265 -0
- pytest_copilot/copilot/config.py +133 -0
- pytest_copilot/copilot/contracts.py +222 -0
- pytest_copilot/copilot/controls.py +230 -0
- pytest_copilot/copilot/events.py +795 -0
- pytest_copilot/copilot/fixtures.py +289 -0
- pytest_copilot/copilot/personas.py +607 -0
- pytest_copilot/copilot/requests.py +350 -0
- pytest_copilot/copilot/result.py +309 -0
- pytest_copilot/copilot/runner.py +266 -0
- pytest_copilot/copilot/skills.py +104 -0
- pytest_copilot/copilot/task.py +546 -0
- pytest_copilot/core/__init__.py +77 -0
- pytest_copilot/core/errors.py +44 -0
- pytest_copilot/core/evals.py +521 -0
- pytest_copilot/core/plugin.py +403 -0
- pytest_copilot/core/prompt.py +167 -0
- pytest_copilot/core/result.py +543 -0
- pytest_copilot/core/serialization.py +128 -0
- pytest_copilot/core/skill.py +406 -0
- pytest_copilot/core/skill_evals.py +148 -0
- pytest_copilot/core/skill_grading.py +78 -0
- pytest_copilot/core/validation.py +83 -0
- pytest_copilot/execution/__init__.py +15 -0
- pytest_copilot/execution/cost.py +159 -0
- pytest_copilot/execution/rate_limiter.py +177 -0
- pytest_copilot/execution/servers.py +442 -0
- pytest_copilot/execution/skill_tools.py +228 -0
- pytest_copilot/fixtures/__init__.py +18 -0
- pytest_copilot/fixtures/factories.py +44 -0
- pytest_copilot/fixtures/iteration.py +16 -0
- pytest_copilot/onboarding.py +413 -0
- pytest_copilot/plugin.py +562 -0
- pytest_copilot/plugin_options.py +32 -0
- pytest_copilot/plugin_report.py +18 -0
- pytest_copilot/reporting/__init__.py +16 -0
- pytest_copilot/reporting/collector.py +203 -0
- pytest_copilot/reporting/generator.py +55 -0
- pytest_copilot/reporting/schema.py +5 -0
- pytest_copilot/testing/__init__.py +17 -0
- pytest_copilot/testing/banking.py +479 -0
- pytest_copilot/testing/banking_mcp.py +167 -0
- pytest_copilot/testing/todo.py +380 -0
- pytest_copilot/testing/todo_mcp.py +156 -0
- pytest_copilot/testing/types.py +31 -0
- pytest_copilot-2.0.0.dist-info/METADATA +177 -0
- pytest_copilot-2.0.0.dist-info/RECORD +55 -0
- pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
- pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
- pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
"""Eval Skills support following agentskills.io specification.
|
|
2
|
+
|
|
3
|
+
Skills are domain-specific knowledge modules that enhance agent capabilities.
|
|
4
|
+
A skill consists of:
|
|
5
|
+
- SKILL.md file with YAML frontmatter (metadata) and markdown body (instructions)
|
|
6
|
+
- Optional references/ directory with additional documentation (.md files)
|
|
7
|
+
- Optional scripts/ directory with executable scripts (.py, .sh, .js, .bash)
|
|
8
|
+
- Optional assets/ directory with supporting files (any type)
|
|
9
|
+
|
|
10
|
+
Example SKILL.md:
|
|
11
|
+
---
|
|
12
|
+
name: my-skill
|
|
13
|
+
description: What this skill does
|
|
14
|
+
version: 1.0.0
|
|
15
|
+
license: Apache-2.0
|
|
16
|
+
compatibility: Requires Python 3.11+
|
|
17
|
+
allowed-tools: bash python
|
|
18
|
+
metadata:
|
|
19
|
+
author: my-team
|
|
20
|
+
category: development
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
# Skill Instructions
|
|
24
|
+
|
|
25
|
+
Content that gets prepended to the agent's system prompt.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import re
|
|
31
|
+
import stat
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
|
|
35
|
+
from pytest_copilot.core.evals import _extract_frontmatter
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SkillError(Exception):
|
|
39
|
+
"""Error loading or validating a skill."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(slots=True, frozen=True)
|
|
43
|
+
class SkillMetadata:
|
|
44
|
+
"""Metadata from SKILL.md frontmatter.
|
|
45
|
+
|
|
46
|
+
Required fields per agentskills.io spec:
|
|
47
|
+
- name: lowercase letters and hyphens only, 1-64 chars
|
|
48
|
+
- description: what the skill does, max 1024 chars
|
|
49
|
+
|
|
50
|
+
Optional fields:
|
|
51
|
+
- version: semantic version string
|
|
52
|
+
- license: SPDX license identifier
|
|
53
|
+
- tags: list of categorization tags
|
|
54
|
+
- compatibility: environment requirements, max 500 chars
|
|
55
|
+
- metadata: arbitrary key-value pairs (stored as tuple of tuples for frozen compat)
|
|
56
|
+
- allowed_tools: tool names the skill is designed to work with
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
name: str
|
|
60
|
+
description: str
|
|
61
|
+
version: str | None = None
|
|
62
|
+
license: str | None = None
|
|
63
|
+
tags: tuple[str, ...] = ()
|
|
64
|
+
compatibility: str | None = None
|
|
65
|
+
metadata_entries: tuple[tuple[str, str], ...] = ()
|
|
66
|
+
allowed_tools: tuple[str, ...] = ()
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def metadata_dict(self) -> dict[str, str]:
|
|
70
|
+
"""Return metadata entries as a dict for convenient access."""
|
|
71
|
+
return dict(self.metadata_entries)
|
|
72
|
+
|
|
73
|
+
def __post_init__(self) -> None:
|
|
74
|
+
"""Validate metadata per agentskills.io spec."""
|
|
75
|
+
# Name validation: lowercase letters, numbers, and hyphens, 1-64 chars
|
|
76
|
+
# Must not start/end with hyphen or contain consecutive hyphens.
|
|
77
|
+
if not isinstance(self.name, str) or not self.name.strip():
|
|
78
|
+
raise SkillError("Skill name must be a non-empty string")
|
|
79
|
+
if len(self.name) > 64:
|
|
80
|
+
raise SkillError(f"Skill name exceeds 64 characters: {len(self.name)}")
|
|
81
|
+
if not re.match(r"^[a-z0-9]+(?:-[a-z0-9]+)*$", self.name):
|
|
82
|
+
raise SkillError(
|
|
83
|
+
f"Invalid skill name '{self.name}': must be lowercase letters, "
|
|
84
|
+
"numbers, and hyphens (no leading/trailing/consecutive hyphens)"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Description validation
|
|
88
|
+
if not isinstance(self.description, str) or not self.description.strip():
|
|
89
|
+
raise SkillError("Skill description must be a non-empty string")
|
|
90
|
+
if len(self.description) > 1024:
|
|
91
|
+
raise SkillError(f"Skill description exceeds 1024 characters: {len(self.description)}")
|
|
92
|
+
|
|
93
|
+
# Compatibility validation
|
|
94
|
+
if self.compatibility is not None:
|
|
95
|
+
if not isinstance(self.compatibility, str):
|
|
96
|
+
raise SkillError("Skill compatibility must be a string")
|
|
97
|
+
if len(self.compatibility) > 500:
|
|
98
|
+
raise SkillError(
|
|
99
|
+
f"Skill compatibility exceeds 500 characters: {len(self.compatibility)}"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# Allowed tools validation
|
|
103
|
+
for tool in self.allowed_tools:
|
|
104
|
+
if not isinstance(tool, str) or not tool.strip():
|
|
105
|
+
raise SkillError("allowed_tools entries must be non-empty strings")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(slots=True)
|
|
109
|
+
class Skill:
|
|
110
|
+
"""An Eval Skill loaded from a SKILL.md file.
|
|
111
|
+
|
|
112
|
+
Skills provide domain knowledge to agents by:
|
|
113
|
+
1. Prepending instructions to the system prompt
|
|
114
|
+
2. Optionally providing reference documents via virtual tools
|
|
115
|
+
|
|
116
|
+
Example:
|
|
117
|
+
skill = Skill.from_path(Path("skills/my-skill"))
|
|
118
|
+
agent = CopilotTask(skill_directories=[str(skill.path)])
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
path: Path
|
|
122
|
+
metadata: SkillMetadata
|
|
123
|
+
content: str
|
|
124
|
+
references: dict[str, str] = field(default_factory=dict)
|
|
125
|
+
scripts: dict[str, str] = field(default_factory=dict)
|
|
126
|
+
assets: tuple[str, ...] = ()
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def name(self) -> str:
|
|
130
|
+
"""Skill name from metadata."""
|
|
131
|
+
return self.metadata.name
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def description(self) -> str:
|
|
135
|
+
"""Skill description from metadata."""
|
|
136
|
+
return self.metadata.description
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def has_references(self) -> bool:
|
|
140
|
+
"""Whether this skill has reference documents."""
|
|
141
|
+
return bool(self.references)
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def has_scripts(self) -> bool:
|
|
145
|
+
"""Whether this skill has executable scripts."""
|
|
146
|
+
return bool(self.scripts)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def has_assets(self) -> bool:
|
|
150
|
+
"""Whether this skill has asset files."""
|
|
151
|
+
return bool(self.assets)
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def assets_dir(self) -> Path | None:
|
|
155
|
+
"""Path to assets directory, if it exists."""
|
|
156
|
+
d = self.path / "assets"
|
|
157
|
+
return d if d.is_dir() else None
|
|
158
|
+
|
|
159
|
+
@classmethod
|
|
160
|
+
def from_path(cls, path: Path | str) -> Skill:
|
|
161
|
+
"""Load a skill from a directory containing SKILL.md.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
path: Path to skill directory or SKILL.md file
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
Loaded Skill instance. Reference keys are slash-separated paths
|
|
168
|
+
relative to references/, including any nested subdirectories.
|
|
169
|
+
|
|
170
|
+
Raises:
|
|
171
|
+
SkillError: If skill cannot be loaded or is invalid
|
|
172
|
+
"""
|
|
173
|
+
path = Path(path)
|
|
174
|
+
|
|
175
|
+
# Handle both directory and direct file paths
|
|
176
|
+
if path.is_file() and path.name == "SKILL.md":
|
|
177
|
+
skill_file = path
|
|
178
|
+
skill_dir = path.parent
|
|
179
|
+
elif path.is_dir():
|
|
180
|
+
skill_file = path / "SKILL.md"
|
|
181
|
+
skill_dir = path
|
|
182
|
+
else:
|
|
183
|
+
raise SkillError(f"Invalid skill path: {path}")
|
|
184
|
+
|
|
185
|
+
if not skill_file.exists():
|
|
186
|
+
raise SkillError(f"SKILL.md not found: {skill_file}")
|
|
187
|
+
|
|
188
|
+
# Parse SKILL.md
|
|
189
|
+
try:
|
|
190
|
+
raw_content = skill_file.read_text(encoding="utf-8")
|
|
191
|
+
metadata, content = _parse_skill_md(raw_content)
|
|
192
|
+
except (SkillError, UnicodeError, OSError) as exc:
|
|
193
|
+
raise SkillError(f"{skill_file}: {exc}") from exc
|
|
194
|
+
if metadata.name != skill_dir.name:
|
|
195
|
+
raise SkillError(
|
|
196
|
+
f"{skill_file}: Skill name '{metadata.name}' must match "
|
|
197
|
+
f"directory name '{skill_dir.name}'"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
for name in ("references", "scripts", "assets"):
|
|
201
|
+
component = skill_dir / name
|
|
202
|
+
if component.exists() and not component.is_dir():
|
|
203
|
+
raise SkillError(f"{component}: skill component must be a directory")
|
|
204
|
+
|
|
205
|
+
# Load references if directory exists
|
|
206
|
+
references: dict[str, str] = {}
|
|
207
|
+
refs_dir = skill_dir / "references"
|
|
208
|
+
if refs_dir.is_dir():
|
|
209
|
+
references = _load_references(refs_dir)
|
|
210
|
+
|
|
211
|
+
# Load scripts if directory exists
|
|
212
|
+
scripts: dict[str, str] = {}
|
|
213
|
+
scripts_dir = skill_dir / "scripts"
|
|
214
|
+
if scripts_dir.is_dir():
|
|
215
|
+
scripts = _load_scripts(scripts_dir)
|
|
216
|
+
|
|
217
|
+
# Discover assets if directory exists
|
|
218
|
+
assets: tuple[str, ...] = ()
|
|
219
|
+
assets_dir = skill_dir / "assets"
|
|
220
|
+
if assets_dir.is_dir():
|
|
221
|
+
assets = tuple(sorted(f.name for f in assets_dir.iterdir() if f.is_file()))
|
|
222
|
+
|
|
223
|
+
return cls(
|
|
224
|
+
path=skill_dir,
|
|
225
|
+
metadata=metadata,
|
|
226
|
+
content=content,
|
|
227
|
+
references=references,
|
|
228
|
+
scripts=scripts,
|
|
229
|
+
assets=assets,
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _parse_skill_md(content: str) -> tuple[SkillMetadata, str]:
|
|
234
|
+
"""Parse SKILL.md into metadata and body content.
|
|
235
|
+
|
|
236
|
+
Format:
|
|
237
|
+
---
|
|
238
|
+
name: my-skill
|
|
239
|
+
description: What this skill does
|
|
240
|
+
version: 1.0.0
|
|
241
|
+
---
|
|
242
|
+
|
|
243
|
+
# Body content here
|
|
244
|
+
"""
|
|
245
|
+
if not re.match(r"\A---[ \t]*(?:\r?\n|\Z)", content):
|
|
246
|
+
raise SkillError(
|
|
247
|
+
"Invalid SKILL.md format: must have YAML frontmatter between --- delimiters"
|
|
248
|
+
)
|
|
249
|
+
try:
|
|
250
|
+
metadata_raw, body = _extract_frontmatter(content)
|
|
251
|
+
except ValueError as exc:
|
|
252
|
+
raise SkillError(f"Invalid SKILL.md frontmatter: {exc}") from exc
|
|
253
|
+
body = body.strip()
|
|
254
|
+
if not body:
|
|
255
|
+
raise SkillError("SKILL.md must have system prompt content after frontmatter")
|
|
256
|
+
|
|
257
|
+
# Extract required fields
|
|
258
|
+
name = metadata_raw.get("name")
|
|
259
|
+
description = metadata_raw.get("description")
|
|
260
|
+
|
|
261
|
+
if not isinstance(name, str) or not name.strip():
|
|
262
|
+
raise SkillError("SKILL.md field 'name' must be a non-empty string")
|
|
263
|
+
if not isinstance(description, str) or not description.strip():
|
|
264
|
+
raise SkillError("SKILL.md field 'description' must be a non-empty string")
|
|
265
|
+
|
|
266
|
+
# Extract optional fields
|
|
267
|
+
version = metadata_raw.get("version")
|
|
268
|
+
license_str = metadata_raw.get("license")
|
|
269
|
+
tags_raw = metadata_raw.get("tags", [])
|
|
270
|
+
compatibility = metadata_raw.get("compatibility")
|
|
271
|
+
metadata_entries_raw = metadata_raw.get("metadata", {})
|
|
272
|
+
allowed_tools_raw = metadata_raw.get("allowed-tools", "")
|
|
273
|
+
for key in ("version", "license", "compatibility"):
|
|
274
|
+
if key in metadata_raw and (
|
|
275
|
+
not isinstance(metadata_raw[key], str) or not metadata_raw[key].strip()
|
|
276
|
+
):
|
|
277
|
+
raise SkillError(f"SKILL.md field '{key}' must be a non-empty string")
|
|
278
|
+
|
|
279
|
+
# Handle tags (could be string or list)
|
|
280
|
+
if isinstance(tags_raw, str):
|
|
281
|
+
tags = tuple(t.strip() for t in tags_raw.split(",") if t.strip())
|
|
282
|
+
elif isinstance(tags_raw, list) and all(
|
|
283
|
+
isinstance(tag, str) and tag.strip() for tag in tags_raw
|
|
284
|
+
):
|
|
285
|
+
tags = tuple(tags_raw)
|
|
286
|
+
else:
|
|
287
|
+
raise SkillError("SKILL.md field 'tags' must be a string or list of non-empty strings")
|
|
288
|
+
|
|
289
|
+
# Handle metadata entries (dict → tuple of tuples for frozen dataclass)
|
|
290
|
+
if isinstance(metadata_entries_raw, dict) and all(
|
|
291
|
+
isinstance(key, str) and isinstance(value, str)
|
|
292
|
+
for key, value in metadata_entries_raw.items()
|
|
293
|
+
):
|
|
294
|
+
metadata_entries = tuple(metadata_entries_raw.items())
|
|
295
|
+
else:
|
|
296
|
+
raise SkillError("SKILL.md field 'metadata' must be a string-to-string mapping")
|
|
297
|
+
|
|
298
|
+
# Handle allowed-tools (space-delimited string or list)
|
|
299
|
+
if isinstance(allowed_tools_raw, str):
|
|
300
|
+
allowed_tools = tuple(t for t in allowed_tools_raw.split() if t)
|
|
301
|
+
elif isinstance(allowed_tools_raw, list) and all(
|
|
302
|
+
isinstance(tool, str) and tool.strip() for tool in allowed_tools_raw
|
|
303
|
+
):
|
|
304
|
+
allowed_tools = tuple(allowed_tools_raw)
|
|
305
|
+
else:
|
|
306
|
+
raise SkillError(
|
|
307
|
+
"SKILL.md field 'allowed-tools' must be a string or list of non-empty strings"
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
metadata = SkillMetadata(
|
|
311
|
+
name=name,
|
|
312
|
+
description=description,
|
|
313
|
+
version=version,
|
|
314
|
+
license=license_str,
|
|
315
|
+
tags=tags,
|
|
316
|
+
compatibility=compatibility,
|
|
317
|
+
metadata_entries=metadata_entries,
|
|
318
|
+
allowed_tools=allowed_tools,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
return metadata, body
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _load_references(refs_dir: Path) -> dict[str, str]:
|
|
325
|
+
"""Load Markdown files recursively from ordinary files and directories.
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
Dict mapping slash-separated paths relative to references/ to content
|
|
329
|
+
"""
|
|
330
|
+
references: dict[str, str] = {}
|
|
331
|
+
pending = [refs_dir]
|
|
332
|
+
while pending:
|
|
333
|
+
directory = pending.pop()
|
|
334
|
+
try:
|
|
335
|
+
entries = sorted(directory.iterdir())
|
|
336
|
+
except OSError as exc:
|
|
337
|
+
raise SkillError(f"Cannot read references directory {directory}: {exc}") from exc
|
|
338
|
+
for file_path in entries:
|
|
339
|
+
try:
|
|
340
|
+
mode = file_path.stat().st_mode
|
|
341
|
+
except OSError as exc:
|
|
342
|
+
raise SkillError(f"Cannot inspect reference entry {file_path}: {exc}") from exc
|
|
343
|
+
if stat.S_ISDIR(mode):
|
|
344
|
+
pending.append(file_path)
|
|
345
|
+
continue
|
|
346
|
+
if not stat.S_ISREG(mode):
|
|
347
|
+
raise SkillError(f"Invalid references entry (must be a file): {file_path}")
|
|
348
|
+
if file_path.suffix.lower() != ".md":
|
|
349
|
+
raise SkillError(
|
|
350
|
+
f"Invalid reference file '{file_path}': only .md files are allowed"
|
|
351
|
+
)
|
|
352
|
+
try:
|
|
353
|
+
content = file_path.read_text(encoding="utf-8")
|
|
354
|
+
except UnicodeDecodeError as exc:
|
|
355
|
+
raise SkillError(f"Reference file must be valid UTF-8 text: {file_path}") from exc
|
|
356
|
+
except OSError as exc:
|
|
357
|
+
raise SkillError(f"Cannot read reference file {file_path}: {exc}") from exc
|
|
358
|
+
if not content.strip():
|
|
359
|
+
raise SkillError(f"Reference file must not be empty: {file_path}")
|
|
360
|
+
references[file_path.relative_to(refs_dir).as_posix()] = content
|
|
361
|
+
|
|
362
|
+
return dict(sorted(references.items()))
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
_SCRIPT_EXTENSIONS = frozenset({".py", ".sh", ".js", ".bash"})
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _load_scripts(scripts_dir: Path) -> dict[str, str]:
|
|
369
|
+
"""Load executable scripts from scripts/ directory.
|
|
370
|
+
|
|
371
|
+
Supports .py, .sh, .js, and .bash files.
|
|
372
|
+
|
|
373
|
+
Returns:
|
|
374
|
+
Dict mapping filename to content
|
|
375
|
+
"""
|
|
376
|
+
scripts: dict[str, str] = {}
|
|
377
|
+
|
|
378
|
+
for file_path in scripts_dir.iterdir():
|
|
379
|
+
if not file_path.is_file():
|
|
380
|
+
continue
|
|
381
|
+
if file_path.suffix.lower() not in _SCRIPT_EXTENSIONS:
|
|
382
|
+
continue
|
|
383
|
+
try:
|
|
384
|
+
content = file_path.read_text(encoding="utf-8")
|
|
385
|
+
except UnicodeDecodeError as exc:
|
|
386
|
+
raise SkillError(f"Script file must be valid UTF-8 text: {file_path}") from exc
|
|
387
|
+
if not content.strip():
|
|
388
|
+
raise SkillError(f"Script file must not be empty: {file_path}")
|
|
389
|
+
scripts[file_path.name] = content
|
|
390
|
+
|
|
391
|
+
return scripts
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def load_skill(path: Path | str) -> Skill:
|
|
395
|
+
"""Load a skill from a path.
|
|
396
|
+
|
|
397
|
+
Convenience function wrapping Skill.from_path(). This is the runner's local
|
|
398
|
+
skill validation, without SDK startup or model execution.
|
|
399
|
+
|
|
400
|
+
Args:
|
|
401
|
+
path: Path to skill directory or SKILL.md file
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
Loaded Skill instance
|
|
405
|
+
"""
|
|
406
|
+
return Skill.from_path(path)
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Bridge between Anthropic's skill-creator eval format and pytest-copilot.
|
|
2
|
+
|
|
3
|
+
Loads evals/evals.json from skill directories and converts them to test cases
|
|
4
|
+
that can be parametrized with pytest. Consumers define explicit output checks;
|
|
5
|
+
free-text expectations are descriptions, not automatically executed assertions.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(slots=True, frozen=True)
|
|
17
|
+
class SkillEvalCase:
|
|
18
|
+
"""A single eval case from skill-creator's evals/evals.json.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
id: Numeric identifier from the eval file
|
|
22
|
+
name: Human-readable name derived from the prompt
|
|
23
|
+
prompt: The user prompt to send to the agent
|
|
24
|
+
expected_output: Description of expected result (for documentation)
|
|
25
|
+
expectations: Descriptions to translate into explicit output checks
|
|
26
|
+
files: Paths to files referenced by this eval case
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
id: int
|
|
30
|
+
name: str
|
|
31
|
+
prompt: str
|
|
32
|
+
expected_output: str | None = None
|
|
33
|
+
expectations: tuple[str, ...] = ()
|
|
34
|
+
files: tuple[str, ...] = ()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def load_skill_evals(skill_path: Path | str) -> list[SkillEvalCase]:
|
|
38
|
+
"""Load eval cases from a skill-creator compatible skill directory.
|
|
39
|
+
|
|
40
|
+
Looks for evals/evals.json in the skill directory and parses it
|
|
41
|
+
into SkillEvalCase objects for use with pytest parametrize.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
skill_path: Path to skill directory containing evals/evals.json,
|
|
45
|
+
or direct path to evals.json
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
List of eval cases
|
|
49
|
+
|
|
50
|
+
Raises:
|
|
51
|
+
FileNotFoundError: If evals/evals.json doesn't exist
|
|
52
|
+
ValueError: If the JSON format is invalid
|
|
53
|
+
"""
|
|
54
|
+
evals_file = _resolve_evals_path(skill_path)
|
|
55
|
+
|
|
56
|
+
raw = evals_file.read_text(encoding="utf-8")
|
|
57
|
+
try:
|
|
58
|
+
data = json.loads(raw)
|
|
59
|
+
except json.JSONDecodeError as exc:
|
|
60
|
+
raise ValueError(f"Invalid JSON in {evals_file}: {exc}") from exc
|
|
61
|
+
|
|
62
|
+
if not isinstance(data, dict):
|
|
63
|
+
raise ValueError(f"Expected JSON object in {evals_file}, got {type(data).__name__}")
|
|
64
|
+
if "evals" not in data:
|
|
65
|
+
raise ValueError(f"Missing required 'evals' array in {evals_file}")
|
|
66
|
+
if not isinstance(data["evals"], list):
|
|
67
|
+
raise ValueError(f"'evals' must be an array in {evals_file}")
|
|
68
|
+
|
|
69
|
+
cases: list[SkillEvalCase] = []
|
|
70
|
+
for i, entry in enumerate(data["evals"]):
|
|
71
|
+
cases.append(_parse_eval_entry(entry, i, evals_file))
|
|
72
|
+
|
|
73
|
+
return cases
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def has_skill_evals(skill_path: Path | str) -> bool:
|
|
77
|
+
"""Check if a skill directory has skill-creator evals."""
|
|
78
|
+
path = Path(skill_path)
|
|
79
|
+
if path.is_file():
|
|
80
|
+
path = path.parent
|
|
81
|
+
return (path / "evals" / "evals.json").exists()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _resolve_evals_path(skill_path: Path | str) -> Path:
|
|
85
|
+
"""Resolve skill_path to the evals/evals.json file."""
|
|
86
|
+
path = Path(skill_path)
|
|
87
|
+
|
|
88
|
+
if path.is_file() and path.name == "evals.json":
|
|
89
|
+
evals_file = path
|
|
90
|
+
elif path.is_dir():
|
|
91
|
+
evals_file = path / "evals" / "evals.json"
|
|
92
|
+
else:
|
|
93
|
+
raise FileNotFoundError(f"Invalid skill path: {path}")
|
|
94
|
+
|
|
95
|
+
if not evals_file.exists():
|
|
96
|
+
raise FileNotFoundError(f"evals/evals.json not found: {evals_file}")
|
|
97
|
+
|
|
98
|
+
return evals_file
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _parse_eval_entry(entry: dict, index: int, source: Path) -> SkillEvalCase:
|
|
102
|
+
"""Parse a single eval entry from the JSON array."""
|
|
103
|
+
if not isinstance(entry, dict):
|
|
104
|
+
raise ValueError(f"Eval entry {index} must be an object in {source}")
|
|
105
|
+
|
|
106
|
+
# Required fields
|
|
107
|
+
eval_id = entry.get("id")
|
|
108
|
+
if eval_id is None:
|
|
109
|
+
raise ValueError(f"Eval entry {index} missing required field 'id' in {source}")
|
|
110
|
+
|
|
111
|
+
prompt = entry.get("prompt")
|
|
112
|
+
if not prompt or not isinstance(prompt, str):
|
|
113
|
+
raise ValueError(f"Eval entry {index} missing required field 'prompt' in {source}")
|
|
114
|
+
|
|
115
|
+
# Optional fields
|
|
116
|
+
expected_output = entry.get("expected_output")
|
|
117
|
+
expectations_raw = entry.get("expectations", [])
|
|
118
|
+
files_raw = entry.get("files", [])
|
|
119
|
+
|
|
120
|
+
if not isinstance(expectations_raw, list):
|
|
121
|
+
raise ValueError(f"Eval entry {index}: 'expectations' must be an array in {source}")
|
|
122
|
+
if not isinstance(files_raw, list):
|
|
123
|
+
raise ValueError(f"Eval entry {index}: 'files' must be an array in {source}")
|
|
124
|
+
|
|
125
|
+
name = _slugify_prompt(prompt, eval_id)
|
|
126
|
+
|
|
127
|
+
return SkillEvalCase(
|
|
128
|
+
id=int(eval_id),
|
|
129
|
+
name=name,
|
|
130
|
+
prompt=prompt,
|
|
131
|
+
expected_output=expected_output if isinstance(expected_output, str) else None,
|
|
132
|
+
expectations=tuple(str(e) for e in expectations_raw),
|
|
133
|
+
files=tuple(str(f) for f in files_raw),
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _slugify_prompt(prompt: str, eval_id: int) -> str:
|
|
138
|
+
"""Generate a human-readable test name from a prompt.
|
|
139
|
+
|
|
140
|
+
Takes the first 50 characters, lowercases, replaces non-alphanumeric
|
|
141
|
+
with hyphens, and prepends the eval id.
|
|
142
|
+
"""
|
|
143
|
+
slug = prompt[:50].lower().strip()
|
|
144
|
+
slug = re.sub(r"[^a-z0-9]+", "-", slug)
|
|
145
|
+
slug = slug.strip("-")
|
|
146
|
+
if not slug:
|
|
147
|
+
return f"eval-{eval_id}"
|
|
148
|
+
return f"{eval_id}-{slug}"
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Export eval results in skill-creator compatible grading.json format.
|
|
2
|
+
|
|
3
|
+
This only formats explicitly supplied booleans and evidence. It does not run,
|
|
4
|
+
interpret, or grade a task. Consumers own their ordinary output assertions.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import TYPE_CHECKING, Any
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
|
|
14
|
+
from pytest_copilot.copilot.result import CopilotResult
|
|
15
|
+
from pytest_copilot.core.result import EvalResult
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def export_grading(
|
|
19
|
+
result: EvalResult | CopilotResult,
|
|
20
|
+
expectations: Sequence[str],
|
|
21
|
+
expectation_results: Sequence[bool],
|
|
22
|
+
evidence: Sequence[str] | None = None,
|
|
23
|
+
) -> dict[str, Any]:
|
|
24
|
+
"""Export eval result as skill-creator compatible grading.json.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
result: The eval result from a test run (EvalResult or CopilotResult)
|
|
28
|
+
expectations: The assertion texts that were checked
|
|
29
|
+
expectation_results: Whether each expectation passed
|
|
30
|
+
evidence: Optional evidence strings for each expectation
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
Dict matching skill-creator's grading.json schema
|
|
34
|
+
|
|
35
|
+
Raises:
|
|
36
|
+
ValueError: If expectations and results have different lengths
|
|
37
|
+
"""
|
|
38
|
+
if len(expectations) != len(expectation_results):
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"expectations ({len(expectations)}) and "
|
|
41
|
+
f"expectation_results ({len(expectation_results)}) must have the same length"
|
|
42
|
+
)
|
|
43
|
+
if evidence is not None and len(evidence) != len(expectations):
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"evidence ({len(evidence)}) must match expectations ({len(expectations)}) length"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
expectation_entries: list[dict[str, Any]] = []
|
|
49
|
+
for i, (text, passed) in enumerate(zip(expectations, expectation_results, strict=True)):
|
|
50
|
+
entry: dict[str, Any] = {
|
|
51
|
+
"text": text,
|
|
52
|
+
"passed": passed,
|
|
53
|
+
}
|
|
54
|
+
if evidence is not None:
|
|
55
|
+
entry["evidence"] = evidence[i]
|
|
56
|
+
expectation_entries.append(entry)
|
|
57
|
+
|
|
58
|
+
passed_count = sum(1 for r in expectation_results if r)
|
|
59
|
+
total = len(expectation_results)
|
|
60
|
+
failed_count = total - passed_count
|
|
61
|
+
|
|
62
|
+
grading: dict[str, Any] = {
|
|
63
|
+
"expectations": expectation_entries,
|
|
64
|
+
"summary": {
|
|
65
|
+
"passed": passed_count,
|
|
66
|
+
"failed": failed_count,
|
|
67
|
+
"total": total,
|
|
68
|
+
"pass_rate": round(passed_count / total, 2) if total > 0 else 0.0,
|
|
69
|
+
},
|
|
70
|
+
"execution_metrics": {
|
|
71
|
+
"tool_calls": len(result.all_tool_calls),
|
|
72
|
+
"turns": len(result.turns),
|
|
73
|
+
"duration_ms": result.duration_ms,
|
|
74
|
+
"success": result.success,
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return grading
|