pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,406 @@
1
+ """Eval Skills support following agentskills.io specification.
2
+
3
+ Skills are domain-specific knowledge modules that enhance agent capabilities.
4
+ A skill consists of:
5
+ - SKILL.md file with YAML frontmatter (metadata) and markdown body (instructions)
6
+ - Optional references/ directory with additional documentation (.md files)
7
+ - Optional scripts/ directory with executable scripts (.py, .sh, .js, .bash)
8
+ - Optional assets/ directory with supporting files (any type)
9
+
10
+ Example SKILL.md:
11
+ ---
12
+ name: my-skill
13
+ description: What this skill does
14
+ version: 1.0.0
15
+ license: Apache-2.0
16
+ compatibility: Requires Python 3.11+
17
+ allowed-tools: bash python
18
+ metadata:
19
+ author: my-team
20
+ category: development
21
+ ---
22
+
23
+ # Skill Instructions
24
+
25
+ Content that gets prepended to the agent's system prompt.
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import re
31
+ import stat
32
+ from dataclasses import dataclass, field
33
+ from pathlib import Path
34
+
35
+ from pytest_copilot.core.evals import _extract_frontmatter
36
+
37
+
38
+ class SkillError(Exception):
39
+ """Error loading or validating a skill."""
40
+
41
+
42
+ @dataclass(slots=True, frozen=True)
43
+ class SkillMetadata:
44
+ """Metadata from SKILL.md frontmatter.
45
+
46
+ Required fields per agentskills.io spec:
47
+ - name: lowercase letters and hyphens only, 1-64 chars
48
+ - description: what the skill does, max 1024 chars
49
+
50
+ Optional fields:
51
+ - version: semantic version string
52
+ - license: SPDX license identifier
53
+ - tags: list of categorization tags
54
+ - compatibility: environment requirements, max 500 chars
55
+ - metadata: arbitrary key-value pairs (stored as tuple of tuples for frozen compat)
56
+ - allowed_tools: tool names the skill is designed to work with
57
+ """
58
+
59
+ name: str
60
+ description: str
61
+ version: str | None = None
62
+ license: str | None = None
63
+ tags: tuple[str, ...] = ()
64
+ compatibility: str | None = None
65
+ metadata_entries: tuple[tuple[str, str], ...] = ()
66
+ allowed_tools: tuple[str, ...] = ()
67
+
68
+ @property
69
+ def metadata_dict(self) -> dict[str, str]:
70
+ """Return metadata entries as a dict for convenient access."""
71
+ return dict(self.metadata_entries)
72
+
73
+ def __post_init__(self) -> None:
74
+ """Validate metadata per agentskills.io spec."""
75
+ # Name validation: lowercase letters, numbers, and hyphens, 1-64 chars
76
+ # Must not start/end with hyphen or contain consecutive hyphens.
77
+ if not isinstance(self.name, str) or not self.name.strip():
78
+ raise SkillError("Skill name must be a non-empty string")
79
+ if len(self.name) > 64:
80
+ raise SkillError(f"Skill name exceeds 64 characters: {len(self.name)}")
81
+ if not re.match(r"^[a-z0-9]+(?:-[a-z0-9]+)*$", self.name):
82
+ raise SkillError(
83
+ f"Invalid skill name '{self.name}': must be lowercase letters, "
84
+ "numbers, and hyphens (no leading/trailing/consecutive hyphens)"
85
+ )
86
+
87
+ # Description validation
88
+ if not isinstance(self.description, str) or not self.description.strip():
89
+ raise SkillError("Skill description must be a non-empty string")
90
+ if len(self.description) > 1024:
91
+ raise SkillError(f"Skill description exceeds 1024 characters: {len(self.description)}")
92
+
93
+ # Compatibility validation
94
+ if self.compatibility is not None:
95
+ if not isinstance(self.compatibility, str):
96
+ raise SkillError("Skill compatibility must be a string")
97
+ if len(self.compatibility) > 500:
98
+ raise SkillError(
99
+ f"Skill compatibility exceeds 500 characters: {len(self.compatibility)}"
100
+ )
101
+
102
+ # Allowed tools validation
103
+ for tool in self.allowed_tools:
104
+ if not isinstance(tool, str) or not tool.strip():
105
+ raise SkillError("allowed_tools entries must be non-empty strings")
106
+
107
+
108
+ @dataclass(slots=True)
109
+ class Skill:
110
+ """An Eval Skill loaded from a SKILL.md file.
111
+
112
+ Skills provide domain knowledge to agents by:
113
+ 1. Prepending instructions to the system prompt
114
+ 2. Optionally providing reference documents via virtual tools
115
+
116
+ Example:
117
+ skill = Skill.from_path(Path("skills/my-skill"))
118
+ agent = CopilotTask(skill_directories=[str(skill.path)])
119
+ """
120
+
121
+ path: Path
122
+ metadata: SkillMetadata
123
+ content: str
124
+ references: dict[str, str] = field(default_factory=dict)
125
+ scripts: dict[str, str] = field(default_factory=dict)
126
+ assets: tuple[str, ...] = ()
127
+
128
+ @property
129
+ def name(self) -> str:
130
+ """Skill name from metadata."""
131
+ return self.metadata.name
132
+
133
+ @property
134
+ def description(self) -> str:
135
+ """Skill description from metadata."""
136
+ return self.metadata.description
137
+
138
+ @property
139
+ def has_references(self) -> bool:
140
+ """Whether this skill has reference documents."""
141
+ return bool(self.references)
142
+
143
+ @property
144
+ def has_scripts(self) -> bool:
145
+ """Whether this skill has executable scripts."""
146
+ return bool(self.scripts)
147
+
148
+ @property
149
+ def has_assets(self) -> bool:
150
+ """Whether this skill has asset files."""
151
+ return bool(self.assets)
152
+
153
+ @property
154
+ def assets_dir(self) -> Path | None:
155
+ """Path to assets directory, if it exists."""
156
+ d = self.path / "assets"
157
+ return d if d.is_dir() else None
158
+
159
+ @classmethod
160
+ def from_path(cls, path: Path | str) -> Skill:
161
+ """Load a skill from a directory containing SKILL.md.
162
+
163
+ Args:
164
+ path: Path to skill directory or SKILL.md file
165
+
166
+ Returns:
167
+ Loaded Skill instance. Reference keys are slash-separated paths
168
+ relative to references/, including any nested subdirectories.
169
+
170
+ Raises:
171
+ SkillError: If skill cannot be loaded or is invalid
172
+ """
173
+ path = Path(path)
174
+
175
+ # Handle both directory and direct file paths
176
+ if path.is_file() and path.name == "SKILL.md":
177
+ skill_file = path
178
+ skill_dir = path.parent
179
+ elif path.is_dir():
180
+ skill_file = path / "SKILL.md"
181
+ skill_dir = path
182
+ else:
183
+ raise SkillError(f"Invalid skill path: {path}")
184
+
185
+ if not skill_file.exists():
186
+ raise SkillError(f"SKILL.md not found: {skill_file}")
187
+
188
+ # Parse SKILL.md
189
+ try:
190
+ raw_content = skill_file.read_text(encoding="utf-8")
191
+ metadata, content = _parse_skill_md(raw_content)
192
+ except (SkillError, UnicodeError, OSError) as exc:
193
+ raise SkillError(f"{skill_file}: {exc}") from exc
194
+ if metadata.name != skill_dir.name:
195
+ raise SkillError(
196
+ f"{skill_file}: Skill name '{metadata.name}' must match "
197
+ f"directory name '{skill_dir.name}'"
198
+ )
199
+
200
+ for name in ("references", "scripts", "assets"):
201
+ component = skill_dir / name
202
+ if component.exists() and not component.is_dir():
203
+ raise SkillError(f"{component}: skill component must be a directory")
204
+
205
+ # Load references if directory exists
206
+ references: dict[str, str] = {}
207
+ refs_dir = skill_dir / "references"
208
+ if refs_dir.is_dir():
209
+ references = _load_references(refs_dir)
210
+
211
+ # Load scripts if directory exists
212
+ scripts: dict[str, str] = {}
213
+ scripts_dir = skill_dir / "scripts"
214
+ if scripts_dir.is_dir():
215
+ scripts = _load_scripts(scripts_dir)
216
+
217
+ # Discover assets if directory exists
218
+ assets: tuple[str, ...] = ()
219
+ assets_dir = skill_dir / "assets"
220
+ if assets_dir.is_dir():
221
+ assets = tuple(sorted(f.name for f in assets_dir.iterdir() if f.is_file()))
222
+
223
+ return cls(
224
+ path=skill_dir,
225
+ metadata=metadata,
226
+ content=content,
227
+ references=references,
228
+ scripts=scripts,
229
+ assets=assets,
230
+ )
231
+
232
+
233
+ def _parse_skill_md(content: str) -> tuple[SkillMetadata, str]:
234
+ """Parse SKILL.md into metadata and body content.
235
+
236
+ Format:
237
+ ---
238
+ name: my-skill
239
+ description: What this skill does
240
+ version: 1.0.0
241
+ ---
242
+
243
+ # Body content here
244
+ """
245
+ if not re.match(r"\A---[ \t]*(?:\r?\n|\Z)", content):
246
+ raise SkillError(
247
+ "Invalid SKILL.md format: must have YAML frontmatter between --- delimiters"
248
+ )
249
+ try:
250
+ metadata_raw, body = _extract_frontmatter(content)
251
+ except ValueError as exc:
252
+ raise SkillError(f"Invalid SKILL.md frontmatter: {exc}") from exc
253
+ body = body.strip()
254
+ if not body:
255
+ raise SkillError("SKILL.md must have system prompt content after frontmatter")
256
+
257
+ # Extract required fields
258
+ name = metadata_raw.get("name")
259
+ description = metadata_raw.get("description")
260
+
261
+ if not isinstance(name, str) or not name.strip():
262
+ raise SkillError("SKILL.md field 'name' must be a non-empty string")
263
+ if not isinstance(description, str) or not description.strip():
264
+ raise SkillError("SKILL.md field 'description' must be a non-empty string")
265
+
266
+ # Extract optional fields
267
+ version = metadata_raw.get("version")
268
+ license_str = metadata_raw.get("license")
269
+ tags_raw = metadata_raw.get("tags", [])
270
+ compatibility = metadata_raw.get("compatibility")
271
+ metadata_entries_raw = metadata_raw.get("metadata", {})
272
+ allowed_tools_raw = metadata_raw.get("allowed-tools", "")
273
+ for key in ("version", "license", "compatibility"):
274
+ if key in metadata_raw and (
275
+ not isinstance(metadata_raw[key], str) or not metadata_raw[key].strip()
276
+ ):
277
+ raise SkillError(f"SKILL.md field '{key}' must be a non-empty string")
278
+
279
+ # Handle tags (could be string or list)
280
+ if isinstance(tags_raw, str):
281
+ tags = tuple(t.strip() for t in tags_raw.split(",") if t.strip())
282
+ elif isinstance(tags_raw, list) and all(
283
+ isinstance(tag, str) and tag.strip() for tag in tags_raw
284
+ ):
285
+ tags = tuple(tags_raw)
286
+ else:
287
+ raise SkillError("SKILL.md field 'tags' must be a string or list of non-empty strings")
288
+
289
+ # Handle metadata entries (dict → tuple of tuples for frozen dataclass)
290
+ if isinstance(metadata_entries_raw, dict) and all(
291
+ isinstance(key, str) and isinstance(value, str)
292
+ for key, value in metadata_entries_raw.items()
293
+ ):
294
+ metadata_entries = tuple(metadata_entries_raw.items())
295
+ else:
296
+ raise SkillError("SKILL.md field 'metadata' must be a string-to-string mapping")
297
+
298
+ # Handle allowed-tools (space-delimited string or list)
299
+ if isinstance(allowed_tools_raw, str):
300
+ allowed_tools = tuple(t for t in allowed_tools_raw.split() if t)
301
+ elif isinstance(allowed_tools_raw, list) and all(
302
+ isinstance(tool, str) and tool.strip() for tool in allowed_tools_raw
303
+ ):
304
+ allowed_tools = tuple(allowed_tools_raw)
305
+ else:
306
+ raise SkillError(
307
+ "SKILL.md field 'allowed-tools' must be a string or list of non-empty strings"
308
+ )
309
+
310
+ metadata = SkillMetadata(
311
+ name=name,
312
+ description=description,
313
+ version=version,
314
+ license=license_str,
315
+ tags=tags,
316
+ compatibility=compatibility,
317
+ metadata_entries=metadata_entries,
318
+ allowed_tools=allowed_tools,
319
+ )
320
+
321
+ return metadata, body
322
+
323
+
324
+ def _load_references(refs_dir: Path) -> dict[str, str]:
325
+ """Load Markdown files recursively from ordinary files and directories.
326
+
327
+ Returns:
328
+ Dict mapping slash-separated paths relative to references/ to content
329
+ """
330
+ references: dict[str, str] = {}
331
+ pending = [refs_dir]
332
+ while pending:
333
+ directory = pending.pop()
334
+ try:
335
+ entries = sorted(directory.iterdir())
336
+ except OSError as exc:
337
+ raise SkillError(f"Cannot read references directory {directory}: {exc}") from exc
338
+ for file_path in entries:
339
+ try:
340
+ mode = file_path.stat().st_mode
341
+ except OSError as exc:
342
+ raise SkillError(f"Cannot inspect reference entry {file_path}: {exc}") from exc
343
+ if stat.S_ISDIR(mode):
344
+ pending.append(file_path)
345
+ continue
346
+ if not stat.S_ISREG(mode):
347
+ raise SkillError(f"Invalid references entry (must be a file): {file_path}")
348
+ if file_path.suffix.lower() != ".md":
349
+ raise SkillError(
350
+ f"Invalid reference file '{file_path}': only .md files are allowed"
351
+ )
352
+ try:
353
+ content = file_path.read_text(encoding="utf-8")
354
+ except UnicodeDecodeError as exc:
355
+ raise SkillError(f"Reference file must be valid UTF-8 text: {file_path}") from exc
356
+ except OSError as exc:
357
+ raise SkillError(f"Cannot read reference file {file_path}: {exc}") from exc
358
+ if not content.strip():
359
+ raise SkillError(f"Reference file must not be empty: {file_path}")
360
+ references[file_path.relative_to(refs_dir).as_posix()] = content
361
+
362
+ return dict(sorted(references.items()))
363
+
364
+
365
+ _SCRIPT_EXTENSIONS = frozenset({".py", ".sh", ".js", ".bash"})
366
+
367
+
368
+ def _load_scripts(scripts_dir: Path) -> dict[str, str]:
369
+ """Load executable scripts from scripts/ directory.
370
+
371
+ Supports .py, .sh, .js, and .bash files.
372
+
373
+ Returns:
374
+ Dict mapping filename to content
375
+ """
376
+ scripts: dict[str, str] = {}
377
+
378
+ for file_path in scripts_dir.iterdir():
379
+ if not file_path.is_file():
380
+ continue
381
+ if file_path.suffix.lower() not in _SCRIPT_EXTENSIONS:
382
+ continue
383
+ try:
384
+ content = file_path.read_text(encoding="utf-8")
385
+ except UnicodeDecodeError as exc:
386
+ raise SkillError(f"Script file must be valid UTF-8 text: {file_path}") from exc
387
+ if not content.strip():
388
+ raise SkillError(f"Script file must not be empty: {file_path}")
389
+ scripts[file_path.name] = content
390
+
391
+ return scripts
392
+
393
+
394
+ def load_skill(path: Path | str) -> Skill:
395
+ """Load a skill from a path.
396
+
397
+ Convenience function wrapping Skill.from_path(). This is the runner's local
398
+ skill validation, without SDK startup or model execution.
399
+
400
+ Args:
401
+ path: Path to skill directory or SKILL.md file
402
+
403
+ Returns:
404
+ Loaded Skill instance
405
+ """
406
+ return Skill.from_path(path)
@@ -0,0 +1,148 @@
1
+ """Bridge between Anthropic's skill-creator eval format and pytest-copilot.
2
+
3
+ Loads evals/evals.json from skill directories and converts them to test cases
4
+ that can be parametrized with pytest. Consumers define explicit output checks;
5
+ free-text expectations are descriptions, not automatically executed assertions.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import re
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+
16
+ @dataclass(slots=True, frozen=True)
17
+ class SkillEvalCase:
18
+ """A single eval case from skill-creator's evals/evals.json.
19
+
20
+ Attributes:
21
+ id: Numeric identifier from the eval file
22
+ name: Human-readable name derived from the prompt
23
+ prompt: The user prompt to send to the agent
24
+ expected_output: Description of expected result (for documentation)
25
+ expectations: Descriptions to translate into explicit output checks
26
+ files: Paths to files referenced by this eval case
27
+ """
28
+
29
+ id: int
30
+ name: str
31
+ prompt: str
32
+ expected_output: str | None = None
33
+ expectations: tuple[str, ...] = ()
34
+ files: tuple[str, ...] = ()
35
+
36
+
37
+ def load_skill_evals(skill_path: Path | str) -> list[SkillEvalCase]:
38
+ """Load eval cases from a skill-creator compatible skill directory.
39
+
40
+ Looks for evals/evals.json in the skill directory and parses it
41
+ into SkillEvalCase objects for use with pytest parametrize.
42
+
43
+ Args:
44
+ skill_path: Path to skill directory containing evals/evals.json,
45
+ or direct path to evals.json
46
+
47
+ Returns:
48
+ List of eval cases
49
+
50
+ Raises:
51
+ FileNotFoundError: If evals/evals.json doesn't exist
52
+ ValueError: If the JSON format is invalid
53
+ """
54
+ evals_file = _resolve_evals_path(skill_path)
55
+
56
+ raw = evals_file.read_text(encoding="utf-8")
57
+ try:
58
+ data = json.loads(raw)
59
+ except json.JSONDecodeError as exc:
60
+ raise ValueError(f"Invalid JSON in {evals_file}: {exc}") from exc
61
+
62
+ if not isinstance(data, dict):
63
+ raise ValueError(f"Expected JSON object in {evals_file}, got {type(data).__name__}")
64
+ if "evals" not in data:
65
+ raise ValueError(f"Missing required 'evals' array in {evals_file}")
66
+ if not isinstance(data["evals"], list):
67
+ raise ValueError(f"'evals' must be an array in {evals_file}")
68
+
69
+ cases: list[SkillEvalCase] = []
70
+ for i, entry in enumerate(data["evals"]):
71
+ cases.append(_parse_eval_entry(entry, i, evals_file))
72
+
73
+ return cases
74
+
75
+
76
+ def has_skill_evals(skill_path: Path | str) -> bool:
77
+ """Check if a skill directory has skill-creator evals."""
78
+ path = Path(skill_path)
79
+ if path.is_file():
80
+ path = path.parent
81
+ return (path / "evals" / "evals.json").exists()
82
+
83
+
84
+ def _resolve_evals_path(skill_path: Path | str) -> Path:
85
+ """Resolve skill_path to the evals/evals.json file."""
86
+ path = Path(skill_path)
87
+
88
+ if path.is_file() and path.name == "evals.json":
89
+ evals_file = path
90
+ elif path.is_dir():
91
+ evals_file = path / "evals" / "evals.json"
92
+ else:
93
+ raise FileNotFoundError(f"Invalid skill path: {path}")
94
+
95
+ if not evals_file.exists():
96
+ raise FileNotFoundError(f"evals/evals.json not found: {evals_file}")
97
+
98
+ return evals_file
99
+
100
+
101
+ def _parse_eval_entry(entry: dict, index: int, source: Path) -> SkillEvalCase:
102
+ """Parse a single eval entry from the JSON array."""
103
+ if not isinstance(entry, dict):
104
+ raise ValueError(f"Eval entry {index} must be an object in {source}")
105
+
106
+ # Required fields
107
+ eval_id = entry.get("id")
108
+ if eval_id is None:
109
+ raise ValueError(f"Eval entry {index} missing required field 'id' in {source}")
110
+
111
+ prompt = entry.get("prompt")
112
+ if not prompt or not isinstance(prompt, str):
113
+ raise ValueError(f"Eval entry {index} missing required field 'prompt' in {source}")
114
+
115
+ # Optional fields
116
+ expected_output = entry.get("expected_output")
117
+ expectations_raw = entry.get("expectations", [])
118
+ files_raw = entry.get("files", [])
119
+
120
+ if not isinstance(expectations_raw, list):
121
+ raise ValueError(f"Eval entry {index}: 'expectations' must be an array in {source}")
122
+ if not isinstance(files_raw, list):
123
+ raise ValueError(f"Eval entry {index}: 'files' must be an array in {source}")
124
+
125
+ name = _slugify_prompt(prompt, eval_id)
126
+
127
+ return SkillEvalCase(
128
+ id=int(eval_id),
129
+ name=name,
130
+ prompt=prompt,
131
+ expected_output=expected_output if isinstance(expected_output, str) else None,
132
+ expectations=tuple(str(e) for e in expectations_raw),
133
+ files=tuple(str(f) for f in files_raw),
134
+ )
135
+
136
+
137
+ def _slugify_prompt(prompt: str, eval_id: int) -> str:
138
+ """Generate a human-readable test name from a prompt.
139
+
140
+ Takes the first 50 characters, lowercases, replaces non-alphanumeric
141
+ with hyphens, and prepends the eval id.
142
+ """
143
+ slug = prompt[:50].lower().strip()
144
+ slug = re.sub(r"[^a-z0-9]+", "-", slug)
145
+ slug = slug.strip("-")
146
+ if not slug:
147
+ return f"eval-{eval_id}"
148
+ return f"{eval_id}-{slug}"
@@ -0,0 +1,78 @@
1
+ """Export eval results in skill-creator compatible grading.json format.
2
+
3
+ This only formats explicitly supplied booleans and evidence. It does not run,
4
+ interpret, or grade a task. Consumers own their ordinary output assertions.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import TYPE_CHECKING, Any
10
+
11
+ if TYPE_CHECKING:
12
+ from collections.abc import Sequence
13
+
14
+ from pytest_copilot.copilot.result import CopilotResult
15
+ from pytest_copilot.core.result import EvalResult
16
+
17
+
18
+ def export_grading(
19
+ result: EvalResult | CopilotResult,
20
+ expectations: Sequence[str],
21
+ expectation_results: Sequence[bool],
22
+ evidence: Sequence[str] | None = None,
23
+ ) -> dict[str, Any]:
24
+ """Export eval result as skill-creator compatible grading.json.
25
+
26
+ Args:
27
+ result: The eval result from a test run (EvalResult or CopilotResult)
28
+ expectations: The assertion texts that were checked
29
+ expectation_results: Whether each expectation passed
30
+ evidence: Optional evidence strings for each expectation
31
+
32
+ Returns:
33
+ Dict matching skill-creator's grading.json schema
34
+
35
+ Raises:
36
+ ValueError: If expectations and results have different lengths
37
+ """
38
+ if len(expectations) != len(expectation_results):
39
+ raise ValueError(
40
+ f"expectations ({len(expectations)}) and "
41
+ f"expectation_results ({len(expectation_results)}) must have the same length"
42
+ )
43
+ if evidence is not None and len(evidence) != len(expectations):
44
+ raise ValueError(
45
+ f"evidence ({len(evidence)}) must match expectations ({len(expectations)}) length"
46
+ )
47
+
48
+ expectation_entries: list[dict[str, Any]] = []
49
+ for i, (text, passed) in enumerate(zip(expectations, expectation_results, strict=True)):
50
+ entry: dict[str, Any] = {
51
+ "text": text,
52
+ "passed": passed,
53
+ }
54
+ if evidence is not None:
55
+ entry["evidence"] = evidence[i]
56
+ expectation_entries.append(entry)
57
+
58
+ passed_count = sum(1 for r in expectation_results if r)
59
+ total = len(expectation_results)
60
+ failed_count = total - passed_count
61
+
62
+ grading: dict[str, Any] = {
63
+ "expectations": expectation_entries,
64
+ "summary": {
65
+ "passed": passed_count,
66
+ "failed": failed_count,
67
+ "total": total,
68
+ "pass_rate": round(passed_count / total, 2) if total > 0 else 0.0,
69
+ },
70
+ "execution_metrics": {
71
+ "tool_calls": len(result.all_tool_calls),
72
+ "turns": len(result.turns),
73
+ "duration_ms": result.duration_ms,
74
+ "success": result.success,
75
+ },
76
+ }
77
+
78
+ return grading