skillbench-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skillbench/__init__.py +1 -0
- skillbench/assertions.py +250 -0
- skillbench/claude.py +277 -0
- skillbench/cli.py +323 -0
- skillbench/config.py +269 -0
- skillbench/dashboard.py +543 -0
- skillbench/progress.py +219 -0
- skillbench/runner.py +471 -0
- skillbench/scaffold.py +106 -0
- skillbench/skills.py +168 -0
- skillbench/workspace.py +124 -0
- skillbench_cli-0.1.0.dist-info/METADATA +218 -0
- skillbench_cli-0.1.0.dist-info/RECORD +16 -0
- skillbench_cli-0.1.0.dist-info/WHEEL +4 -0
- skillbench_cli-0.1.0.dist-info/entry_points.txt +2 -0
- skillbench_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
skillbench/scaffold.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""`skillbench init`: the project folder with defaults, sample tests and a sample repo."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from skillbench.config import CONFIG_FILE_NAME, PROJECT_DIR_NAME
|
|
6
|
+
|
|
7
|
+
_CONFIG = """\
|
|
8
|
+
# Defaults for every test in this folder. A test's own fields win over these,
|
|
9
|
+
# and command-line flags (--model, --repeat) win over both.
|
|
10
|
+
|
|
11
|
+
# model: claude-sonnet-5-5 # passed straight to claude
|
|
12
|
+
timeout_s: 600 # seconds before one turn is stopped
|
|
13
|
+
# repeat: 1 # run every test N times (each run uses real quota)
|
|
14
|
+
# min_pass_rate: 1.0 # with repeats: the share of runs that must pass
|
|
15
|
+
# fixture: . # the repo tests work in when they don't name one ("." = this repo)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
_GITIGNORE = """\
|
|
19
|
+
runs/
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
_SKILL_TEST = """\
|
|
23
|
+
# A skill test: skillbench invokes one skill directly and checks what it did.
|
|
24
|
+
# Run it with: skillbench skill
|
|
25
|
+
#
|
|
26
|
+
# skill: the skill's name, or its path in the repo. skillbench looks in .claude/skills/,
|
|
27
|
+
# skills/, .github/skills/ and any plugin, and builds the /command.
|
|
28
|
+
# input: the text after the /command.
|
|
29
|
+
# fixture: the repo the agent works in, copied to a temp folder first so nothing real is
|
|
30
|
+
# touched. "." is this repo, a path is a folder in it, a git URL is cloned:
|
|
31
|
+
# fixture: {git: https://github.com/you/app.git, ref: main}
|
|
32
|
+
name: hello-greets-by-name
|
|
33
|
+
skill: hello
|
|
34
|
+
input: the platform team
|
|
35
|
+
fixture: skillbench/fixtures/sample-repo
|
|
36
|
+
assert:
|
|
37
|
+
- skill_invoked: hello
|
|
38
|
+
- final_text_contains: Hello
|
|
39
|
+
- tool_not_called: Write
|
|
40
|
+
|
|
41
|
+
# Trigger tests: does the skill's description make the agent pick it? Each prompt is one
|
|
42
|
+
# short run that stops as soon as the agent decides. Uncomment to try:
|
|
43
|
+
# triggers:
|
|
44
|
+
# should: ["say hi to Ada for me"]
|
|
45
|
+
# should_not: ["what is 2 + 2?"]
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
_AGENT_TEST = """\
|
|
49
|
+
# An agent loop test: a normal prompt, nothing forced. Checks that the agent picked the right
|
|
50
|
+
# skills and tools on its own and finished the job.
|
|
51
|
+
# Run it with: skillbench agent
|
|
52
|
+
#
|
|
53
|
+
# answers: replies sent one per turn when the agent stops to ask something.
|
|
54
|
+
# assert: checks on what happened. tool_order passes if those tools were called in that
|
|
55
|
+
# order (other calls in between are fine).
|
|
56
|
+
name: hello-loop
|
|
57
|
+
prompt: Please greet the platform team for me.
|
|
58
|
+
fixture: skillbench/fixtures/sample-repo
|
|
59
|
+
answers: []
|
|
60
|
+
assert:
|
|
61
|
+
- skill_invoked: hello
|
|
62
|
+
- skill_not_invoked: deploy
|
|
63
|
+
- tool_order: [Skill]
|
|
64
|
+
- final_text_contains: Hello
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
_SAMPLE_REPO_README = """\
|
|
68
|
+
# sample-repo
|
|
69
|
+
|
|
70
|
+
A tiny repo for skillbench's sample tests. Point `fixture:` at your own repo instead.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
_SAMPLE_SKILL = """\
|
|
74
|
+
---
|
|
75
|
+
name: hello
|
|
76
|
+
description: Greet a person or group by name. Use when asked to greet someone or say hello.
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
Reply with exactly one line, `Hello, <name>!`, using the name you were given.
|
|
80
|
+
Do not create or edit any files.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
_FILES = {
|
|
84
|
+
CONFIG_FILE_NAME: _CONFIG,
|
|
85
|
+
".gitignore": _GITIGNORE,
|
|
86
|
+
"skills/example.yaml": _SKILL_TEST,
|
|
87
|
+
"agent/example.yaml": _AGENT_TEST,
|
|
88
|
+
"fixtures/sample-repo/README.md": _SAMPLE_REPO_README,
|
|
89
|
+
"fixtures/sample-repo/.claude/skills/hello/SKILL.md": _SAMPLE_SKILL,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def scaffold(parent: Path) -> tuple[list[Path], list[Path]]:
|
|
94
|
+
"""Write the project folder under `parent`, never overwriting a file. Returns (created,
|
|
95
|
+
already existing) paths."""
|
|
96
|
+
project_dir = parent / PROJECT_DIR_NAME
|
|
97
|
+
created, existing = [], []
|
|
98
|
+
for relative_path, content in _FILES.items():
|
|
99
|
+
path = project_dir / relative_path
|
|
100
|
+
if path.exists():
|
|
101
|
+
existing.append(path)
|
|
102
|
+
continue
|
|
103
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
path.write_text(content, encoding="utf-8")
|
|
105
|
+
created.append(path)
|
|
106
|
+
return created, existing
|
skillbench/skills.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Finding the skills and plugins in a workspace repo, picking the one a test names, and making
|
|
2
|
+
every skill visible to the agent CLI."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import shutil
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
_SKILL_FILE = "SKILL.md"
|
|
12
|
+
_PLUGIN_MANIFEST = Path(".claude-plugin") / "plugin.json"
|
|
13
|
+
_CLAUDE_SKILLS = Path(".claude") / "skills"
|
|
14
|
+
_GITHUB_SKILLS = Path(".github") / "skills"
|
|
15
|
+
_ROOT_SKILLS = Path("skills")
|
|
16
|
+
# Plugins are looked for at the repo root and this many folders below it (e.g. plugins/foo).
|
|
17
|
+
_PLUGIN_SEARCH_DEPTH = 3
|
|
18
|
+
_SKIPPED_DIRS = frozenset({".git", "node_modules", ".venv", "venv", "__pycache__", "dist"})
|
|
19
|
+
_GIT_EXCLUDE = Path(".git") / "info" / "exclude"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SkillNotFoundError(Exception):
|
|
23
|
+
"""The skill a test names is not in the workspace, or more than one skill matches."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class FoundSkill:
|
|
28
|
+
name: str # the skill's own name (frontmatter `name:`, else its folder name)
|
|
29
|
+
directory: Path # the folder holding SKILL.md
|
|
30
|
+
plugin_name: str | None = None # set for a skill that ships inside a plugin
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def qualified_name(self) -> str:
|
|
34
|
+
return f"{self.plugin_name}:{self.name}" if self.plugin_name else self.name
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def invocation(self) -> str:
|
|
38
|
+
"""The /command that runs this skill; a plugin's skills are prefixed with its name."""
|
|
39
|
+
return f"/{self.qualified_name}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class WorkspaceSkills:
|
|
44
|
+
plugin_dirs: list[Path]
|
|
45
|
+
skills: list[FoundSkill]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def discover(workspace: Path) -> WorkspaceSkills:
|
|
49
|
+
"""Every plugin and skill in the repo: in plugins, .claude/skills/, .github/skills/ and
|
|
50
|
+
skills/."""
|
|
51
|
+
plugin_dirs = _find_plugins(workspace)
|
|
52
|
+
skills = [
|
|
53
|
+
FoundSkill(_skill_name(skill_dir), skill_dir, _plugin_name(plugin_dir))
|
|
54
|
+
for plugin_dir in plugin_dirs
|
|
55
|
+
for skill_dir in _skill_dirs(plugin_dir / "skills")
|
|
56
|
+
]
|
|
57
|
+
loose_roots = [_CLAUDE_SKILLS, _GITHUB_SKILLS]
|
|
58
|
+
if workspace not in plugin_dirs: # a plugin at the root owns skills/
|
|
59
|
+
loose_roots.append(_ROOT_SKILLS)
|
|
60
|
+
skills += [
|
|
61
|
+
FoundSkill(_skill_name(skill_dir), skill_dir)
|
|
62
|
+
for root in loose_roots
|
|
63
|
+
for skill_dir in _skill_dirs(workspace / root)
|
|
64
|
+
]
|
|
65
|
+
return WorkspaceSkills(plugin_dirs=plugin_dirs, skills=skills)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def resolve(workspace: Path, found: WorkspaceSkills, wanted: str) -> FoundSkill:
|
|
69
|
+
"""The skill a test's `skill:` names: a path inside the repo, `plugin:skill`, or a bare
|
|
70
|
+
skill name. Raises SkillNotFoundError if none or several match."""
|
|
71
|
+
is_path = "/" in wanted or "\\" in wanted
|
|
72
|
+
if is_path:
|
|
73
|
+
target = (workspace / wanted).resolve()
|
|
74
|
+
matches = [skill for skill in found.skills if skill.directory.resolve() == target]
|
|
75
|
+
else:
|
|
76
|
+
matches = [skill for skill in found.skills if wanted in (skill.name, skill.qualified_name)]
|
|
77
|
+
if len(matches) == 1:
|
|
78
|
+
return matches[0]
|
|
79
|
+
if not matches:
|
|
80
|
+
known = ", ".join(sorted({skill.qualified_name for skill in found.skills})) or "none"
|
|
81
|
+
looked = f"{_CLAUDE_SKILLS}/, {_GITHUB_SKILLS}/, {_ROOT_SKILLS}/ and plugins"
|
|
82
|
+
raise SkillNotFoundError(
|
|
83
|
+
f"skill '{wanted}' not found in the workspace (looked in {looked}; found: {known})"
|
|
84
|
+
)
|
|
85
|
+
candidates = "; ".join(
|
|
86
|
+
f"{skill.qualified_name} at {skill.directory.relative_to(workspace).as_posix()}"
|
|
87
|
+
for skill in matches
|
|
88
|
+
)
|
|
89
|
+
raise SkillNotFoundError(
|
|
90
|
+
f"skill '{wanted}' matches more than one skill ({candidates}); "
|
|
91
|
+
"use plugin:skill or the skill's path"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def make_visible(workspace: Path, found: WorkspaceSkills) -> None:
|
|
96
|
+
"""Copy skills Claude wouldn't find by itself into .claude/skills/. Plugins are passed by
|
|
97
|
+
flag instead. Copies are git-excluded so the agent sees a clean tree."""
|
|
98
|
+
for skill in found.skills:
|
|
99
|
+
if skill.plugin_name is not None:
|
|
100
|
+
continue
|
|
101
|
+
relative = skill.directory.relative_to(workspace)
|
|
102
|
+
if relative.parent == _CLAUDE_SKILLS:
|
|
103
|
+
continue
|
|
104
|
+
target = workspace / _CLAUDE_SKILLS / skill.directory.name
|
|
105
|
+
if target.exists():
|
|
106
|
+
continue
|
|
107
|
+
shutil.copytree(skill.directory, target)
|
|
108
|
+
_git_exclude(workspace, (_CLAUDE_SKILLS / skill.directory.name).as_posix())
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _find_plugins(workspace: Path) -> list[Path]:
|
|
112
|
+
plugins = []
|
|
113
|
+
pending = [(workspace, 0)]
|
|
114
|
+
while pending:
|
|
115
|
+
directory, depth = pending.pop(0)
|
|
116
|
+
if (directory / _PLUGIN_MANIFEST).is_file():
|
|
117
|
+
plugins.append(directory)
|
|
118
|
+
if depth >= _PLUGIN_SEARCH_DEPTH:
|
|
119
|
+
continue
|
|
120
|
+
try:
|
|
121
|
+
children = sorted(entry for entry in directory.iterdir() if entry.is_dir())
|
|
122
|
+
except OSError:
|
|
123
|
+
continue
|
|
124
|
+
pending += [(child, depth + 1) for child in children if child.name not in _SKIPPED_DIRS]
|
|
125
|
+
return plugins
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _skill_dirs(root: Path) -> list[Path]:
|
|
129
|
+
if not root.is_dir():
|
|
130
|
+
return []
|
|
131
|
+
return sorted(entry for entry in root.iterdir() if (entry / _SKILL_FILE).is_file())
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _skill_name(skill_dir: Path) -> str:
|
|
135
|
+
frontmatter = _frontmatter(skill_dir / _SKILL_FILE)
|
|
136
|
+
name = frontmatter.get("name")
|
|
137
|
+
return name if isinstance(name, str) and name.strip() else skill_dir.name
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _frontmatter(skill_file: Path) -> dict:
|
|
141
|
+
text = skill_file.read_text(encoding="utf-8", errors="replace")
|
|
142
|
+
if not text.startswith("---"):
|
|
143
|
+
return {}
|
|
144
|
+
end = text.find("\n---", 3)
|
|
145
|
+
if end == -1:
|
|
146
|
+
return {}
|
|
147
|
+
try:
|
|
148
|
+
parsed = yaml.safe_load(text[3:end])
|
|
149
|
+
except yaml.YAMLError:
|
|
150
|
+
return {}
|
|
151
|
+
return parsed if isinstance(parsed, dict) else {}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _plugin_name(plugin_dir: Path) -> str:
|
|
155
|
+
try:
|
|
156
|
+
manifest = json.loads((plugin_dir / _PLUGIN_MANIFEST).read_text(encoding="utf-8"))
|
|
157
|
+
except (OSError, json.JSONDecodeError):
|
|
158
|
+
return plugin_dir.name
|
|
159
|
+
name = manifest.get("name") if isinstance(manifest, dict) else None
|
|
160
|
+
return name if isinstance(name, str) and name else plugin_dir.name
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _git_exclude(workspace: Path, pattern: str) -> None:
|
|
164
|
+
exclude_file = workspace / _GIT_EXCLUDE
|
|
165
|
+
if not exclude_file.parent.is_dir():
|
|
166
|
+
return
|
|
167
|
+
with exclude_file.open("a", encoding="utf-8") as handle:
|
|
168
|
+
handle.write(f"/{pattern}/\n")
|
skillbench/workspace.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Creating the fresh temp workspace each run happens in, from a local folder or a git repo."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import tempfile
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
# git:// ssh:// https:// file://, scp-style user@host:path, or anything ending in .git
|
|
12
|
+
_GIT_URL = re.compile(r"^(https?|ssh|git|file)://|^[\w.-]+@[\w.-]+:|\.git/?$")
|
|
13
|
+
# Never let git stop to ask for a password: a headless run would hang on it.
|
|
14
|
+
_GIT_ENV = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
|
|
15
|
+
# Windows' 260-character path limit trips over deep repos without this; harmless elsewhere.
|
|
16
|
+
_GIT_CONFIG = ["-c", "core.longpaths=true"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FixtureError(Exception):
|
|
20
|
+
"""The fixture could not be copied or cloned; the message says why."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class GitFixture:
|
|
25
|
+
url: str
|
|
26
|
+
ref: str | None = None # branch, tag or commit; None = the default branch
|
|
27
|
+
|
|
28
|
+
def __str__(self) -> str:
|
|
29
|
+
return f"{self.url}@{self.ref}" if self.ref else self.url
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
Fixture = Path | GitFixture
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def is_git_url(text: str) -> bool:
|
|
36
|
+
return bool(_GIT_URL.search(text))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def create_workspace(fixture: Fixture | None, exclude: Path) -> tuple[Path, str | None]:
|
|
40
|
+
"""Make a new temp dir (kept after the run) holding the fixture, with git initialised.
|
|
41
|
+
Returns it and, for a git fixture, the commit it was checked out at. `exclude` (the
|
|
42
|
+
skillbench folder) is never copied. Raises FixtureError."""
|
|
43
|
+
workspace = Path(tempfile.mkdtemp(prefix="skillbench-ws-"))
|
|
44
|
+
if isinstance(fixture, GitFixture):
|
|
45
|
+
return workspace, _clone(fixture, workspace)
|
|
46
|
+
if fixture is not None:
|
|
47
|
+
_copy_folder(fixture, workspace, exclude)
|
|
48
|
+
_git(["init", "--quiet"], workspace, "initialising the workspace")
|
|
49
|
+
return workspace, None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _clone(fixture: GitFixture, workspace: Path) -> str:
|
|
53
|
+
"""Clone into the workspace, check out the ref if any, and return the commit."""
|
|
54
|
+
_git(["clone", "--quiet", fixture.url, "."], workspace, f"cloning {fixture.url}")
|
|
55
|
+
if fixture.ref is not None:
|
|
56
|
+
doing = f"checking out ref '{fixture.ref}' of {fixture.url}"
|
|
57
|
+
_git(["checkout", "--quiet", fixture.ref], workspace, doing)
|
|
58
|
+
return _git(["rev-parse", "HEAD"], workspace, f"reading the commit of {fixture.url}").strip()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _copy_folder(source: Path, workspace: Path, exclude: Path) -> None:
|
|
62
|
+
"""Copy what git would see (tracked and untracked files, not ignored ones) when `source`
|
|
63
|
+
is in a git repo, otherwise everything; never the excluded folder, unless the fixture
|
|
64
|
+
lives inside it."""
|
|
65
|
+
excluded: Path | None = exclude.resolve()
|
|
66
|
+
if source.resolve().is_relative_to(excluded):
|
|
67
|
+
excluded = None
|
|
68
|
+
listed = _git_visible_files(source)
|
|
69
|
+
if listed is None:
|
|
70
|
+
ignore = None if excluded is None else _ignore_path(excluded)
|
|
71
|
+
shutil.copytree(source, workspace, dirs_exist_ok=True, ignore=ignore)
|
|
72
|
+
return
|
|
73
|
+
for relative in listed:
|
|
74
|
+
file_path = source / relative
|
|
75
|
+
is_excluded = excluded is not None and file_path.resolve().is_relative_to(excluded)
|
|
76
|
+
if file_path.is_file() and not is_excluded: # tracked files may be deleted locally
|
|
77
|
+
target = workspace / relative
|
|
78
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
79
|
+
shutil.copy2(file_path, target)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _git_visible_files(source: Path) -> list[str] | None:
|
|
83
|
+
"""Paths relative to `source`, or None if it isn't inside a git work tree."""
|
|
84
|
+
try:
|
|
85
|
+
listed = subprocess.run(
|
|
86
|
+
["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
|
|
87
|
+
cwd=source,
|
|
88
|
+
capture_output=True,
|
|
89
|
+
env=_GIT_ENV,
|
|
90
|
+
check=False,
|
|
91
|
+
)
|
|
92
|
+
except OSError:
|
|
93
|
+
return None
|
|
94
|
+
if listed.returncode != 0:
|
|
95
|
+
return None
|
|
96
|
+
return [path for path in listed.stdout.decode("utf-8").split("\0") if path]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _ignore_path(excluded: Path):
|
|
100
|
+
"""A copytree `ignore` that skips one folder."""
|
|
101
|
+
|
|
102
|
+
def ignore(directory: str, names: list[str]) -> list[str]:
|
|
103
|
+
return [name for name in names if (Path(directory) / name).resolve() == excluded]
|
|
104
|
+
|
|
105
|
+
return ignore
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _git(args: list[str], cwd: Path, doing: str) -> str:
|
|
109
|
+
"""Run git and return its stdout. Raises FixtureError saying what was being done."""
|
|
110
|
+
try:
|
|
111
|
+
completed = subprocess.run(
|
|
112
|
+
["git", *_GIT_CONFIG, *args],
|
|
113
|
+
cwd=cwd,
|
|
114
|
+
stdin=subprocess.DEVNULL,
|
|
115
|
+
capture_output=True,
|
|
116
|
+
env=_GIT_ENV,
|
|
117
|
+
check=False,
|
|
118
|
+
)
|
|
119
|
+
except OSError as error:
|
|
120
|
+
raise FixtureError(f"{doing}: git could not be run ({error})") from error
|
|
121
|
+
if completed.returncode != 0:
|
|
122
|
+
detail = completed.stderr.decode("utf-8", errors="replace").strip()
|
|
123
|
+
raise FixtureError(f"{doing} failed: {detail or f'git exited {completed.returncode}'}")
|
|
124
|
+
return completed.stdout.decode("utf-8", errors="replace")
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: skillbench-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Test your Claude Code skills and agent workflows, the way you test code.
|
|
5
|
+
Project-URL: Homepage, https://github.com/adamsaul314/skillbench
|
|
6
|
+
Project-URL: Issues, https://github.com/adamsaul314/skillbench/issues
|
|
7
|
+
Author: Adam Saul
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agents,claude,claude-code,evals,skills,testing
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: pyyaml>=6.0
|
|
19
|
+
Requires-Dist: rich>=13.7
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# skillbench
|
|
23
|
+
|
|
24
|
+
Test your Claude Code skills and agent workflows, the way you test code.
|
|
25
|
+
|
|
26
|
+
You write a small YAML file that says *"send this to Claude, then check these things"*. skillbench runs Claude Code for real on a copy of your repo, records everything it did, and tells you **pass** or **fail**.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## 1. Install
|
|
31
|
+
|
|
32
|
+
You need Python 3.11+, git, and Claude Code installed and logged in (check with `claude --version`). Then install with whichever tool you use:
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
pip install skillbench-cli
|
|
36
|
+
pipx install skillbench-cli
|
|
37
|
+
uv tool install skillbench-cli
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The command is `skillbench` either way. To try it without installing: `uvx --from skillbench-cli skillbench init`.
|
|
41
|
+
|
|
42
|
+
## 2. Try it (2 minutes)
|
|
43
|
+
|
|
44
|
+
In your repo:
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
skillbench init # creates a skillbench/ folder with two example tests
|
|
48
|
+
skillbench skill # runs the skill test
|
|
49
|
+
skillbench agent # runs the agent test
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
You'll see something like:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
[1/1] hello-greets-by-name default model
|
|
56
|
+
hello-greets-by-name: pass in 2.3s
|
|
57
|
+
dashboard: skillbench/runs/index.html
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Open `skillbench/runs/index.html` in your browser to see exactly what Claude did: every message, tool call and check.
|
|
61
|
+
|
|
62
|
+
That's it. The rest of this page shows how to write your own tests.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 3. Two kinds of test
|
|
67
|
+
|
|
68
|
+
| | **Skill test** | **Agent test** |
|
|
69
|
+
|---|---|---|
|
|
70
|
+
| Question it answers | "When my skill runs, does it do the right thing?" | "Given a normal request, does Claude pick the right skills and get the job done?" |
|
|
71
|
+
| Lives in | `skillbench/skills/` | `skillbench/agent/` |
|
|
72
|
+
| Run with | `skillbench skill` | `skillbench agent` |
|
|
73
|
+
| You write | `skill:` and `input:` | `prompt:` |
|
|
74
|
+
|
|
75
|
+
### A skill test
|
|
76
|
+
|
|
77
|
+
`skillbench/skills/grilling.yaml`
|
|
78
|
+
|
|
79
|
+
```yaml
|
|
80
|
+
skill: grilling # your skill's name
|
|
81
|
+
input: an idea for a todo app # what to send it
|
|
82
|
+
fixture: . # run it on a copy of this repo
|
|
83
|
+
|
|
84
|
+
assert:
|
|
85
|
+
- final_text_contains: "Q1"
|
|
86
|
+
- tool_not_called: Write
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
skillbench finds the skill wherever it lives in your repo (`.claude/skills/`, `skills/`, `.github/skills/` or a plugin) and runs it as `/grilling an idea for a todo app`.
|
|
90
|
+
|
|
91
|
+
### An agent test
|
|
92
|
+
|
|
93
|
+
`skillbench/agent/csv-export.yaml`
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
prompt: add CSV export to the report page
|
|
97
|
+
fixture: .
|
|
98
|
+
|
|
99
|
+
answers: # replies, sent one at a time when Claude asks something
|
|
100
|
+
- Yes to all your recommendations.
|
|
101
|
+
- approved
|
|
102
|
+
|
|
103
|
+
assert:
|
|
104
|
+
- skill_invoked: planner
|
|
105
|
+
- tool_order: [Read, Edit, Bash]
|
|
106
|
+
- command_succeeds: python -m pytest -q
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Nothing is forced here. Claude sees every skill in your repo and decides for itself, and the test checks what it chose.
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## 4. Checks you can make
|
|
114
|
+
|
|
115
|
+
Put any of these under `assert:`.
|
|
116
|
+
|
|
117
|
+
| Check | Passes when |
|
|
118
|
+
|---|---|
|
|
119
|
+
| `skill_invoked: grilling` | Claude used that skill |
|
|
120
|
+
| `skill_not_invoked: grilling` | Claude didn't use that skill |
|
|
121
|
+
| `tool_order: [Read, Edit]` | Claude used these tools in this order (other tools in between are fine) |
|
|
122
|
+
| `tool_not_called: Write` | Claude never used that tool |
|
|
123
|
+
| `final_text_contains: "Done"` | Claude's last message contains the text |
|
|
124
|
+
| `file_contains: {path: notes.md, text: "approved"}` | That file exists afterwards and contains the text |
|
|
125
|
+
| `command_succeeds: python -m pytest -q` | The command runs successfully in the workspace afterwards |
|
|
126
|
+
|
|
127
|
+
A test passes when every check passes.
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 5. Which repo Claude works in (`fixture:`)
|
|
132
|
+
|
|
133
|
+
Claude always works on a **copy**, so your real files are never touched.
|
|
134
|
+
|
|
135
|
+
| `fixture:` | Claude gets |
|
|
136
|
+
|---|---|
|
|
137
|
+
| *(leave it out)* | An empty folder |
|
|
138
|
+
| `.` | A copy of your repo, including work you haven't committed |
|
|
139
|
+
| `some/folder` | A copy of that folder |
|
|
140
|
+
| `https://github.com/you/app.git` | A fresh clone |
|
|
141
|
+
| `{git: https://github.com/you/app.git, ref: v1.2}` | A clone at that branch, tag or commit |
|
|
142
|
+
|
|
143
|
+
Paths are relative to your repo root. The copy is kept after the run so you can look inside, and its path is in the dashboard.
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## 6. Running tests
|
|
148
|
+
|
|
149
|
+
```
|
|
150
|
+
skillbench skill # all skill tests
|
|
151
|
+
skillbench skill grilling # just the tests for one skill (or one test by name)
|
|
152
|
+
skillbench agent csv-export # one agent test
|
|
153
|
+
skillbench agent --repeat 5 # run each test 5 times
|
|
154
|
+
skillbench agent --model claude-sonnet-5-5
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
While it runs you'll see the current test, what Claude is doing, and roughly how long is left.
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## 7. More options
|
|
162
|
+
|
|
163
|
+
You won't need these to start.
|
|
164
|
+
|
|
165
|
+
**Check a skill gets picked on its own.** List prompts that should and shouldn't trigger it. Each runs briefly and stops as soon as Claude decides:
|
|
166
|
+
|
|
167
|
+
```yaml
|
|
168
|
+
skill: grilling
|
|
169
|
+
triggers:
|
|
170
|
+
should: ["stress-test my plan"]
|
|
171
|
+
should_not: ["fix this typo"]
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
**Answer Claude yourself.** Add `interactive: true` and skillbench asks *you* in the terminal whenever Claude is waiting for a reply. Press Enter on an empty line to finish.
|
|
175
|
+
|
|
176
|
+
**Check for flakiness.** Claude doesn't do exactly the same thing every time. `--repeat 5` (or `repeat: 5` in the test) runs it 5 times and shows how often each check passed. Add `min_pass_rate: 0.8` to pass when 4 out of 5 runs pass.
|
|
177
|
+
|
|
178
|
+
**Set defaults for every test** in `skillbench/skillbench.yaml`:
|
|
179
|
+
|
|
180
|
+
```yaml
|
|
181
|
+
model: claude-sonnet-5-5
|
|
182
|
+
timeout_s: 600 # seconds before a turn is stopped
|
|
183
|
+
fixture: .
|
|
184
|
+
repeat: 1
|
|
185
|
+
min_pass_rate: 1.0
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
A test's own settings win over these, and command-line flags win over both.
|
|
189
|
+
|
|
190
|
+
**All test fields:**
|
|
191
|
+
|
|
192
|
+
| Field | Meaning |
|
|
193
|
+
|---|---|
|
|
194
|
+
| `skill` / `input` | Skill tests: which skill, and what to send it |
|
|
195
|
+
| `prompt` | Agent tests: the first message |
|
|
196
|
+
| `answers` | Replies sent one per turn |
|
|
197
|
+
| `interactive` | Ask you in the terminal for replies |
|
|
198
|
+
| `triggers` | Prompts that should / shouldn't pick the skill |
|
|
199
|
+
| `fixture` | Which repo Claude works in |
|
|
200
|
+
| `assert` | The checks |
|
|
201
|
+
| `name` | A display name (defaults to the file name) |
|
|
202
|
+
| `timeout_s`, `repeat`, `min_pass_rate` | As above |
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## Good to know
|
|
207
|
+
|
|
208
|
+
- **It costs real usage.** Every run is a real Claude session. Start with skill tests; long agent tests take longer and cost more.
|
|
209
|
+
- **Your personal setup is ignored.** Only the skills and plugins in the repo being tested are loaded, so results don't depend on whose machine runs them.
|
|
210
|
+
- **Only test repos you trust.** Claude has full tool access inside the copy, and `command_succeeds` runs commands there.
|
|
211
|
+
- **`command_succeeds` uses your system shell** (`sh` on Mac/Linux, `cmd` on Windows), so keep commands simple if you share tests across systems.
|
|
212
|
+
- **Results** are in `skillbench/runs/`, which is git-ignored. Each run has a `run.json` and Claude's raw output.
|
|
213
|
+
|
|
214
|
+
| Status | Meaning |
|
|
215
|
+
|---|---|
|
|
216
|
+
| `pass` | Every check passed |
|
|
217
|
+
| `fail` | Claude finished, but a check failed |
|
|
218
|
+
| `error` | Claude crashed or timed out, or the skill or fixture couldn't be found |
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
skillbench/__init__.py,sha256=xuhAlrEiq6nBnACBFYL-pqeeeQOC_KI__zriGl_y5HQ,51
|
|
2
|
+
skillbench/assertions.py,sha256=QPeDuNGfpldW826wbS9qxsmuNeB-AwFXxWgz_8TcZ_c,9563
|
|
3
|
+
skillbench/claude.py,sha256=xUrSihI1UjdC734pEWtnEnvB6BTg4MxaKyKN5ZlgbuA,10770
|
|
4
|
+
skillbench/cli.py,sha256=JakEyZtrryVAQJXBy0ocgnWHf65Vtooqr6afvuWDGa0,12389
|
|
5
|
+
skillbench/config.py,sha256=MPUvz1wyWUpZw_C-YI0p7CNa_0HW7Q9NFxjq1kqHmAQ,9947
|
|
6
|
+
skillbench/dashboard.py,sha256=HUBG61uZ_qwNtwXdCMBnwp2yVjnY99BAY5ylPaN1MhA,24329
|
|
7
|
+
skillbench/progress.py,sha256=0cc1CQM5ntDkY8rIfpEODmyw2UfjeHzgPaN-9yItrg4,7685
|
|
8
|
+
skillbench/runner.py,sha256=vB_CuQ3qV_owufJAf88d5vK5upZ5ZZm3jpaOUHWyb9U,18305
|
|
9
|
+
skillbench/scaffold.py,sha256=WgB-U8HoV75bkRDcPe_mCj5qUmjGCnQcmFkuK8HqRW8,3841
|
|
10
|
+
skillbench/skills.py,sha256=Qio0am02G6tPg5dCNEiHcUMbvBjsgvZPqoXjbGqJHb8,6417
|
|
11
|
+
skillbench/workspace.py,sha256=F2WCfvlV7JuEfvWkJ2r0zy6BO8KTcSE2725fSwQjdlg,4881
|
|
12
|
+
skillbench_cli-0.1.0.dist-info/METADATA,sha256=26gJCU48lMHOvvI2CPMhWAuVe3aFicIjgUhwgu6ALPE,7503
|
|
13
|
+
skillbench_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
14
|
+
skillbench_cli-0.1.0.dist-info/entry_points.txt,sha256=YtKWfB9WaELMltae7MJnMmUL37oJXoyDWr3uOb2P4FU,51
|
|
15
|
+
skillbench_cli-0.1.0.dist-info/licenses/LICENSE,sha256=gABCeZ1x3voBu99KF6B55f6HGNdEiOpMKi5ZsBAveLg,1066
|
|
16
|
+
skillbench_cli-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Adam Saul
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|