skillbench-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
skillbench/scaffold.py ADDED
@@ -0,0 +1,106 @@
1
+ """`skillbench init`: the project folder with defaults, sample tests and a sample repo."""
2
+
3
+ from pathlib import Path
4
+
5
+ from skillbench.config import CONFIG_FILE_NAME, PROJECT_DIR_NAME
6
+
7
+ _CONFIG = """\
8
+ # Defaults for every test in this folder. A test's own fields win over these,
9
+ # and command-line flags (--model, --repeat) win over both.
10
+
11
+ # model: claude-sonnet-5-5 # passed straight to claude
12
+ timeout_s: 600 # seconds before one turn is stopped
13
+ # repeat: 1 # run every test N times (each run uses real quota)
14
+ # min_pass_rate: 1.0 # with repeats: the share of runs that must pass
15
+ # fixture: . # the repo tests work in when they don't name one ("." = this repo)
16
+ """
17
+
18
+ _GITIGNORE = """\
19
+ runs/
20
+ """
21
+
22
+ _SKILL_TEST = """\
23
+ # A skill test: skillbench invokes one skill directly and checks what it did.
24
+ # Run it with: skillbench skill
25
+ #
26
+ # skill: the skill's name, or its path in the repo. skillbench looks in .claude/skills/,
27
+ # skills/, .github/skills/ and any plugin, and builds the /command.
28
+ # input: the text after the /command.
29
+ # fixture: the repo the agent works in, copied to a temp folder first so nothing real is
30
+ # touched. "." is this repo, a path is a folder in it, a git URL is cloned:
31
+ # fixture: {git: https://github.com/you/app.git, ref: main}
32
+ name: hello-greets-by-name
33
+ skill: hello
34
+ input: the platform team
35
+ fixture: skillbench/fixtures/sample-repo
36
+ assert:
37
+ - skill_invoked: hello
38
+ - final_text_contains: Hello
39
+ - tool_not_called: Write
40
+
41
+ # Trigger tests: does the skill's description make the agent pick it? Each prompt is one
42
+ # short run that stops as soon as the agent decides. Uncomment to try:
43
+ # triggers:
44
+ # should: ["say hi to Ada for me"]
45
+ # should_not: ["what is 2 + 2?"]
46
+ """
47
+
48
+ _AGENT_TEST = """\
49
+ # An agent loop test: a normal prompt, nothing forced. Checks that the agent picked the right
50
+ # skills and tools on its own and finished the job.
51
+ # Run it with: skillbench agent
52
+ #
53
+ # answers: replies sent one per turn when the agent stops to ask something.
54
+ # assert: checks on what happened. tool_order passes if those tools were called in that
55
+ # order (other calls in between are fine).
56
+ name: hello-loop
57
+ prompt: Please greet the platform team for me.
58
+ fixture: skillbench/fixtures/sample-repo
59
+ answers: []
60
+ assert:
61
+ - skill_invoked: hello
62
+ - skill_not_invoked: deploy
63
+ - tool_order: [Skill]
64
+ - final_text_contains: Hello
65
+ """
66
+
67
+ _SAMPLE_REPO_README = """\
68
+ # sample-repo
69
+
70
+ A tiny repo for skillbench's sample tests. Point `fixture:` at your own repo instead.
71
+ """
72
+
73
+ _SAMPLE_SKILL = """\
74
+ ---
75
+ name: hello
76
+ description: Greet a person or group by name. Use when asked to greet someone or say hello.
77
+ ---
78
+
79
+ Reply with exactly one line, `Hello, <name>!`, using the name you were given.
80
+ Do not create or edit any files.
81
+ """
82
+
83
+ _FILES = {
84
+ CONFIG_FILE_NAME: _CONFIG,
85
+ ".gitignore": _GITIGNORE,
86
+ "skills/example.yaml": _SKILL_TEST,
87
+ "agent/example.yaml": _AGENT_TEST,
88
+ "fixtures/sample-repo/README.md": _SAMPLE_REPO_README,
89
+ "fixtures/sample-repo/.claude/skills/hello/SKILL.md": _SAMPLE_SKILL,
90
+ }
91
+
92
+
93
+ def scaffold(parent: Path) -> tuple[list[Path], list[Path]]:
94
+ """Write the project folder under `parent`, never overwriting a file. Returns (created,
95
+ already existing) paths."""
96
+ project_dir = parent / PROJECT_DIR_NAME
97
+ created, existing = [], []
98
+ for relative_path, content in _FILES.items():
99
+ path = project_dir / relative_path
100
+ if path.exists():
101
+ existing.append(path)
102
+ continue
103
+ path.parent.mkdir(parents=True, exist_ok=True)
104
+ path.write_text(content, encoding="utf-8")
105
+ created.append(path)
106
+ return created, existing
skillbench/skills.py ADDED
@@ -0,0 +1,168 @@
1
+ """Finding the skills and plugins in a workspace repo, picking the one a test names, and making
2
+ every skill visible to the agent CLI."""
3
+
4
+ import json
5
+ import shutil
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+
9
+ import yaml
10
+
11
+ _SKILL_FILE = "SKILL.md"
12
+ _PLUGIN_MANIFEST = Path(".claude-plugin") / "plugin.json"
13
+ _CLAUDE_SKILLS = Path(".claude") / "skills"
14
+ _GITHUB_SKILLS = Path(".github") / "skills"
15
+ _ROOT_SKILLS = Path("skills")
16
+ # Plugins are looked for at the repo root and this many folders below it (e.g. plugins/foo).
17
+ _PLUGIN_SEARCH_DEPTH = 3
18
+ _SKIPPED_DIRS = frozenset({".git", "node_modules", ".venv", "venv", "__pycache__", "dist"})
19
+ _GIT_EXCLUDE = Path(".git") / "info" / "exclude"
20
+
21
+
22
+ class SkillNotFoundError(Exception):
23
+ """The skill a test names is not in the workspace, or more than one skill matches."""
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class FoundSkill:
28
+ name: str # the skill's own name (frontmatter `name:`, else its folder name)
29
+ directory: Path # the folder holding SKILL.md
30
+ plugin_name: str | None = None # set for a skill that ships inside a plugin
31
+
32
+ @property
33
+ def qualified_name(self) -> str:
34
+ return f"{self.plugin_name}:{self.name}" if self.plugin_name else self.name
35
+
36
+ @property
37
+ def invocation(self) -> str:
38
+ """The /command that runs this skill; a plugin's skills are prefixed with its name."""
39
+ return f"/{self.qualified_name}"
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class WorkspaceSkills:
44
+ plugin_dirs: list[Path]
45
+ skills: list[FoundSkill]
46
+
47
+
48
+ def discover(workspace: Path) -> WorkspaceSkills:
49
+ """Every plugin and skill in the repo: in plugins, .claude/skills/, .github/skills/ and
50
+ skills/."""
51
+ plugin_dirs = _find_plugins(workspace)
52
+ skills = [
53
+ FoundSkill(_skill_name(skill_dir), skill_dir, _plugin_name(plugin_dir))
54
+ for plugin_dir in plugin_dirs
55
+ for skill_dir in _skill_dirs(plugin_dir / "skills")
56
+ ]
57
+ loose_roots = [_CLAUDE_SKILLS, _GITHUB_SKILLS]
58
+ if workspace not in plugin_dirs: # a plugin at the root owns skills/
59
+ loose_roots.append(_ROOT_SKILLS)
60
+ skills += [
61
+ FoundSkill(_skill_name(skill_dir), skill_dir)
62
+ for root in loose_roots
63
+ for skill_dir in _skill_dirs(workspace / root)
64
+ ]
65
+ return WorkspaceSkills(plugin_dirs=plugin_dirs, skills=skills)
66
+
67
+
68
+ def resolve(workspace: Path, found: WorkspaceSkills, wanted: str) -> FoundSkill:
69
+ """The skill a test's `skill:` names: a path inside the repo, `plugin:skill`, or a bare
70
+ skill name. Raises SkillNotFoundError if none or several match."""
71
+ is_path = "/" in wanted or "\\" in wanted
72
+ if is_path:
73
+ target = (workspace / wanted).resolve()
74
+ matches = [skill for skill in found.skills if skill.directory.resolve() == target]
75
+ else:
76
+ matches = [skill for skill in found.skills if wanted in (skill.name, skill.qualified_name)]
77
+ if len(matches) == 1:
78
+ return matches[0]
79
+ if not matches:
80
+ known = ", ".join(sorted({skill.qualified_name for skill in found.skills})) or "none"
81
+ looked = f"{_CLAUDE_SKILLS}/, {_GITHUB_SKILLS}/, {_ROOT_SKILLS}/ and plugins"
82
+ raise SkillNotFoundError(
83
+ f"skill '{wanted}' not found in the workspace (looked in {looked}; found: {known})"
84
+ )
85
+ candidates = "; ".join(
86
+ f"{skill.qualified_name} at {skill.directory.relative_to(workspace).as_posix()}"
87
+ for skill in matches
88
+ )
89
+ raise SkillNotFoundError(
90
+ f"skill '{wanted}' matches more than one skill ({candidates}); "
91
+ "use plugin:skill or the skill's path"
92
+ )
93
+
94
+
95
+ def make_visible(workspace: Path, found: WorkspaceSkills) -> None:
96
+ """Copy skills Claude wouldn't find by itself into .claude/skills/. Plugins are passed by
97
+ flag instead. Copies are git-excluded so the agent sees a clean tree."""
98
+ for skill in found.skills:
99
+ if skill.plugin_name is not None:
100
+ continue
101
+ relative = skill.directory.relative_to(workspace)
102
+ if relative.parent == _CLAUDE_SKILLS:
103
+ continue
104
+ target = workspace / _CLAUDE_SKILLS / skill.directory.name
105
+ if target.exists():
106
+ continue
107
+ shutil.copytree(skill.directory, target)
108
+ _git_exclude(workspace, (_CLAUDE_SKILLS / skill.directory.name).as_posix())
109
+
110
+
111
+ def _find_plugins(workspace: Path) -> list[Path]:
112
+ plugins = []
113
+ pending = [(workspace, 0)]
114
+ while pending:
115
+ directory, depth = pending.pop(0)
116
+ if (directory / _PLUGIN_MANIFEST).is_file():
117
+ plugins.append(directory)
118
+ if depth >= _PLUGIN_SEARCH_DEPTH:
119
+ continue
120
+ try:
121
+ children = sorted(entry for entry in directory.iterdir() if entry.is_dir())
122
+ except OSError:
123
+ continue
124
+ pending += [(child, depth + 1) for child in children if child.name not in _SKIPPED_DIRS]
125
+ return plugins
126
+
127
+
128
+ def _skill_dirs(root: Path) -> list[Path]:
129
+ if not root.is_dir():
130
+ return []
131
+ return sorted(entry for entry in root.iterdir() if (entry / _SKILL_FILE).is_file())
132
+
133
+
134
+ def _skill_name(skill_dir: Path) -> str:
135
+ frontmatter = _frontmatter(skill_dir / _SKILL_FILE)
136
+ name = frontmatter.get("name")
137
+ return name if isinstance(name, str) and name.strip() else skill_dir.name
138
+
139
+
140
+ def _frontmatter(skill_file: Path) -> dict:
141
+ text = skill_file.read_text(encoding="utf-8", errors="replace")
142
+ if not text.startswith("---"):
143
+ return {}
144
+ end = text.find("\n---", 3)
145
+ if end == -1:
146
+ return {}
147
+ try:
148
+ parsed = yaml.safe_load(text[3:end])
149
+ except yaml.YAMLError:
150
+ return {}
151
+ return parsed if isinstance(parsed, dict) else {}
152
+
153
+
154
+ def _plugin_name(plugin_dir: Path) -> str:
155
+ try:
156
+ manifest = json.loads((plugin_dir / _PLUGIN_MANIFEST).read_text(encoding="utf-8"))
157
+ except (OSError, json.JSONDecodeError):
158
+ return plugin_dir.name
159
+ name = manifest.get("name") if isinstance(manifest, dict) else None
160
+ return name if isinstance(name, str) and name else plugin_dir.name
161
+
162
+
163
+ def _git_exclude(workspace: Path, pattern: str) -> None:
164
+ exclude_file = workspace / _GIT_EXCLUDE
165
+ if not exclude_file.parent.is_dir():
166
+ return
167
+ with exclude_file.open("a", encoding="utf-8") as handle:
168
+ handle.write(f"/{pattern}/\n")
@@ -0,0 +1,124 @@
1
+ """Creating the fresh temp workspace each run happens in, from a local folder or a git repo."""
2
+
3
+ import os
4
+ import re
5
+ import shutil
6
+ import subprocess
7
+ import tempfile
8
+ from dataclasses import dataclass
9
+ from pathlib import Path
10
+
11
+ # git:// ssh:// https:// file://, scp-style user@host:path, or anything ending in .git
12
+ _GIT_URL = re.compile(r"^(https?|ssh|git|file)://|^[\w.-]+@[\w.-]+:|\.git/?$")
13
+ # Never let git stop to ask for a password: a headless run would hang on it.
14
+ _GIT_ENV = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
15
+ # Windows' 260-character path limit trips over deep repos without this; harmless elsewhere.
16
+ _GIT_CONFIG = ["-c", "core.longpaths=true"]
17
+
18
+
19
+ class FixtureError(Exception):
20
+ """The fixture could not be copied or cloned; the message says why."""
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class GitFixture:
25
+ url: str
26
+ ref: str | None = None # branch, tag or commit; None = the default branch
27
+
28
+ def __str__(self) -> str:
29
+ return f"{self.url}@{self.ref}" if self.ref else self.url
30
+
31
+
32
+ Fixture = Path | GitFixture
33
+
34
+
35
+ def is_git_url(text: str) -> bool:
36
+ return bool(_GIT_URL.search(text))
37
+
38
+
39
+ def create_workspace(fixture: Fixture | None, exclude: Path) -> tuple[Path, str | None]:
40
+ """Make a new temp dir (kept after the run) holding the fixture, with git initialised.
41
+ Returns it and, for a git fixture, the commit it was checked out at. `exclude` (the
42
+ skillbench folder) is never copied. Raises FixtureError."""
43
+ workspace = Path(tempfile.mkdtemp(prefix="skillbench-ws-"))
44
+ if isinstance(fixture, GitFixture):
45
+ return workspace, _clone(fixture, workspace)
46
+ if fixture is not None:
47
+ _copy_folder(fixture, workspace, exclude)
48
+ _git(["init", "--quiet"], workspace, "initialising the workspace")
49
+ return workspace, None
50
+
51
+
52
+ def _clone(fixture: GitFixture, workspace: Path) -> str:
53
+ """Clone into the workspace, check out the ref if any, and return the commit."""
54
+ _git(["clone", "--quiet", fixture.url, "."], workspace, f"cloning {fixture.url}")
55
+ if fixture.ref is not None:
56
+ doing = f"checking out ref '{fixture.ref}' of {fixture.url}"
57
+ _git(["checkout", "--quiet", fixture.ref], workspace, doing)
58
+ return _git(["rev-parse", "HEAD"], workspace, f"reading the commit of {fixture.url}").strip()
59
+
60
+
61
+ def _copy_folder(source: Path, workspace: Path, exclude: Path) -> None:
62
+ """Copy what git would see (tracked and untracked files, not ignored ones) when `source`
63
+ is in a git repo, otherwise everything; never the excluded folder, unless the fixture
64
+ lives inside it."""
65
+ excluded: Path | None = exclude.resolve()
66
+ if source.resolve().is_relative_to(excluded):
67
+ excluded = None
68
+ listed = _git_visible_files(source)
69
+ if listed is None:
70
+ ignore = None if excluded is None else _ignore_path(excluded)
71
+ shutil.copytree(source, workspace, dirs_exist_ok=True, ignore=ignore)
72
+ return
73
+ for relative in listed:
74
+ file_path = source / relative
75
+ is_excluded = excluded is not None and file_path.resolve().is_relative_to(excluded)
76
+ if file_path.is_file() and not is_excluded: # tracked files may be deleted locally
77
+ target = workspace / relative
78
+ target.parent.mkdir(parents=True, exist_ok=True)
79
+ shutil.copy2(file_path, target)
80
+
81
+
82
+ def _git_visible_files(source: Path) -> list[str] | None:
83
+ """Paths relative to `source`, or None if it isn't inside a git work tree."""
84
+ try:
85
+ listed = subprocess.run(
86
+ ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
87
+ cwd=source,
88
+ capture_output=True,
89
+ env=_GIT_ENV,
90
+ check=False,
91
+ )
92
+ except OSError:
93
+ return None
94
+ if listed.returncode != 0:
95
+ return None
96
+ return [path for path in listed.stdout.decode("utf-8").split("\0") if path]
97
+
98
+
99
+ def _ignore_path(excluded: Path):
100
+ """A copytree `ignore` that skips one folder."""
101
+
102
+ def ignore(directory: str, names: list[str]) -> list[str]:
103
+ return [name for name in names if (Path(directory) / name).resolve() == excluded]
104
+
105
+ return ignore
106
+
107
+
108
+ def _git(args: list[str], cwd: Path, doing: str) -> str:
109
+ """Run git and return its stdout. Raises FixtureError saying what was being done."""
110
+ try:
111
+ completed = subprocess.run(
112
+ ["git", *_GIT_CONFIG, *args],
113
+ cwd=cwd,
114
+ stdin=subprocess.DEVNULL,
115
+ capture_output=True,
116
+ env=_GIT_ENV,
117
+ check=False,
118
+ )
119
+ except OSError as error:
120
+ raise FixtureError(f"{doing}: git could not be run ({error})") from error
121
+ if completed.returncode != 0:
122
+ detail = completed.stderr.decode("utf-8", errors="replace").strip()
123
+ raise FixtureError(f"{doing} failed: {detail or f'git exited {completed.returncode}'}")
124
+ return completed.stdout.decode("utf-8", errors="replace")
@@ -0,0 +1,218 @@
1
+ Metadata-Version: 2.5
2
+ Name: skillbench-cli
3
+ Version: 0.1.0
4
+ Summary: Test your Claude Code skills and agent workflows, the way you test code.
5
+ Project-URL: Homepage, https://github.com/adamsaul314/skillbench
6
+ Project-URL: Issues, https://github.com/adamsaul314/skillbench/issues
7
+ Author: Adam Saul
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: agents,claude,claude-code,evals,skills,testing
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Topic :: Software Development :: Testing
17
+ Requires-Python: >=3.11
18
+ Requires-Dist: pyyaml>=6.0
19
+ Requires-Dist: rich>=13.7
20
+ Description-Content-Type: text/markdown
21
+
22
+ # skillbench
23
+
24
+ Test your Claude Code skills and agent workflows, the way you test code.
25
+
26
+ You write a small YAML file that says *"send this to Claude, then check these things"*. skillbench runs Claude Code for real on a copy of your repo, records everything it did, and tells you **pass** or **fail**.
27
+
28
+ ---
29
+
30
+ ## 1. Install
31
+
32
+ You need Python 3.11+, git, and Claude Code installed and logged in (check with `claude --version`). Then install with whichever tool you use:
33
+
34
+ ```
35
+ pip install skillbench-cli
36
+ pipx install skillbench-cli
37
+ uv tool install skillbench-cli
38
+ ```
39
+
40
+ The command is `skillbench` either way. To try it without installing: `uvx --from skillbench-cli skillbench init`.
41
+
42
+ ## 2. Try it (2 minutes)
43
+
44
+ In your repo:
45
+
46
+ ```
47
+ skillbench init # creates a skillbench/ folder with two example tests
48
+ skillbench skill # runs the skill test
49
+ skillbench agent # runs the agent test
50
+ ```
51
+
52
+ You'll see something like:
53
+
54
+ ```
55
+ [1/1] hello-greets-by-name default model
56
+ hello-greets-by-name: pass in 2.3s
57
+ dashboard: skillbench/runs/index.html
58
+ ```
59
+
60
+ Open `skillbench/runs/index.html` in your browser to see exactly what Claude did: every message, tool call and check.
61
+
62
+ That's it. The rest of this page shows how to write your own tests.
63
+
64
+ ---
65
+
66
+ ## 3. Two kinds of test
67
+
68
+ | | **Skill test** | **Agent test** |
69
+ |---|---|---|
70
+ | Question it answers | "When my skill runs, does it do the right thing?" | "Given a normal request, does Claude pick the right skills and get the job done?" |
71
+ | Lives in | `skillbench/skills/` | `skillbench/agent/` |
72
+ | Run with | `skillbench skill` | `skillbench agent` |
73
+ | You write | `skill:` and `input:` | `prompt:` |
74
+
75
+ ### A skill test
76
+
77
+ `skillbench/skills/grilling.yaml`
78
+
79
+ ```yaml
80
+ skill: grilling # your skill's name
81
+ input: an idea for a todo app # what to send it
82
+ fixture: . # run it on a copy of this repo
83
+
84
+ assert:
85
+ - final_text_contains: "Q1"
86
+ - tool_not_called: Write
87
+ ```
88
+
89
+ skillbench finds the skill wherever it lives in your repo (`.claude/skills/`, `skills/`, `.github/skills/` or a plugin) and runs it as `/grilling an idea for a todo app`.
90
+
91
+ ### An agent test
92
+
93
+ `skillbench/agent/csv-export.yaml`
94
+
95
+ ```yaml
96
+ prompt: add CSV export to the report page
97
+ fixture: .
98
+
99
+ answers: # replies, sent one at a time when Claude asks something
100
+ - Yes to all your recommendations.
101
+ - approved
102
+
103
+ assert:
104
+ - skill_invoked: planner
105
+ - tool_order: [Read, Edit, Bash]
106
+ - command_succeeds: python -m pytest -q
107
+ ```
108
+
109
+ Nothing is forced here. Claude sees every skill in your repo and decides for itself, and the test checks what it chose.
110
+
111
+ ---
112
+
113
+ ## 4. Checks you can make
114
+
115
+ Put any of these under `assert:`.
116
+
117
+ | Check | Passes when |
118
+ |---|---|
119
+ | `skill_invoked: grilling` | Claude used that skill |
120
+ | `skill_not_invoked: grilling` | Claude didn't use that skill |
121
+ | `tool_order: [Read, Edit]` | Claude used these tools in this order (other tools in between are fine) |
122
+ | `tool_not_called: Write` | Claude never used that tool |
123
+ | `final_text_contains: "Done"` | Claude's last message contains the text |
124
+ | `file_contains: {path: notes.md, text: "approved"}` | That file exists afterwards and contains the text |
125
+ | `command_succeeds: python -m pytest -q` | The command runs successfully in the workspace afterwards |
126
+
127
+ A test passes when every check passes.
128
+
129
+ ---
130
+
131
+ ## 5. Which repo Claude works in (`fixture:`)
132
+
133
+ Claude always works on a **copy**, so your real files are never touched.
134
+
135
+ | `fixture:` | Claude gets |
136
+ |---|---|
137
+ | *(leave it out)* | An empty folder |
138
+ | `.` | A copy of your repo, including work you haven't committed |
139
+ | `some/folder` | A copy of that folder |
140
+ | `https://github.com/you/app.git` | A fresh clone |
141
+ | `{git: https://github.com/you/app.git, ref: v1.2}` | A clone at that branch, tag or commit |
142
+
143
+ Paths are relative to your repo root. The copy is kept after the run so you can look inside, and its path is in the dashboard.
144
+
145
+ ---
146
+
147
+ ## 6. Running tests
148
+
149
+ ```
150
+ skillbench skill # all skill tests
151
+ skillbench skill grilling # just the tests for one skill (or one test by name)
152
+ skillbench agent csv-export # one agent test
153
+ skillbench agent --repeat 5 # run each test 5 times
154
+ skillbench agent --model claude-sonnet-5-5
155
+ ```
156
+
157
+ While it runs you'll see the current test, what Claude is doing, and roughly how long is left.
158
+
159
+ ---
160
+
161
+ ## 7. More options
162
+
163
+ You won't need these to start.
164
+
165
+ **Check a skill gets picked on its own.** List prompts that should and shouldn't trigger it. Each runs briefly and stops as soon as Claude decides:
166
+
167
+ ```yaml
168
+ skill: grilling
169
+ triggers:
170
+ should: ["stress-test my plan"]
171
+ should_not: ["fix this typo"]
172
+ ```
173
+
174
+ **Answer Claude yourself.** Add `interactive: true` and skillbench asks *you* in the terminal whenever Claude is waiting for a reply. Press Enter on an empty line to finish.
175
+
176
+ **Check for flakiness.** Claude doesn't do exactly the same thing every time. `--repeat 5` (or `repeat: 5` in the test) runs it 5 times and shows how often each check passed. Add `min_pass_rate: 0.8` to pass when 4 out of 5 runs pass.
177
+
178
+ **Set defaults for every test** in `skillbench/skillbench.yaml`:
179
+
180
+ ```yaml
181
+ model: claude-sonnet-5-5
182
+ timeout_s: 600 # seconds before a turn is stopped
183
+ fixture: .
184
+ repeat: 1
185
+ min_pass_rate: 1.0
186
+ ```
187
+
188
+ A test's own settings win over these, and command-line flags win over both.
189
+
190
+ **All test fields:**
191
+
192
+ | Field | Meaning |
193
+ |---|---|
194
+ | `skill` / `input` | Skill tests: which skill, and what to send it |
195
+ | `prompt` | Agent tests: the first message |
196
+ | `answers` | Replies sent one per turn |
197
+ | `interactive` | Ask you in the terminal for replies |
198
+ | `triggers` | Prompts that should / shouldn't pick the skill |
199
+ | `fixture` | Which repo Claude works in |
200
+ | `assert` | The checks |
201
+ | `name` | A display name (defaults to the file name) |
202
+ | `timeout_s`, `repeat`, `min_pass_rate` | As above |
203
+
204
+ ---
205
+
206
+ ## Good to know
207
+
208
+ - **It costs real usage.** Every run is a real Claude session. Start with skill tests; long agent tests take longer and cost more.
209
+ - **Your personal setup is ignored.** Only the skills and plugins in the repo being tested are loaded, so results don't depend on whose machine runs them.
210
+ - **Only test repos you trust.** Claude has full tool access inside the copy, and `command_succeeds` runs commands there.
211
+ - **`command_succeeds` uses your system shell** (`sh` on Mac/Linux, `cmd` on Windows), so keep commands simple if you share tests across systems.
212
+ - **Results** are in `skillbench/runs/`, which is git-ignored. Each run has a `run.json` and Claude's raw output.
213
+
214
+ | Status | Meaning |
215
+ |---|---|
216
+ | `pass` | Every check passed |
217
+ | `fail` | Claude finished, but a check failed |
218
+ | `error` | Claude crashed or timed out, or the skill or fixture couldn't be found |
@@ -0,0 +1,16 @@
1
+ skillbench/__init__.py,sha256=xuhAlrEiq6nBnACBFYL-pqeeeQOC_KI__zriGl_y5HQ,51
2
+ skillbench/assertions.py,sha256=QPeDuNGfpldW826wbS9qxsmuNeB-AwFXxWgz_8TcZ_c,9563
3
+ skillbench/claude.py,sha256=xUrSihI1UjdC734pEWtnEnvB6BTg4MxaKyKN5ZlgbuA,10770
4
+ skillbench/cli.py,sha256=JakEyZtrryVAQJXBy0ocgnWHf65Vtooqr6afvuWDGa0,12389
5
+ skillbench/config.py,sha256=MPUvz1wyWUpZw_C-YI0p7CNa_0HW7Q9NFxjq1kqHmAQ,9947
6
+ skillbench/dashboard.py,sha256=HUBG61uZ_qwNtwXdCMBnwp2yVjnY99BAY5ylPaN1MhA,24329
7
+ skillbench/progress.py,sha256=0cc1CQM5ntDkY8rIfpEODmyw2UfjeHzgPaN-9yItrg4,7685
8
+ skillbench/runner.py,sha256=vB_CuQ3qV_owufJAf88d5vK5upZ5ZZm3jpaOUHWyb9U,18305
9
+ skillbench/scaffold.py,sha256=WgB-U8HoV75bkRDcPe_mCj5qUmjGCnQcmFkuK8HqRW8,3841
10
+ skillbench/skills.py,sha256=Qio0am02G6tPg5dCNEiHcUMbvBjsgvZPqoXjbGqJHb8,6417
11
+ skillbench/workspace.py,sha256=F2WCfvlV7JuEfvWkJ2r0zy6BO8KTcSE2725fSwQjdlg,4881
12
+ skillbench_cli-0.1.0.dist-info/METADATA,sha256=26gJCU48lMHOvvI2CPMhWAuVe3aFicIjgUhwgu6ALPE,7503
13
+ skillbench_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
14
+ skillbench_cli-0.1.0.dist-info/entry_points.txt,sha256=YtKWfB9WaELMltae7MJnMmUL37oJXoyDWr3uOb2P4FU,51
15
+ skillbench_cli-0.1.0.dist-info/licenses/LICENSE,sha256=gABCeZ1x3voBu99KF6B55f6HGNdEiOpMKi5ZsBAveLg,1066
16
+ skillbench_cli-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ skillbench = skillbench.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Adam Saul
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.