agent-skill-description-optimizer 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_skill_description_optimizer-0.2.0.dist-info/METADATA +361 -0
- agent_skill_description_optimizer-0.2.0.dist-info/RECORD +17 -0
- agent_skill_description_optimizer-0.2.0.dist-info/WHEEL +4 -0
- agent_skill_description_optimizer-0.2.0.dist-info/entry_points.txt +3 -0
- agent_skill_description_optimizer-0.2.0.dist-info/licenses/LICENSE +21 -0
- skill_optimizer/__init__.py +78 -0
- skill_optimizer/__main__.py +6 -0
- skill_optimizer/_process.py +42 -0
- skill_optimizer/cli.py +1450 -0
- skill_optimizer/evaluation.py +475 -0
- skill_optimizer/improver.py +636 -0
- skill_optimizer/interpreter.py +171 -0
- skill_optimizer/models.py +155 -0
- skill_optimizer/py.typed +0 -0
- skill_optimizer/report.py +297 -0
- skill_optimizer/selection.py +250 -0
- skill_optimizer/skill_md.py +123 -0
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agent-skill-description-optimizer
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Optimize a Claude Code / Agent skill's description frontmatter for reliable triggering — no API key required.
|
|
5
|
+
Keywords: claude,claude-code,skill,agent,description,prompt-optimization,evaluation,llm
|
|
6
|
+
Author: netopsengineer
|
|
7
|
+
Author-email: netopsengineer <enterprise.code.developer@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
18
|
+
Classifier: Topic :: Utilities
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.14
|
|
21
|
+
Project-URL: Homepage, https://github.com/netopsengineer/agent-skill-description-optimizer
|
|
22
|
+
Project-URL: Repository, https://github.com/netopsengineer/agent-skill-description-optimizer
|
|
23
|
+
Project-URL: Issues, https://github.com/netopsengineer/agent-skill-description-optimizer/issues
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# Skill Description Optimizer (no API key)
|
|
27
|
+
|
|
28
|
+
[](https://docs.claude.com/en/docs/claude-code)
|
|
29
|
+
[](https://github.com/netopsengineer/agent-skill-description-optimizer/actions/workflows/release.yml)
|
|
30
|
+
[](https://opensource.org/licenses/MIT)
|
|
31
|
+
|
|
32
|
+
Optimizes the `description:` frontmatter of a Claude Code / Agent **skill** - the
|
|
33
|
+
text that decides whether Claude invokes the skill. Both halves of the loop
|
|
34
|
+
(evaluation **and** improvement) run entirely through the `claude -p` CLI, so **no
|
|
35
|
+
`ANTHROPIC_API_KEY` is required** - it reuses the auth the CLI already has.
|
|
36
|
+
|
|
37
|
+
## Why this exists
|
|
38
|
+
|
|
39
|
+
Skill-creator's own `run_eval.py` already evaluates triggering with `claude -p` (no
|
|
40
|
+
key). The *only* part of its optimization loop that needs an API key is the
|
|
41
|
+
**improver** (`improve_description.py` / `run_loop.py`), which calls the `anthropic`
|
|
42
|
+
Python SDK. This project replaces that piece: **both** the evaluation and the
|
|
43
|
+
improvement go through `claude -p`.
|
|
44
|
+
|
|
45
|
+
## Relationship to skill-creator
|
|
46
|
+
|
|
47
|
+
This is a re-implementation of skill-creator's optimization loop, intended to be
|
|
48
|
+
usable in its place without an API key. Where it matches and where it deliberately
|
|
49
|
+
differs:
|
|
50
|
+
|
|
51
|
+
| Behavior | This tool | skill-creator |
|
|
52
|
+
|---------------------------------------------|---------------------------------------------------|----------------------------|
|
|
53
|
+
| Trigger detection (stream tool-call intent) | **Equivalent** logic (see note) | `run_eval.py` |
|
|
54
|
+
| Improver transport | `claude -p` | `anthropic` SDK |
|
|
55
|
+
| Improver avoids repeating past attempts | Yes | Yes |
|
|
56
|
+
| 1024-char limit with shorten retry | Yes | Yes |
|
|
57
|
+
| Early-exit when all train queries pass | Yes | Yes |
|
|
58
|
+
| Held-out selection (avoid overfitting) | Yes - mean accuracy, ε tie-break on weakest model | Yes - test pass count |
|
|
59
|
+
| Multiple eval models | Yes (`--models haiku,sonnet,opus`) | Single `--model` |
|
|
60
|
+
| Train/test split | Seeded shuffle, stratified | Seeded shuffle, stratified |
|
|
61
|
+
| Live HTML report | Yes (`--report` / `--results-dir`) | Yes |
|
|
62
|
+
|
|
63
|
+
Trigger detection is **equivalent**, not byte-identical: the streamed path is a
|
|
64
|
+
faithful port of skill-creator's tool-intent detection, but the non-streaming
|
|
65
|
+
`assistant` fallback here intentionally searches the serialized tool input more broadly
|
|
66
|
+
(matching the injected command anywhere in it) than upstream's field-specific matching -
|
|
67
|
+
functionally equivalent and arguably more robust.
|
|
68
|
+
|
|
69
|
+
### Drop-in for skill-creator's optimization step
|
|
70
|
+
|
|
71
|
+
It accepts skill-creator's `run_loop.py` flags (`--model`, `--max-iterations`,
|
|
72
|
+
`--holdout`, `--runs-per-query`, `--num-workers`, `--trigger-threshold`, `--verbose`),
|
|
73
|
+
makes `--out` optional, and prints a JSON object containing `best_description` to
|
|
74
|
+
**stdout** (progress goes to stderr) - so skill-creator's Step 3/4 invocation works
|
|
75
|
+
against it unchanged. From another repository, the canonical invocation is
|
|
76
|
+
project-targeted `uv` (always the pinned 3.14 interpreter); agent/headless callers pass
|
|
77
|
+
`--report none` so stdout stays a clean JSON capture and no browser opens:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
uv run --project /ABSOLUTE/PATH/TO/agent-skill-description-optimizer \
|
|
81
|
+
optimize-skill-description \
|
|
82
|
+
--eval-set /ABSOLUTE/PATH/TO/eval.json --skill-path /ABSOLUTE/PATH/TO/skill \
|
|
83
|
+
--model MODEL_ID --report none --max-iterations 5 --verbose
|
|
84
|
+
# then read `best_description` from the JSON it prints to stdout
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
For direct execution without `uv`, name a 3.14+ interpreter explicitly
|
|
88
|
+
(`python3.14 optimize_description_v2.py ...`); an arbitrary `python3` is unsupported and
|
|
89
|
+
exits 1 with a `Requires Python >=3.14; ...` message rather than a traceback.
|
|
90
|
+
|
|
91
|
+
The auto-refreshing **HTML report** is ported too: `--report auto` (the default) opens
|
|
92
|
+
a live, self-refreshing report in your browser, `--report none` disables it (recommended
|
|
93
|
+
for agents), and `--results-dir <dir>` collects `results.json`, `report.html`, and
|
|
94
|
+
`logs/` under a timestamped subdirectory. Both halves of the loop still run through
|
|
95
|
+
`claude -p`.
|
|
96
|
+
|
|
97
|
+
## Requirements
|
|
98
|
+
|
|
99
|
+
- The `claude` CLI, logged in (run `claude` once interactively if unsure). No API key.
|
|
100
|
+
It is checked at startup: a missing or non-executable CLI exits 1 with a one-line
|
|
101
|
+
message, not a mid-run traceback.
|
|
102
|
+
- Python **3.14+** and [`uv`](https://docs.astral.sh/uv/).
|
|
103
|
+
- For triggering to be measurable, the skill's tasks should be ones Claude would
|
|
104
|
+
actually consult a skill for (see [Caveats](#caveats)).
|
|
105
|
+
|
|
106
|
+
## Install
|
|
107
|
+
|
|
108
|
+
The CLI is installable from PyPI:
|
|
109
|
+
|
|
110
|
+
Via `uv`:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
uv tool install agent-skill-description-optimizer --python '>=3.14'
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Via `uvx` for a temporary environment or CI:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
uvx --python '>=3.14' agent-skill-description-optimizer --help
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Via `pip`:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pip install agent-skill-description-optimizer
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Install from source:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
uv sync
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
This installs the `optimize-skill-description` console script. The tool can be run
|
|
135
|
+
three equivalent ways:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
uv run optimize-skill-description --skill-path ... --eval-set ... --out ...
|
|
139
|
+
uv run python -m skill_optimizer --skill-path ... --eval-set ... --out ...
|
|
140
|
+
uv run python optimize_description_v2.py --skill-path ... --eval-set ... --out ...
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
## How it works
|
|
144
|
+
|
|
145
|
+
1. Reads the skill's current description from `SKILL.md`.
|
|
146
|
+
2. **Evaluates** it: injects the description as a temporary slash-command into a
|
|
147
|
+
throwaway temp project, runs each eval query through `claude -p`, and detects -
|
|
148
|
+
from the streamed tool-call intent - whether the model decided to invoke it.
|
|
149
|
+
Runs across one or more models, `--repeats` times each.
|
|
150
|
+
3. **Improves** it: sends the current description + the failing queries to an
|
|
151
|
+
improver model via `claude -p --effort high`, which returns a rewritten
|
|
152
|
+
description as JSON.
|
|
153
|
+
4. Splits the eval set into **train / held-out test** (stratified by
|
|
154
|
+
`should_trigger`), iterates, and keeps the description with the best **held-out**
|
|
155
|
+
score - selecting on held-out rather than train avoids overfitting the eval set.
|
|
156
|
+
`--holdout 0` disables the split (skill-creator's semantics): every query trains
|
|
157
|
+
and selection falls back to the **train** score. A positive holdout needs at least
|
|
158
|
+
two queries in each `should_trigger` class and must leave a train and a test member
|
|
159
|
+
in each, or the run is rejected up front.
|
|
160
|
+
|
|
161
|
+
## Eval set format
|
|
162
|
+
|
|
163
|
+
A JSON array (a `{"queries": [...]}` or `{"evals": [...]}` wrapper is also accepted -
|
|
164
|
+
exactly one of those keys, with unrelated metadata alongside it ignored):
|
|
165
|
+
|
|
166
|
+
```json
|
|
167
|
+
[
|
|
168
|
+
{"query": "my imports are a mess in billing.py, sort them and drop the unused ones", "should_trigger": true},
|
|
169
|
+
{"query": "format this json blob so it's readable", "should_trigger": false}
|
|
170
|
+
]
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Each item must be an object with a string `query` and a boolean `should_trigger`
|
|
174
|
+
(integers and truthy values are **not** coerced); item order, duplicate queries, and
|
|
175
|
+
extra item keys are preserved. A missing or unreadable file, an empty set, a malformed
|
|
176
|
+
root, or a bad item type is rejected up front with an `Invalid eval set: ...` message,
|
|
177
|
+
before any evaluation runs.
|
|
178
|
+
|
|
179
|
+
Aim for 8-10 of each. The valuable `should_trigger: false` cases are **near-misses** -
|
|
180
|
+
queries that share keywords with the skill but actually need something else - not
|
|
181
|
+
obviously-irrelevant queries.
|
|
182
|
+
|
|
183
|
+
## Usage
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
# Minimal: tune one skill against an eval set, on Sonnet.
|
|
187
|
+
uv run optimize-skill-description \
|
|
188
|
+
--skill-path /path/to/skills/my-skill \
|
|
189
|
+
--eval-set my-skill-queries.json \
|
|
190
|
+
--out runs/my-skill
|
|
191
|
+
|
|
192
|
+
# Robust across models, disable a competing installed plugin during eval,
|
|
193
|
+
# and write the winner back into SKILL.md (creates SKILL.md.bak).
|
|
194
|
+
uv run optimize-skill-description \
|
|
195
|
+
--skill-path /path/to/skills/my-skill \
|
|
196
|
+
--eval-set my-skill-queries.json \
|
|
197
|
+
--out runs/my-skill \
|
|
198
|
+
--models haiku,sonnet,opus \
|
|
199
|
+
--disable-plugin some-plugin@some-marketplace \
|
|
200
|
+
--iterations 3 \
|
|
201
|
+
--write
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
The chosen description is reported at the end and saved in `report.json`. Without
|
|
205
|
+
`--write`, nothing touches your skill - you just get the recommendation.
|
|
206
|
+
|
|
207
|
+
### Key flags
|
|
208
|
+
|
|
209
|
+
| Flag | Default | Notes |
|
|
210
|
+
|---------------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
211
|
+
| `--skill-path` | (required) | Dir containing `SKILL.md`. |
|
|
212
|
+
| `--eval-set` | (required) | JSON eval set (see format above). |
|
|
213
|
+
| `--out` | temp dir | Where run artifacts are written; defaults to a temporary directory. |
|
|
214
|
+
| `--models` | `sonnet` | Comma list; aliases `haiku`/`sonnet`/`opus` or full model ids. Optimizes on the **mean** accuracy across them but reports the **min** so a winner can't tank the weakest model. |
|
|
215
|
+
| `--improver-model` | `opus` | Model that rewrites the description. |
|
|
216
|
+
| `--model` | (none) | Single model for **both** eval and improver (skill-creator compat); `--models`/`--improver-model` override it. |
|
|
217
|
+
| `--description` | (none) | Override the starting description instead of reading `SKILL.md`'s. |
|
|
218
|
+
| `--improver-effort` | `high` | Passed as `--effort`; use `none` to omit. |
|
|
219
|
+
| `--repeats` | `3` | Runs per (query, model). More = less noise, more cost. |
|
|
220
|
+
| `--iterations` | `5` | Improve->re-eval rounds, inclusive `0`-`50` (`0` = baseline-only, no proposals). Stops early once all train queries pass. |
|
|
221
|
+
| `--threshold` | `0.5` | Trigger-rate at/above which a query counts as "triggered". |
|
|
222
|
+
| `--test-frac` | `0.4` | Held-out fraction (stratified by class), in `[0, 1)`. `0` disables the holdout and selects on train; a positive value needs >=2 queries per class and leaves a member in each. |
|
|
223
|
+
| `--seed` | `42` | RNG seed for the stratified split. Fixed for reproducibility and echoed in the output JSON, so a run reproduces from its own record; vary it to check split robustness. |
|
|
224
|
+
| `--select-epsilon` | `0.05` | Held-out mean differences within this band count as ties, broken by the weakest-model accuracy. `0` = strict mean-only selection. |
|
|
225
|
+
| `--max-desc-chars` | `1024` | Hard character budget. An over-budget candidate can never be selected, and `--write` refuses an over-budget winner. |
|
|
226
|
+
| `--report` | `auto` | HTML report: `auto` (temp file, opened in a browser), `none` to disable, or an explicit output path. |
|
|
227
|
+
| `--dry-run` | off | Validate inputs (eval set, skill, holdout split, `claude` availability) and print the run plan as JSON to stdout with an estimated `claude -p` call count, then exit spending no tokens. |
|
|
228
|
+
| `--results-dir` | (none) | Save `results.json`, `report.html`, and `logs/` under a timestamped subdirectory here. Mutually exclusive with `--out`. |
|
|
229
|
+
| `--disable-plugin` | (none) | Repeatable. Disables a **plugin-provided** skill (by plugin id) during eval so it can't out-compete the injected candidate. Does not affect standalone user/project skills - see Caveats. |
|
|
230
|
+
| `--write` | off | Write the best description into `SKILL.md` (backs up to `SKILL.md.bak`). |
|
|
231
|
+
| `--version` | - | Print the installed package version and exit. |
|
|
232
|
+
|
|
233
|
+
Skill-creator aliases are accepted for the matching flags: `--max-iterations`
|
|
234
|
+
(`--iterations`), `--holdout` (`--test-frac`), `--runs-per-query` (`--repeats`),
|
|
235
|
+
`--num-workers` (`--workers`), `--trigger-threshold` (`--threshold`). The
|
|
236
|
+
`best_description` is also printed as a JSON object to stdout.
|
|
237
|
+
|
|
238
|
+
The `haiku`/`sonnet`/`opus` aliases are **source-pinned** convenience defaults that
|
|
239
|
+
currently resolve to `claude-haiku-4-5-20251001`, `claude-sonnet-5`, and
|
|
240
|
+
`claude-opus-4-8`; any other value is passed through as a full model id unchanged. The
|
|
241
|
+
alias targets track a moving lineup, so for reproducible runs pass an explicit full
|
|
242
|
+
`--model`/`--models`/`--improver-model` id rather than relying on the aliases.
|
|
243
|
+
|
|
244
|
+
### Output
|
|
245
|
+
|
|
246
|
+
The best description is reported on stderr, printed as JSON to **stdout**, and
|
|
247
|
+
written (with full history) to the artifact dir:
|
|
248
|
+
|
|
249
|
+
- `baseline.json` - full-set eval of the current description.
|
|
250
|
+
- `iterN_prompt.txt`, `iterN_proposal.json`, `iterN_eval.json`,
|
|
251
|
+
`iterN_improve.json` (raw improver transcript), plus `iterN_improve_retry.json` when
|
|
252
|
+
a slot's first attempt failed retryably and was retried - per iteration. The raw
|
|
253
|
+
transcripts are the **only** place raw improver stdout/stderr and return codes are
|
|
254
|
+
kept; public output never carries them.
|
|
255
|
+
- `report.json` - baseline vs best description, held-out scores, full history.
|
|
256
|
+
|
|
257
|
+
The stdout JSON is a **superset** of skill-creator's `run_loop` envelope
|
|
258
|
+
(`best_description`, `best_score`, `exit_reason`, `iterations_run`, `is_best` per
|
|
259
|
+
history entry, char counts, ...) plus this tool's own keys (`best_test_mean`,
|
|
260
|
+
`select_epsilon`, per-model accuracy, `seed`, `estimated_claude_calls`, ...) - nothing
|
|
261
|
+
from the old shape is removed. `--dry-run` instead prints a `{"dry_run": true, ...}`
|
|
262
|
+
plan object (same estimate) and exits without spending tokens. Two
|
|
263
|
+
of the shared keys carry different **semantics**, though: `iterations_run` counts the
|
|
264
|
+
improve rounds actually entered - excluding the baseline, and *including* a round whose
|
|
265
|
+
improver retries were exhausted (which produces no `history` row) - whereas skill-creator
|
|
266
|
+
reports `len(history)`; and `history` numbers the baseline as `iteration` 0 here, where
|
|
267
|
+
skill-creator starts at 1. Consumers that only read `best_description` are unaffected. The
|
|
268
|
+
score strings (`best_score`, history `train_passed/train_total`, ...) use a
|
|
269
|
+
**judged-query** denominator (`k/N` with `(+u unjudged)` when probes errored), so a
|
|
270
|
+
transient CLI failure reads as "unjudged", not as a miss - a deliberate deviation from
|
|
271
|
+
skill-creator's total-query denominator.
|
|
272
|
+
|
|
273
|
+
#### Improver failures and retries
|
|
274
|
+
|
|
275
|
+
An improver call that **times out**, returns **unparseable JSON**, or is still over the
|
|
276
|
+
character budget **after the shorten retry** is a *retryable* failure: the slot is
|
|
277
|
+
retried once, and if the second attempt also fails retryably the slot is recorded and
|
|
278
|
+
the loop continues to the next iteration (the last verified best is preserved). A
|
|
279
|
+
**completed non-zero exit** or launch-budget exhaustion is *fatal* and aborts the run
|
|
280
|
+
without emitting a success envelope: the CLI reports it as a single `stderr` line (exit
|
|
281
|
+
1, empty stdout, no traceback - so a stdout-parsing caller fails legibly), while the raw
|
|
282
|
+
child return code/stderr stay in the per-iteration transcript. Any other unclassified
|
|
283
|
+
error propagates unchanged. Retry work is bounded: at most 50 proposal slots and 200
|
|
284
|
+
improver child processes per run.
|
|
285
|
+
|
|
286
|
+
Every result (stdout, `report.json`, live report) always carries an
|
|
287
|
+
`improver_failed_iterations` list - `[]` when nothing failed. Each entry is
|
|
288
|
+
`{"iteration": N, "attempt_count": 2, "errors": [{"attempt", "kind", "message"}, ...]}`,
|
|
289
|
+
where `kind` is one of `timeout` / `invalid_output` / `length_limit` and `message` is a
|
|
290
|
+
fixed, non-sensitive template (raw stdout/stderr/paths never appear here).
|
|
291
|
+
|
|
292
|
+
With `--report auto` (default) a live, self-refreshing `report.html` opens in your
|
|
293
|
+
browser and updates after each iteration; `--results-dir <dir>` additionally writes
|
|
294
|
+
`results.json`, `report.html`, and per-iteration `logs/` under a timestamped
|
|
295
|
+
subdirectory.
|
|
296
|
+
|
|
297
|
+
## Project layout
|
|
298
|
+
|
|
299
|
+
```text
|
|
300
|
+
src/skill_optimizer/ # the package
|
|
301
|
+
skill_md.py # SKILL.md frontmatter parse / write
|
|
302
|
+
interpreter.py # pure trigger-decision state machine over stream-json
|
|
303
|
+
evaluation.py # claude -p transport, aggregation, concurrent eval
|
|
304
|
+
improver.py # improver prompt + claude -p call
|
|
305
|
+
selection.py # train/test split, model resolution, candidate selection
|
|
306
|
+
cli.py # argparse + orchestration
|
|
307
|
+
models.py # shared types (EvalConfig, TypedDicts) and aliases
|
|
308
|
+
optimize_description_v2.py # entry/compat shim re-exporting the package
|
|
309
|
+
tests/ # pytest suite + recorded stream-json fixtures
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
## Development
|
|
313
|
+
|
|
314
|
+
```bash
|
|
315
|
+
uv run pytest # test suite (characterization + unit + integration)
|
|
316
|
+
uv run pyright # strict type checking
|
|
317
|
+
uv run ruff check . # lint
|
|
318
|
+
uv run pydoclint src/skill_optimizer optimize_description_v2.py # docstring checks
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
Integration tests exercise the real subprocess/stream-parsing transport against a
|
|
322
|
+
fake `claude` binary and recorded `claude -p` streams in `tests/fixtures/` - no live
|
|
323
|
+
CLI calls or API key needed to run the suite.
|
|
324
|
+
|
|
325
|
+
## Caveats
|
|
326
|
+
|
|
327
|
+
These are what the evals taught us - read them before trusting a score:
|
|
328
|
+
|
|
329
|
+
- **Triggering is a proxy.** We inject the description as a slash-command and watch
|
|
330
|
+
whether the model decides to call it. It's a consistent *relative* signal for
|
|
331
|
+
comparing descriptions; it is not a perfect predictor of real installed-plugin
|
|
332
|
+
triggering.
|
|
333
|
+
- **A non-skill tool first reads as no-trigger.** Detection keys on the *first*
|
|
334
|
+
tool the model reaches for; if a substantive query makes it run `Bash` (or
|
|
335
|
+
another tool) before the skill, that run scores as a no-trigger. This matches
|
|
336
|
+
skill-creator's own detection - design eval queries with it in mind.
|
|
337
|
+
- **Capable models under-trigger by design.** Claude only consults a skill for tasks
|
|
338
|
+
it can't easily do itself. Simple, one-step queries often won't trigger *any*
|
|
339
|
+
skill no matter how good the description is - and for skills that wrap tools the
|
|
340
|
+
model already knows well, the achievable trigger rate has a low ceiling that
|
|
341
|
+
description tuning cannot raise. Make queries substantive, and judge results with
|
|
342
|
+
that ceiling in mind.
|
|
343
|
+
- **Disable competing skills.** If the skill you're tuning is already installed, its
|
|
344
|
+
real description will compete with the injected candidate and pollute results.
|
|
345
|
+
`--disable-plugin <id@marketplace>` handles this **only for plugin-provided skills**
|
|
346
|
+
(it writes `enabledPlugins: {<id>: false}` into `--settings`, keyed by plugin id). A
|
|
347
|
+
standalone **user/project** skill of the same name (e.g. under `~/.claude/skills/`) is
|
|
348
|
+
not a plugin and is unaffected; isolate it from discovery separately for the run
|
|
349
|
+
(move it aside) and restore it afterward. This tool never mutates a user skill
|
|
350
|
+
directory for you.
|
|
351
|
+
- **Cost.** Total `claude -p` calls ≈ `queries x models x repeats x (iterations + 1)` -
|
|
352
|
+
one full-set evaluation per baseline/candidate (train and held-out views are sliced
|
|
353
|
+
from it, not re-evaluated). An unjudgeable probe (timeout/CLI error) is retried at
|
|
354
|
+
most once, so real counts run slightly above this estimate - not a 2x multiplier.
|
|
355
|
+
Start small (`--models sonnet --repeats 2 --iterations 1`) to sanity-check before
|
|
356
|
+
scaling up.
|
|
357
|
+
|
|
358
|
+
## Contributing
|
|
359
|
+
|
|
360
|
+
Local setup, commit conventions, the release process, and CI/CD - see
|
|
361
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
skill_optimizer/__init__.py,sha256=kwXxCTYqMsdFg7t6TiuEhXdF4szlCSswQevncnExEto,2230
|
|
2
|
+
skill_optimizer/__main__.py,sha256=anvEpyKGZv9xKC7ipd51mx1GaLqiwf269ZDHIr9CcRw,139
|
|
3
|
+
skill_optimizer/_process.py,sha256=N_LMY2aL9yAh0RR5bFDHJ38tSCRD4e58HQ0OHe1Elj8,1426
|
|
4
|
+
skill_optimizer/cli.py,sha256=wBxQxf99o3tquysOqsvzmJWfR0UYe1heiZxRUgXDSsg,54685
|
|
5
|
+
skill_optimizer/evaluation.py,sha256=z1CpReC99sUTP5-a4TZaegiHVHLUnCeWEGIScjAoi9U,17593
|
|
6
|
+
skill_optimizer/improver.py,sha256=Eemkej-RCtileDjQC6SzOUWTUkRwrd83mgMFD-e4X-g,25894
|
|
7
|
+
skill_optimizer/interpreter.py,sha256=oYYCC6YlNIHpQgC7EzBCtxFC0fj3bF9aCihMozuovNM,6291
|
|
8
|
+
skill_optimizer/models.py,sha256=g1jk_9JgVm_TbyA_AhRn-iz39reM7pE6K1iF69H0yjU,5619
|
|
9
|
+
skill_optimizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
skill_optimizer/report.py,sha256=2brOv6D4qxUi7bhWup0ICJ2up29312_CPjnDgLNvVnM,11643
|
|
11
|
+
skill_optimizer/selection.py,sha256=-SqkEjDeqMqBvu6KBjWjcRjtpPzyk0RNYsAvPFYwYt4,10644
|
|
12
|
+
skill_optimizer/skill_md.py,sha256=ejNEIvEAjJI4Y1oDusndjIsgdLykximI5BZGsaqVr-U,5222
|
|
13
|
+
agent_skill_description_optimizer-0.2.0.dist-info/licenses/LICENSE,sha256=8rjTRwiCOr04SNdwgDrfbx4GwrtictpSlknWZstWg5A,1071
|
|
14
|
+
agent_skill_description_optimizer-0.2.0.dist-info/WHEEL,sha256=YR4QUxGCbsyoOnWBVTv-p1I4yc3seKGPev46mvmc8Cg,81
|
|
15
|
+
agent_skill_description_optimizer-0.2.0.dist-info/entry_points.txt,sha256=CMwF6pkgtIQQf9puTMK4cKLV1Gcy4QuYwoFOCe9c_ZA,73
|
|
16
|
+
agent_skill_description_optimizer-0.2.0.dist-info/METADATA,sha256=iyV2sI93_1u_4GyivZPVLFSzftIsFM6bnzg4zfE1vgc,22853
|
|
17
|
+
agent_skill_description_optimizer-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 netopsengineer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Project-agnostic skill *description* optimizer — no ANTHROPIC_API_KEY required.
|
|
2
|
+
|
|
3
|
+
The skill-creator's own ``run_eval.py`` already evaluates triggering with ``claude
|
|
4
|
+
-p`` (no key). The only piece that needs an API key is the improver, which calls the
|
|
5
|
+
``anthropic`` SDK. This package replaces that piece: both evaluation and improvement
|
|
6
|
+
go through ``claude -p``, so the whole loop runs with the same auth the CLI uses.
|
|
7
|
+
|
|
8
|
+
For ANY single skill it: reads the current ``description`` from ``SKILL.md``;
|
|
9
|
+
evaluates how reliably it triggers across one or more models via ``claude -p``; asks
|
|
10
|
+
an improver model to rewrite it from the failures; and selects the best description
|
|
11
|
+
by held-out score, optionally writing it back with ``--write``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from importlib import metadata
|
|
15
|
+
|
|
16
|
+
from skill_optimizer.cli import build_parser, main, run
|
|
17
|
+
from skill_optimizer.evaluation import (
|
|
18
|
+
aggregate,
|
|
19
|
+
evaluate,
|
|
20
|
+
run_query_with_retry,
|
|
21
|
+
run_single_query,
|
|
22
|
+
subset_result,
|
|
23
|
+
)
|
|
24
|
+
from skill_optimizer.improver import (
|
|
25
|
+
IMPROVER_TEMPLATE,
|
|
26
|
+
build_improver_prompt,
|
|
27
|
+
call_improver,
|
|
28
|
+
)
|
|
29
|
+
from skill_optimizer.interpreter import interpret_events
|
|
30
|
+
from skill_optimizer.models import (
|
|
31
|
+
MODEL_ALIASES,
|
|
32
|
+
EvalConfig,
|
|
33
|
+
EvalQuery,
|
|
34
|
+
EvalResult,
|
|
35
|
+
ModelResult,
|
|
36
|
+
PerQuery,
|
|
37
|
+
)
|
|
38
|
+
from skill_optimizer.report import generate_html
|
|
39
|
+
from skill_optimizer.selection import (
|
|
40
|
+
is_better_candidate,
|
|
41
|
+
resolve_models,
|
|
42
|
+
stratified_split,
|
|
43
|
+
summarize,
|
|
44
|
+
)
|
|
45
|
+
from skill_optimizer.skill_md import parse_skill_md, write_description
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
__version__ = metadata.version("agent-skill-description-optimizer")
|
|
49
|
+
except metadata.PackageNotFoundError: # source checkout without an install
|
|
50
|
+
__version__ = "0.0.0+unknown"
|
|
51
|
+
|
|
52
|
+
__all__ = [
|
|
53
|
+
"IMPROVER_TEMPLATE",
|
|
54
|
+
"MODEL_ALIASES",
|
|
55
|
+
"EvalConfig",
|
|
56
|
+
"EvalQuery",
|
|
57
|
+
"EvalResult",
|
|
58
|
+
"ModelResult",
|
|
59
|
+
"PerQuery",
|
|
60
|
+
"aggregate",
|
|
61
|
+
"build_improver_prompt",
|
|
62
|
+
"build_parser",
|
|
63
|
+
"call_improver",
|
|
64
|
+
"evaluate",
|
|
65
|
+
"generate_html",
|
|
66
|
+
"interpret_events",
|
|
67
|
+
"is_better_candidate",
|
|
68
|
+
"main",
|
|
69
|
+
"parse_skill_md",
|
|
70
|
+
"resolve_models",
|
|
71
|
+
"run",
|
|
72
|
+
"run_query_with_retry",
|
|
73
|
+
"run_single_query",
|
|
74
|
+
"stratified_split",
|
|
75
|
+
"subset_result",
|
|
76
|
+
"summarize",
|
|
77
|
+
"write_description",
|
|
78
|
+
]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Shared helpers for invoking the ``claude`` CLI as a subprocess."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
# Env var to override the CLI binary, primarily so tests can point at a fake
|
|
7
|
+
# executable. Defaults to ``claude`` on PATH.
|
|
8
|
+
_CLAUDE_BIN_ENV = "SKILL_OPTIMIZER_CLAUDE_BIN"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def claude_bin() -> str:
|
|
12
|
+
"""Return the ``claude`` executable to invoke.
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
The path/name from ``$SKILL_OPTIMIZER_CLAUDE_BIN``, or ``"claude"``.
|
|
16
|
+
"""
|
|
17
|
+
return os.environ.get(_CLAUDE_BIN_ENV, "claude")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def claude_available() -> bool:
|
|
21
|
+
"""Report whether the resolved ``claude`` binary is present and executable.
|
|
22
|
+
|
|
23
|
+
Uses :func:`shutil.which`, which resolves a bare name against ``PATH`` and checks a
|
|
24
|
+
path with a directory separator directly — matching how the subprocess spawn locates
|
|
25
|
+
the binary. Detects only that the executable is invocable, not whether it is logged
|
|
26
|
+
in (that needs a live call).
|
|
27
|
+
|
|
28
|
+
Returns:
|
|
29
|
+
``True`` when :func:`claude_bin` resolves to an existing executable file, else
|
|
30
|
+
``False``.
|
|
31
|
+
"""
|
|
32
|
+
return shutil.which(claude_bin()) is not None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def subprocess_env() -> dict[str, str]:
|
|
36
|
+
"""Build the child-process environment.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
A copy of the current environment with ``CLAUDECODE`` removed so a nested
|
|
40
|
+
``claude -p`` does not detect that it is running inside Claude Code.
|
|
41
|
+
"""
|
|
42
|
+
return {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
|