pi-ui-extend 0.1.38 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/app.d.ts +0 -1
- package/dist/app/app.js +28 -21
- package/dist/app/constants.js +1 -1
- package/dist/app/input/input-action-controller.d.ts +1 -0
- package/dist/app/input/input-action-controller.js +3 -0
- package/dist/app/input/input-controller.d.ts +1 -0
- package/dist/app/input/input-controller.js +40 -12
- package/dist/app/model/model-usage-status.js +4 -2
- package/dist/app/process.js +11 -0
- package/dist/app/rendering/conversation-tool-renderer.js +4 -6
- package/dist/app/session/request-history.js +2 -0
- package/dist/app/session/session-event-controller.d.ts +13 -0
- package/dist/app/session/session-event-controller.js +27 -0
- package/dist/app/session/tabs-controller.d.ts +8 -0
- package/dist/app/session/tabs-controller.js +37 -6
- package/dist/app/workspace/workspace-actions-controller.d.ts +1 -0
- package/dist/app/workspace/workspace-actions-controller.js +2 -1
- package/dist/bundled-extensions/terminal-bell/index.js +55 -1
- package/dist/config.js +1 -1
- package/dist/default-pix-config.js +1 -1
- package/dist/markdown-format.js +14 -25
- package/dist/terminal-width.d.ts +14 -0
- package/dist/terminal-width.js +31 -2
- package/dist/theme.js +2 -2
- package/external/pi-tools-suite/README.md +34 -9
- package/external/pi-tools-suite/package.json +3 -3
- package/external/pi-tools-suite/src/async-subagents/async-subagents.sample.jsonc +35 -21
- package/external/pi-tools-suite/src/async-subagents/commands.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/agent-strategy.ts +2 -2
- package/external/pi-tools-suite/src/async-subagents/core/config.ts +70 -12
- package/external/pi-tools-suite/src/async-subagents/core/routing.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/core/types.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/index.ts +6 -6
- package/external/pi-tools-suite/src/async-subagents/lib.ts +1 -1
- package/external/pi-tools-suite/src/async-subagents/tools/spawn.ts +4 -2
- package/external/pi-tools-suite/src/async-subagents/tools/subagents.ts +2 -2
- package/external/pi-tools-suite/src/{glm-coding-discipline → coding-discipline}/index.ts +17 -8
- package/external/pi-tools-suite/src/config.ts +1 -1
- package/external/pi-tools-suite/src/dcp/auto-compress.ts +368 -0
- package/external/pi-tools-suite/src/dcp/compress-tool.ts +3 -0
- package/external/pi-tools-suite/src/dcp/config.ts +23 -0
- package/external/pi-tools-suite/src/dcp/index.ts +112 -7
- package/external/pi-tools-suite/src/dcp/prompts.ts +8 -0
- package/external/pi-tools-suite/src/dcp/state.ts +41 -0
- package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +30 -22
- package/external/pi-tools-suite/src/index.ts +2 -1
- package/external/pi-tools-suite/src/session-name/index.ts +37 -0
- package/external/pi-tools-suite/src/tool-descriptions.ts +16 -4
- package/package.json +4 -4
- package/skills/skill-creator/SKILL.md +36 -40
- package/skills/skill-creator/eval-viewer/viewer.html +2 -2
- package/skills/skill-creator/references/schemas.md +1 -1
- package/skills/skill-creator/scripts/__pycache__/__init__.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/aggregate_benchmark.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/generate_report.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/improve_description.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/package_skill.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/run_eval.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/run_loop.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/__pycache__/utils.cpython-314.pyc +0 -0
- package/skills/skill-creator/scripts/generate_report.py +1 -1
- package/skills/skill-creator/scripts/improve_description.py +14 -24
- package/skills/skill-creator/scripts/run_eval.py +89 -82
|
@@ -11,7 +11,7 @@ At a high level, the process of creating a skill goes like this:
|
|
|
11
11
|
|
|
12
12
|
- Decide what you want the skill to do and roughly how it should do it
|
|
13
13
|
- Write a draft of the skill
|
|
14
|
-
- Create a few test prompts and run
|
|
14
|
+
- Create a few test prompts and run pi-with-access-to-the-skill on them
|
|
15
15
|
- Help the user evaluate the results both qualitatively and quantitatively
|
|
16
16
|
- While the runs happen in the background, draft some quantitative evals if there aren't any (if there are some, you can either use as is or modify if you feel something needs to change about them). Then explain them to the user (or if they already existed, explain the ones that already exist)
|
|
17
17
|
- Use the `eval-viewer/generate_review.py` script to show the user the results for them to look at, and also let them look at the quantitative metrics
|
|
@@ -31,7 +31,7 @@ Cool? Cool.
|
|
|
31
31
|
|
|
32
32
|
## Communicating with the user
|
|
33
33
|
|
|
34
|
-
The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of
|
|
34
|
+
The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of AI coding agents like pi is inspiring plumbers to open up their terminals, parents and grandparents to google "how to install npm". On the other hand, the bulk of users are probably fairly computer-literate.
|
|
35
35
|
|
|
36
36
|
So please pay attention to context cues to understand how to phrase your communication! In the default case, just to give you some idea:
|
|
37
37
|
|
|
@@ -48,7 +48,7 @@ It's OK to briefly explain terms if you're in doubt, and feel free to clarify te
|
|
|
48
48
|
|
|
49
49
|
Start by understanding the user's intent. The current conversation might already contain a workflow the user wants to capture (e.g., they say "turn this into a skill"). If so, extract answers from the conversation history first — the tools used, the sequence of steps, corrections the user made, input/output formats observed. The user may need to fill the gaps, and should confirm before proceeding to the next step.
|
|
50
50
|
|
|
51
|
-
1. What should this skill enable
|
|
51
|
+
1. What should this skill enable pi to do?
|
|
52
52
|
2. When should this skill trigger? (what user phrases/contexts)
|
|
53
53
|
3. What's the expected output format?
|
|
54
54
|
4. Should we set up test cases to verify the skill works? Skills with objectively verifiable outputs (file transforms, data extraction, code generation, fixed workflow steps) benefit from test cases. Skills with subjective outputs (writing style, art) often don't need them. Suggest the appropriate default based on the skill type, but let the user decide.
|
|
@@ -63,8 +63,8 @@ Check available MCPs - if useful for research (searching docs, finding similar s
|
|
|
63
63
|
|
|
64
64
|
Based on the user interview, fill in these components:
|
|
65
65
|
|
|
66
|
-
- **name**: Skill identifier
|
|
67
|
-
- **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently
|
|
66
|
+
- **name**: Skill identifier (lowercase, hyphens, a-z0-9 — see pi's skill validation rules)
|
|
67
|
+
- **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently models have a tendency to "undertrigger" skills -- to not use them when they'd be useful. To combat this, please make the skill descriptions a little bit "pushy". So for instance, instead of "How to build a simple fast dashboard to display internal Anthropic data.", you might write "How to build a simple fast dashboard to display internal Anthropic data. Make sure to use this skill whenever the user mentions dashboards, data visualization, internal metrics, or wants to display any kind of company data, even if they don't explicitly ask for a 'dashboard.'"
|
|
68
68
|
- **compatibility**: Required tools, dependencies (optional, rarely needed)
|
|
69
69
|
- **the rest of the skill :)**
|
|
70
70
|
|
|
@@ -106,7 +106,15 @@ cloud-deploy/
|
|
|
106
106
|
├── gcp.md
|
|
107
107
|
└── azure.md
|
|
108
108
|
```
|
|
109
|
-
|
|
109
|
+
The agent reads only the relevant reference file.
|
|
110
|
+
|
|
111
|
+
#### How pi discovers and loads skills
|
|
112
|
+
|
|
113
|
+
This matters both for writing skills and for testing them. Pi scans skill locations at startup (`~/.pi/agent/skills/`, `~/.agents/skills/`, project `.pi/skills/` and `.agents/skills/`, settings `skills` arrays, and the `--skill <path>` CLI flag). It extracts each skill's `name` + `description` and lists them in the system prompt as `available_skills`. When a task matches a skill, the agent uses the `read` tool to load the full `SKILL.md` on demand — that's progressive disclosure. Skills also register as `/skill:name` commands the user can invoke directly to force-load them.
|
|
114
|
+
|
|
115
|
+
Two practical consequences for this skill:
|
|
116
|
+
- **Where to install a finished skill**: global skills go in `~/.pi/agent/skills/<skill-name>/`; project skills go in `.pi/skills/<skill-name>/` (only loaded once the project is trusted).
|
|
117
|
+
- **How triggering is measured**: the eval scripts in `scripts/` test triggering by watching whether pi actually performs that `read` on the skill's `SKILL.md`. They invoke `pi -p --mode json --skill <temp-skill-dir>` so the description under test is evaluated in isolation (with `--no-skills` to suppress all other discovered skills).
|
|
110
118
|
|
|
111
119
|
#### Principle of Lack of Surprise
|
|
112
120
|
|
|
@@ -168,9 +176,9 @@ Put results in `<skill-name>-workspace/` as a sibling to the skill directory. Wi
|
|
|
168
176
|
|
|
169
177
|
### Step 1: Spawn all runs (with-skill AND baseline) in the same turn
|
|
170
178
|
|
|
171
|
-
For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time.
|
|
179
|
+
For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time. (If your harness has no subagent tool, see "Running without subagents" below — run them inline, one at a time, and skip baselines.)
|
|
172
180
|
|
|
173
|
-
**With-skill run
|
|
181
|
+
**With-skill run** (give the subagent a task like this):
|
|
174
182
|
|
|
175
183
|
```
|
|
176
184
|
Execute this task:
|
|
@@ -179,6 +187,8 @@ Execute this task:
|
|
|
179
187
|
- Input files: <eval files if any, or "none">
|
|
180
188
|
- Save outputs to: <workspace>/iteration-<N>/eval-<ID>/with_skill/outputs/
|
|
181
189
|
- Outputs to save: <what the user cares about — e.g., "the .docx file", "the final CSV">
|
|
190
|
+
|
|
191
|
+
Load the skill at <path-to-skill> (read its SKILL.md) before starting, and follow it.
|
|
182
192
|
```
|
|
183
193
|
|
|
184
194
|
**Baseline run** (same prompt, but the baseline depends on context):
|
|
@@ -206,7 +216,7 @@ Update the `eval_metadata.json` files and `evals/evals.json` with the assertions
|
|
|
206
216
|
|
|
207
217
|
### Step 3: As runs complete, capture timing data
|
|
208
218
|
|
|
209
|
-
When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms
|
|
219
|
+
When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms` (or equivalent usage info from the result). Save this data immediately to `timing.json` in the run directory:
|
|
210
220
|
|
|
211
221
|
```json
|
|
212
222
|
{
|
|
@@ -216,7 +226,7 @@ When each subagent task completes, you receive a notification containing `total_
|
|
|
216
226
|
}
|
|
217
227
|
```
|
|
218
228
|
|
|
219
|
-
|
|
229
|
+
If your harness's completion notification carries a different shape, capture whatever token/duration fields it provides. Process each notification as it arrives rather than trying to batch them.
|
|
220
230
|
|
|
221
231
|
### Step 4: Grade, aggregate, and launch the viewer
|
|
222
232
|
|
|
@@ -244,7 +254,7 @@ Put each with_skill version before its baseline counterpart.
|
|
|
244
254
|
```
|
|
245
255
|
For iteration 2+, also pass `--previous-workspace <workspace>/iteration-<N-1>`.
|
|
246
256
|
|
|
247
|
-
**
|
|
257
|
+
**Headless / no-display environments:** If `webbrowser.open()` is not available or the environment has no display (remote server, CI), use `--static <output_path>` to write a standalone HTML file instead of starting a server. Feedback will be downloaded as a `feedback.json` file when the user clicks "Submit All Reviews". After download, copy `feedback.json` into the workspace directory for the next iteration to pick up.
|
|
248
258
|
|
|
249
259
|
Note: please use generate_review.py to create the viewer; there's no need to write custom HTML.
|
|
250
260
|
|
|
@@ -332,7 +342,7 @@ This is optional, requires subagents, and most users won't need it. The human re
|
|
|
332
342
|
|
|
333
343
|
## Description Optimization
|
|
334
344
|
|
|
335
|
-
The description field in SKILL.md frontmatter is the primary mechanism that determines whether
|
|
345
|
+
The description field in SKILL.md frontmatter is the primary mechanism that determines whether pi invokes a skill. After creating or improving a skill, offer to optimize the description for better triggering accuracy.
|
|
336
346
|
|
|
337
347
|
### Step 1: Generate trigger eval queries
|
|
338
348
|
|
|
@@ -345,7 +355,7 @@ Create 20 eval queries — a mix of should-trigger and should-not-trigger. Save
|
|
|
345
355
|
]
|
|
346
356
|
```
|
|
347
357
|
|
|
348
|
-
The queries must be realistic and something a
|
|
358
|
+
The queries must be realistic and something a pi user would actually type. Not abstract requests, but requests that are concrete and specific and have a good amount of detail. For instance, file paths, personal context about the user's job or situation, column names and values, company names, URLs. A little bit of backstory. Some might be in lowercase or contain abbreviations or typos or casual speech. Use a mix of different lengths, and focus on edge cases rather than making them clear-cut (the user will get a chance to sign off on them).
|
|
349
359
|
|
|
350
360
|
Bad: `"Format this data"`, `"Extract text from PDF"`, `"Create a chart"`
|
|
351
361
|
|
|
@@ -387,17 +397,17 @@ python -m scripts.run_loop \
|
|
|
387
397
|
--verbose
|
|
388
398
|
```
|
|
389
399
|
|
|
390
|
-
Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences.
|
|
400
|
+
Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences. For pi this is a model pattern like `anthropic/claude-sonnet-4-20250514` or whatever you're running as (with an optional `:thinking` suffix).
|
|
391
401
|
|
|
392
402
|
While it runs, periodically tail the output to give the user updates on which iteration it's on and what the scores look like.
|
|
393
403
|
|
|
394
|
-
This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls
|
|
404
|
+
This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls pi to propose improvements based on what failed. It re-evaluates each new description on both train and test, iterating up to 5 times. When it's done, it opens an HTML report in the browser showing the results per iteration and returns JSON with `best_description` — selected by test score rather than train score to avoid overfitting.
|
|
395
405
|
|
|
396
406
|
### How skill triggering works
|
|
397
407
|
|
|
398
|
-
Understanding the triggering mechanism helps design better eval queries. Skills appear in
|
|
408
|
+
Understanding the triggering mechanism helps design better eval queries. Skills appear in pi's `available_skills` list (in the system prompt) with their name + description, and the agent decides whether to consult a skill based on that description. The important thing to know is that the agent only consults skills for tasks it can't easily handle on its own — simple, one-step queries like "read this PDF" may not trigger a skill even if the description matches perfectly, because the agent can handle them directly with basic tools. Complex, multi-step, or specialized queries reliably trigger skills when the description matches.
|
|
399
409
|
|
|
400
|
-
This means your eval queries should be substantive enough that
|
|
410
|
+
This means your eval queries should be substantive enough that the agent would actually benefit from consulting a skill. Simple queries like "read file X" are poor test cases — they won't trigger skills regardless of description quality.
|
|
401
411
|
|
|
402
412
|
### Step 4: Apply the result
|
|
403
413
|
|
|
@@ -417,45 +427,31 @@ After packaging, direct the user to the resulting `.skill` file path so they can
|
|
|
417
427
|
|
|
418
428
|
---
|
|
419
429
|
|
|
420
|
-
##
|
|
430
|
+
## Running without subagents
|
|
421
431
|
|
|
422
|
-
|
|
432
|
+
The core workflow is the same (draft → test → review → improve → repeat), but if your harness has no subagent tool, some mechanics change. Here's what to adapt:
|
|
423
433
|
|
|
424
434
|
**Running test cases**: No subagents means no parallel execution. For each test case, read the skill's SKILL.md, then follow its instructions to accomplish the test prompt yourself. Do them one at a time. This is less rigorous than independent subagents (you wrote the skill and you're also running it, so you have full context), but it's a useful sanity check — and the human review step compensates. Skip the baseline runs — just use the skill to complete the task as requested.
|
|
425
435
|
|
|
426
|
-
**Reviewing results**: If you can't open a browser (e.g.,
|
|
436
|
+
**Reviewing results**: If you can't open a browser (e.g., a remote server with no display), skip the browser reviewer entirely. Instead, present results directly in the conversation. For each test case, show the prompt and the output. If the output is a file the user needs to see (like a .docx or .xlsx), save it to the filesystem and tell them where it is so they can inspect it. Ask for feedback inline: "How does this look? Anything you'd change?"
|
|
427
437
|
|
|
428
438
|
**Benchmarking**: Skip the quantitative benchmarking — it relies on baseline comparisons which aren't meaningful without subagents. Focus on qualitative feedback from the user.
|
|
429
439
|
|
|
430
|
-
**The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem
|
|
440
|
+
**The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem.
|
|
431
441
|
|
|
432
|
-
**Description optimization**: This
|
|
442
|
+
**Description optimization**: This requires the `pi` CLI (specifically `pi -p`) which the eval scripts call via subprocess. It works as long as `pi` is on PATH. If it isn't available in the environment, skip description optimization.
|
|
433
443
|
|
|
434
444
|
**Blind comparison**: Requires subagents. Skip it.
|
|
435
445
|
|
|
436
|
-
**Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem.
|
|
446
|
+
**Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem.
|
|
437
447
|
|
|
438
448
|
**Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. In this case:
|
|
439
449
|
- **Preserve the original name.** Note the skill's directory name and `name` frontmatter field -- use them unchanged. E.g., if the installed skill is `research-helper`, output `research-helper.skill` (not `research-helper-v2`).
|
|
440
|
-
- **Copy to a writeable location before editing.**
|
|
450
|
+
- **Copy to a writeable location before editing.** An installed skill path may be read-only (e.g., a global `~/.pi/agent/skills/` install). Copy to `/tmp/skill-name/`, edit there, and package from the copy.
|
|
441
451
|
- **If packaging manually, stage in `/tmp/` first**, then copy to the output directory -- direct writes may fail due to permissions.
|
|
442
452
|
|
|
443
453
|
---
|
|
444
454
|
|
|
445
|
-
## Cowork-Specific Instructions
|
|
446
|
-
|
|
447
|
-
If you're in Cowork, the main things to know are:
|
|
448
|
-
|
|
449
|
-
- You have subagents, so the main workflow (spawn test cases in parallel, run baselines, grade, etc.) all works. (However, if you run into severe problems with timeouts, it's OK to run the test prompts in series rather than parallel.)
|
|
450
|
-
- You don't have a browser or display, so when generating the eval viewer, use `--static <output_path>` to write a standalone HTML file instead of starting a server. Then proffer a link that the user can click to open the HTML in their browser.
|
|
451
|
-
- For whatever reason, the Cowork setup seems to disincline Claude from generating the eval viewer after running the tests, so just to reiterate: whether you're in Cowork or in Claude Code, after running tests, you should always generate the eval viewer for the human to look at examples before revising the skill yourself and trying to make corrections, using `generate_review.py` (not writing your own boutique html code). Sorry in advance but I'm gonna go all caps here: GENERATE THE EVAL VIEWER *BEFORE* evaluating inputs yourself. You want to get them in front of the human ASAP!
|
|
452
|
-
- Feedback works differently: since there's no running server, the viewer's "Submit All Reviews" button will download `feedback.json` as a file. You can then read it from there (you may have to request access first).
|
|
453
|
-
- Packaging works — `package_skill.py` just needs Python and a filesystem.
|
|
454
|
-
- Description optimization (`run_loop.py` / `run_eval.py`) should work in Cowork just fine since it uses `claude -p` via subprocess, not a browser, but please save it until you've fully finished making the skill and the user agrees it's in good shape.
|
|
455
|
-
- **Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. Follow the update guidance in the claude.ai section above.
|
|
456
|
-
|
|
457
|
-
---
|
|
458
|
-
|
|
459
455
|
## Reference files
|
|
460
456
|
|
|
461
457
|
The agents/ directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent.
|
|
@@ -473,13 +469,13 @@ Repeating one more time the core loop here for emphasis:
|
|
|
473
469
|
|
|
474
470
|
- Figure out what the skill is about
|
|
475
471
|
- Draft or edit the skill
|
|
476
|
-
- Run
|
|
472
|
+
- Run pi-with-access-to-the-skill on test prompts
|
|
477
473
|
- With the user, evaluate the outputs:
|
|
478
474
|
- Create benchmark.json and run `eval-viewer/generate_review.py` to help the user review them
|
|
479
475
|
- Run quantitative evals
|
|
480
476
|
- Repeat until you and the user are satisfied
|
|
481
477
|
- Package the final skill and return it to the user.
|
|
482
478
|
|
|
483
|
-
Please add steps to your TodoList, if you have such a thing, to make sure you don't forget.
|
|
479
|
+
Please add steps to your TodoList, if you have such a thing, to make sure you don't forget. Specifically put "Create evals JSON and run `eval-viewer/generate_review.py` so human can review test cases" in your TodoList to make sure it happens.
|
|
484
480
|
|
|
485
481
|
Good luck!
|
|
@@ -545,7 +545,7 @@
|
|
|
545
545
|
<div class="header">
|
|
546
546
|
<div>
|
|
547
547
|
<h1>Eval Review: <span id="skill-name"></span></h1>
|
|
548
|
-
<div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into
|
|
548
|
+
<div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into your pi session.</div>
|
|
549
549
|
</div>
|
|
550
550
|
<div class="progress" id="progress"></div>
|
|
551
551
|
</div>
|
|
@@ -634,7 +634,7 @@
|
|
|
634
634
|
<div class="done-overlay" id="done-overlay">
|
|
635
635
|
<div class="done-card">
|
|
636
636
|
<h2>Review Complete</h2>
|
|
637
|
-
<p>Your feedback has been saved. Go back to your
|
|
637
|
+
<p>Your feedback has been saved. Go back to your pi session and tell the agent you're done reviewing.</p>
|
|
638
638
|
<div class="btn-row">
|
|
639
639
|
<button onclick="closeDoneDialog()">OK</button>
|
|
640
640
|
</div>
|
|
@@ -225,7 +225,7 @@ Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
|
|
|
225
225
|
"metadata": {
|
|
226
226
|
"skill_name": "pdf",
|
|
227
227
|
"skill_path": "/path/to/pdf",
|
|
228
|
-
"executor_model": "claude-sonnet-4-20250514",
|
|
228
|
+
"executor_model": "anthropic/claude-sonnet-4-20250514",
|
|
229
229
|
"analyzer_model": "most-capable-model",
|
|
230
230
|
"timestamp": "2026-01-15T10:30:00Z",
|
|
231
231
|
"evals_run": [1, 2, 3],
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -148,7 +148,7 @@ def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "")
|
|
|
148
148
|
<body>
|
|
149
149
|
<h1>""" + title_prefix + """Skill Description Optimization</h1>
|
|
150
150
|
<div class="explainer">
|
|
151
|
-
<strong>Optimizing your skill's description.</strong> This page updates automatically as
|
|
151
|
+
<strong>Optimizing your skill's description.</strong> This page updates automatically as pi tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, pi will apply the best-performing description to your skill.
|
|
152
152
|
</div>
|
|
153
153
|
"""]
|
|
154
154
|
|
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
"""Improve a skill description based on eval results.
|
|
3
3
|
|
|
4
4
|
Takes eval results (from run_eval.py) and generates an improved description
|
|
5
|
-
by calling `
|
|
6
|
-
|
|
5
|
+
by calling `pi -p` as a subprocess (uses the session's pi auth/config, no
|
|
6
|
+
separate API key needed).
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import argparse
|
|
10
10
|
import json
|
|
11
|
-
import os
|
|
12
11
|
import re
|
|
13
12
|
import subprocess
|
|
14
13
|
import sys
|
|
@@ -17,32 +16,26 @@ from pathlib import Path
|
|
|
17
16
|
from scripts.utils import parse_skill_md
|
|
18
17
|
|
|
19
18
|
|
|
20
|
-
def
|
|
21
|
-
"""Run `
|
|
19
|
+
def _call_pi(prompt: str, model: str | None, timeout: int = 300) -> str:
|
|
20
|
+
"""Run `pi -p` with the prompt on stdin and return the text response.
|
|
22
21
|
|
|
23
22
|
Prompt goes over stdin (not argv) because it embeds the full SKILL.md
|
|
24
23
|
body and can easily exceed comfortable argv length.
|
|
25
24
|
"""
|
|
26
|
-
cmd = ["
|
|
25
|
+
cmd = ["pi", "-p", "--mode", "text", "--no-session"]
|
|
27
26
|
if model:
|
|
28
27
|
cmd.extend(["--model", model])
|
|
29
28
|
|
|
30
|
-
# Remove CLAUDECODE env var to allow nesting claude -p inside a
|
|
31
|
-
# Claude Code session. The guard is for interactive terminal conflicts;
|
|
32
|
-
# programmatic subprocess usage is safe. Same pattern as run_eval.py.
|
|
33
|
-
env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
|
|
34
|
-
|
|
35
29
|
result = subprocess.run(
|
|
36
30
|
cmd,
|
|
37
31
|
input=prompt,
|
|
38
32
|
capture_output=True,
|
|
39
33
|
text=True,
|
|
40
|
-
env=env,
|
|
41
34
|
timeout=timeout,
|
|
42
35
|
)
|
|
43
36
|
if result.returncode != 0:
|
|
44
37
|
raise RuntimeError(
|
|
45
|
-
f"
|
|
38
|
+
f"pi -p exited {result.returncode}\nstderr: {result.stderr}"
|
|
46
39
|
)
|
|
47
40
|
return result.stdout
|
|
48
41
|
|
|
@@ -58,7 +51,7 @@ def improve_description(
|
|
|
58
51
|
log_dir: Path | None = None,
|
|
59
52
|
iteration: int | None = None,
|
|
60
53
|
) -> str:
|
|
61
|
-
"""Call
|
|
54
|
+
"""Call pi to improve the description based on eval results."""
|
|
62
55
|
failed_triggers = [
|
|
63
56
|
r for r in eval_results["results"]
|
|
64
57
|
if r["should_trigger"] and not r["pass"]
|
|
@@ -68,7 +61,6 @@ def improve_description(
|
|
|
68
61
|
if not r["should_trigger"] and not r["pass"]
|
|
69
62
|
]
|
|
70
63
|
|
|
71
|
-
# Build scores summary
|
|
72
64
|
train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
|
|
73
65
|
if test_results:
|
|
74
66
|
test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
|
|
@@ -76,9 +68,9 @@ def improve_description(
|
|
|
76
68
|
else:
|
|
77
69
|
scores_summary = f"Train: {train_score}"
|
|
78
70
|
|
|
79
|
-
prompt = f"""You are optimizing a skill description for a
|
|
71
|
+
prompt = f"""You are optimizing a skill description for a pi skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that the agent sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
|
|
80
72
|
|
|
81
|
-
The description appears in
|
|
73
|
+
The description appears in the agent's "available_skills" list. When a user sends a query, the agent decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
|
|
82
74
|
|
|
83
75
|
Here's the current description:
|
|
84
76
|
<current_description>
|
|
@@ -134,14 +126,14 @@ Concretely, your description should not be more than about 100-200 words, even i
|
|
|
134
126
|
Here are some tips that we've found to work well in writing these descriptions:
|
|
135
127
|
- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
|
|
136
128
|
- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
|
|
137
|
-
- The description competes with other skills for
|
|
129
|
+
- The description competes with other skills for the agent's attention — make it distinctive and immediately recognizable.
|
|
138
130
|
- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
|
|
139
131
|
|
|
140
132
|
I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
|
|
141
133
|
|
|
142
134
|
Please respond with only the new description text in <new_description> tags, nothing else."""
|
|
143
135
|
|
|
144
|
-
text =
|
|
136
|
+
text = _call_pi(prompt, model)
|
|
145
137
|
|
|
146
138
|
match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
|
|
147
139
|
description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
|
|
@@ -157,9 +149,8 @@ Please respond with only the new description text in <new_description> tags, not
|
|
|
157
149
|
|
|
158
150
|
# Safety net: the prompt already states the 1024-char hard limit, but if
|
|
159
151
|
# the model blew past it anyway, make one fresh single-turn call that
|
|
160
|
-
# quotes the too-long version and asks for a shorter rewrite. (
|
|
161
|
-
#
|
|
162
|
-
# inline the prior output into the new prompt instead.)
|
|
152
|
+
# quotes the too-long version and asks for a shorter rewrite. (pi -p is
|
|
153
|
+
# one-shot, so we inline the prior output into the new prompt instead.)
|
|
163
154
|
if len(description) > 1024:
|
|
164
155
|
shorten_prompt = (
|
|
165
156
|
f"{prompt}\n\n"
|
|
@@ -171,7 +162,7 @@ Please respond with only the new description text in <new_description> tags, not
|
|
|
171
162
|
f"important trigger words and intent coverage. Respond with only "
|
|
172
163
|
f"the new description in <new_description> tags."
|
|
173
164
|
)
|
|
174
|
-
shorten_text =
|
|
165
|
+
shorten_text = _call_pi(shorten_prompt, model)
|
|
175
166
|
match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
|
|
176
167
|
shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
|
|
177
168
|
|
|
@@ -229,7 +220,6 @@ def main():
|
|
|
229
220
|
if args.verbose:
|
|
230
221
|
print(f"Improved: {new_description}", file=sys.stderr)
|
|
231
222
|
|
|
232
|
-
# Output as JSON with both the new description and updated history
|
|
233
223
|
output = {
|
|
234
224
|
"description": new_description,
|
|
235
225
|
"history": history + [{
|