pi-ui-extend 0.1.38 → 0.1.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/app/app.d.ts +0 -1
  2. package/dist/app/app.js +28 -21
  3. package/dist/app/constants.js +1 -1
  4. package/dist/app/input/input-action-controller.d.ts +1 -0
  5. package/dist/app/input/input-action-controller.js +3 -0
  6. package/dist/app/input/input-controller.d.ts +1 -0
  7. package/dist/app/input/input-controller.js +40 -12
  8. package/dist/app/model/model-usage-status.js +4 -2
  9. package/dist/app/process.js +11 -0
  10. package/dist/app/rendering/conversation-tool-renderer.js +4 -6
  11. package/dist/app/session/request-history.js +2 -0
  12. package/dist/app/session/session-event-controller.d.ts +13 -0
  13. package/dist/app/session/session-event-controller.js +27 -0
  14. package/dist/app/session/tabs-controller.d.ts +8 -0
  15. package/dist/app/session/tabs-controller.js +37 -6
  16. package/dist/app/workspace/workspace-actions-controller.d.ts +1 -0
  17. package/dist/app/workspace/workspace-actions-controller.js +2 -1
  18. package/dist/bundled-extensions/terminal-bell/index.js +55 -1
  19. package/dist/config.js +1 -1
  20. package/dist/default-pix-config.js +1 -1
  21. package/dist/markdown-format.js +14 -25
  22. package/dist/terminal-width.d.ts +14 -0
  23. package/dist/terminal-width.js +31 -2
  24. package/dist/theme.js +2 -2
  25. package/external/pi-tools-suite/README.md +34 -9
  26. package/external/pi-tools-suite/package.json +3 -3
  27. package/external/pi-tools-suite/src/async-subagents/async-subagents.sample.jsonc +35 -21
  28. package/external/pi-tools-suite/src/async-subagents/commands.ts +1 -1
  29. package/external/pi-tools-suite/src/async-subagents/core/agent-strategy.ts +2 -2
  30. package/external/pi-tools-suite/src/async-subagents/core/config.ts +70 -12
  31. package/external/pi-tools-suite/src/async-subagents/core/routing.ts +1 -1
  32. package/external/pi-tools-suite/src/async-subagents/core/spawn.ts +1 -1
  33. package/external/pi-tools-suite/src/async-subagents/core/types.ts +1 -1
  34. package/external/pi-tools-suite/src/async-subagents/index.ts +6 -6
  35. package/external/pi-tools-suite/src/async-subagents/lib.ts +1 -1
  36. package/external/pi-tools-suite/src/async-subagents/tools/spawn.ts +4 -2
  37. package/external/pi-tools-suite/src/async-subagents/tools/subagents.ts +2 -2
  38. package/external/pi-tools-suite/src/{glm-coding-discipline → coding-discipline}/index.ts +17 -8
  39. package/external/pi-tools-suite/src/config.ts +1 -1
  40. package/external/pi-tools-suite/src/dcp/auto-compress.ts +368 -0
  41. package/external/pi-tools-suite/src/dcp/compress-tool.ts +3 -0
  42. package/external/pi-tools-suite/src/dcp/config.ts +23 -0
  43. package/external/pi-tools-suite/src/dcp/index.ts +112 -7
  44. package/external/pi-tools-suite/src/dcp/prompts.ts +8 -0
  45. package/external/pi-tools-suite/src/dcp/state.ts +41 -0
  46. package/external/pi-tools-suite/src/default-pi-tools-suite-config.ts +30 -22
  47. package/external/pi-tools-suite/src/index.ts +2 -1
  48. package/external/pi-tools-suite/src/session-name/index.ts +37 -0
  49. package/external/pi-tools-suite/src/tool-descriptions.ts +16 -4
  50. package/package.json +4 -4
  51. package/skills/skill-creator/SKILL.md +36 -40
  52. package/skills/skill-creator/eval-viewer/viewer.html +2 -2
  53. package/skills/skill-creator/references/schemas.md +1 -1
  54. package/skills/skill-creator/scripts/__pycache__/__init__.cpython-314.pyc +0 -0
  55. package/skills/skill-creator/scripts/__pycache__/aggregate_benchmark.cpython-314.pyc +0 -0
  56. package/skills/skill-creator/scripts/__pycache__/generate_report.cpython-314.pyc +0 -0
  57. package/skills/skill-creator/scripts/__pycache__/improve_description.cpython-314.pyc +0 -0
  58. package/skills/skill-creator/scripts/__pycache__/package_skill.cpython-314.pyc +0 -0
  59. package/skills/skill-creator/scripts/__pycache__/run_eval.cpython-314.pyc +0 -0
  60. package/skills/skill-creator/scripts/__pycache__/run_loop.cpython-314.pyc +0 -0
  61. package/skills/skill-creator/scripts/__pycache__/utils.cpython-314.pyc +0 -0
  62. package/skills/skill-creator/scripts/generate_report.py +1 -1
  63. package/skills/skill-creator/scripts/improve_description.py +14 -24
  64. package/skills/skill-creator/scripts/run_eval.py +89 -82
@@ -11,7 +11,7 @@ At a high level, the process of creating a skill goes like this:
11
11
 
12
12
  - Decide what you want the skill to do and roughly how it should do it
13
13
  - Write a draft of the skill
14
- - Create a few test prompts and run claude-with-access-to-the-skill on them
14
+ - Create a few test prompts and run pi-with-access-to-the-skill on them
15
15
  - Help the user evaluate the results both qualitatively and quantitatively
16
16
  - While the runs happen in the background, draft some quantitative evals if there aren't any (if there are some, you can either use as is or modify if you feel something needs to change about them). Then explain them to the user (or if they already existed, explain the ones that already exist)
17
17
  - Use the `eval-viewer/generate_review.py` script to show the user the results for them to look at, and also let them look at the quantitative metrics
@@ -31,7 +31,7 @@ Cool? Cool.
31
31
 
32
32
  ## Communicating with the user
33
33
 
34
- The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of Claude is inspiring plumbers to open up their terminals, parents and grandparents to google "how to install npm". On the other hand, the bulk of users are probably fairly computer-literate.
34
+ The skill creator is liable to be used by people across a wide range of familiarity with coding jargon. If you haven't heard (and how could you, it's only very recently that it started), there's a trend now where the power of AI coding agents like pi is inspiring plumbers to open up their terminals, parents and grandparents to google "how to install npm". On the other hand, the bulk of users are probably fairly computer-literate.
35
35
 
36
36
  So please pay attention to context cues to understand how to phrase your communication! In the default case, just to give you some idea:
37
37
 
@@ -48,7 +48,7 @@ It's OK to briefly explain terms if you're in doubt, and feel free to clarify te
48
48
 
49
49
  Start by understanding the user's intent. The current conversation might already contain a workflow the user wants to capture (e.g., they say "turn this into a skill"). If so, extract answers from the conversation history first — the tools used, the sequence of steps, corrections the user made, input/output formats observed. The user may need to fill the gaps, and should confirm before proceeding to the next step.
50
50
 
51
- 1. What should this skill enable Claude to do?
51
+ 1. What should this skill enable pi to do?
52
52
  2. When should this skill trigger? (what user phrases/contexts)
53
53
  3. What's the expected output format?
54
54
  4. Should we set up test cases to verify the skill works? Skills with objectively verifiable outputs (file transforms, data extraction, code generation, fixed workflow steps) benefit from test cases. Skills with subjective outputs (writing style, art) often don't need them. Suggest the appropriate default based on the skill type, but let the user decide.
@@ -63,8 +63,8 @@ Check available MCPs - if useful for research (searching docs, finding similar s
63
63
 
64
64
  Based on the user interview, fill in these components:
65
65
 
66
- - **name**: Skill identifier
67
- - **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently Claude has a tendency to "undertrigger" skills -- to not use them when they'd be useful. To combat this, please make the skill descriptions a little bit "pushy". So for instance, instead of "How to build a simple fast dashboard to display internal Anthropic data.", you might write "How to build a simple fast dashboard to display internal Anthropic data. Make sure to use this skill whenever the user mentions dashboards, data visualization, internal metrics, or wants to display any kind of company data, even if they don't explicitly ask for a 'dashboard.'"
66
+ - **name**: Skill identifier (lowercase, hyphens, a-z0-9 — see pi's skill validation rules)
67
+ - **description**: When to trigger, what it does. This is the primary triggering mechanism - include both what the skill does AND specific contexts for when to use it. All "when to use" info goes here, not in the body. Note: currently models have a tendency to "undertrigger" skills -- to not use them when they'd be useful. To combat this, please make the skill descriptions a little bit "pushy". So for instance, instead of "How to build a simple fast dashboard to display internal Anthropic data.", you might write "How to build a simple fast dashboard to display internal Anthropic data. Make sure to use this skill whenever the user mentions dashboards, data visualization, internal metrics, or wants to display any kind of company data, even if they don't explicitly ask for a 'dashboard.'"
68
68
  - **compatibility**: Required tools, dependencies (optional, rarely needed)
69
69
  - **the rest of the skill :)**
70
70
 
@@ -106,7 +106,15 @@ cloud-deploy/
106
106
  ├── gcp.md
107
107
  └── azure.md
108
108
  ```
109
- Claude reads only the relevant reference file.
109
+ The agent reads only the relevant reference file.
110
+
111
+ #### How pi discovers and loads skills
112
+
113
+ This matters both for writing skills and for testing them. Pi scans skill locations at startup (`~/.pi/agent/skills/`, `~/.agents/skills/`, project `.pi/skills/` and `.agents/skills/`, settings `skills` arrays, and the `--skill <path>` CLI flag). It extracts each skill's `name` + `description` and lists them in the system prompt as `available_skills`. When a task matches a skill, the agent uses the `read` tool to load the full `SKILL.md` on demand — that's progressive disclosure. Skills also register as `/skill:name` commands the user can invoke directly to force-load them.
114
+
115
+ Two practical consequences for this skill:
116
+ - **Where to install a finished skill**: global skills go in `~/.pi/agent/skills/<skill-name>/`; project skills go in `.pi/skills/<skill-name>/` (only loaded once the project is trusted).
117
+ - **How triggering is measured**: the eval scripts in `scripts/` test triggering by watching whether pi actually performs that `read` on the skill's `SKILL.md`. They invoke `pi -p --mode json --skill <temp-skill-dir>` so the description under test is evaluated in isolation (with `--no-skills` to suppress all other discovered skills).
110
118
 
111
119
  #### Principle of Lack of Surprise
112
120
 
@@ -168,9 +176,9 @@ Put results in `<skill-name>-workspace/` as a sibling to the skill directory. Wi
168
176
 
169
177
  ### Step 1: Spawn all runs (with-skill AND baseline) in the same turn
170
178
 
171
- For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time.
179
+ For each test case, spawn two subagents in the same turn — one with the skill, one without. This is important: don't spawn the with-skill runs first and then come back for baselines later. Launch everything at once so it all finishes around the same time. (If your harness has no subagent tool, see "Running without subagents" below — run them inline, one at a time, and skip baselines.)
172
180
 
173
- **With-skill run:**
181
+ **With-skill run** (give the subagent a task like this):
174
182
 
175
183
  ```
176
184
  Execute this task:
@@ -179,6 +187,8 @@ Execute this task:
179
187
  - Input files: <eval files if any, or "none">
180
188
  - Save outputs to: <workspace>/iteration-<N>/eval-<ID>/with_skill/outputs/
181
189
  - Outputs to save: <what the user cares about — e.g., "the .docx file", "the final CSV">
190
+
191
+ Load the skill at <path-to-skill> (read its SKILL.md) before starting, and follow it.
182
192
  ```
183
193
 
184
194
  **Baseline run** (same prompt, but the baseline depends on context):
@@ -206,7 +216,7 @@ Update the `eval_metadata.json` files and `evals/evals.json` with the assertions
206
216
 
207
217
  ### Step 3: As runs complete, capture timing data
208
218
 
209
- When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms`. Save this data immediately to `timing.json` in the run directory:
219
+ When each subagent task completes, you receive a notification containing `total_tokens` and `duration_ms` (or equivalent usage info from the result). Save this data immediately to `timing.json` in the run directory:
210
220
 
211
221
  ```json
212
222
  {
@@ -216,7 +226,7 @@ When each subagent task completes, you receive a notification containing `total_
216
226
  }
217
227
  ```
218
228
 
219
- This is the only opportunity to capture this data it comes through the task notification and isn't persisted elsewhere. Process each notification as it arrives rather than trying to batch them.
229
+ If your harness's completion notification carries a different shape, capture whatever token/duration fields it provides. Process each notification as it arrives rather than trying to batch them.
220
230
 
221
231
  ### Step 4: Grade, aggregate, and launch the viewer
222
232
 
@@ -244,7 +254,7 @@ Put each with_skill version before its baseline counterpart.
244
254
  ```
245
255
  For iteration 2+, also pass `--previous-workspace <workspace>/iteration-<N-1>`.
246
256
 
247
- **Cowork / headless environments:** If `webbrowser.open()` is not available or the environment has no display, use `--static <output_path>` to write a standalone HTML file instead of starting a server. Feedback will be downloaded as a `feedback.json` file when the user clicks "Submit All Reviews". After download, copy `feedback.json` into the workspace directory for the next iteration to pick up.
257
+ **Headless / no-display environments:** If `webbrowser.open()` is not available or the environment has no display (remote server, CI), use `--static <output_path>` to write a standalone HTML file instead of starting a server. Feedback will be downloaded as a `feedback.json` file when the user clicks "Submit All Reviews". After download, copy `feedback.json` into the workspace directory for the next iteration to pick up.
248
258
 
249
259
  Note: please use generate_review.py to create the viewer; there's no need to write custom HTML.
250
260
 
@@ -332,7 +342,7 @@ This is optional, requires subagents, and most users won't need it. The human re
332
342
 
333
343
  ## Description Optimization
334
344
 
335
- The description field in SKILL.md frontmatter is the primary mechanism that determines whether Claude invokes a skill. After creating or improving a skill, offer to optimize the description for better triggering accuracy.
345
+ The description field in SKILL.md frontmatter is the primary mechanism that determines whether pi invokes a skill. After creating or improving a skill, offer to optimize the description for better triggering accuracy.
336
346
 
337
347
  ### Step 1: Generate trigger eval queries
338
348
 
@@ -345,7 +355,7 @@ Create 20 eval queries — a mix of should-trigger and should-not-trigger. Save
345
355
  ]
346
356
  ```
347
357
 
348
- The queries must be realistic and something a Claude Code or Claude.ai user would actually type. Not abstract requests, but requests that are concrete and specific and have a good amount of detail. For instance, file paths, personal context about the user's job or situation, column names and values, company names, URLs. A little bit of backstory. Some might be in lowercase or contain abbreviations or typos or casual speech. Use a mix of different lengths, and focus on edge cases rather than making them clear-cut (the user will get a chance to sign off on them).
358
+ The queries must be realistic and something a pi user would actually type. Not abstract requests, but requests that are concrete and specific and have a good amount of detail. For instance, file paths, personal context about the user's job or situation, column names and values, company names, URLs. A little bit of backstory. Some might be in lowercase or contain abbreviations or typos or casual speech. Use a mix of different lengths, and focus on edge cases rather than making them clear-cut (the user will get a chance to sign off on them).
349
359
 
350
360
  Bad: `"Format this data"`, `"Extract text from PDF"`, `"Create a chart"`
351
361
 
@@ -387,17 +397,17 @@ python -m scripts.run_loop \
387
397
  --verbose
388
398
  ```
389
399
 
390
- Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences.
400
+ Use the model ID from your system prompt (the one powering the current session) so the triggering test matches what the user actually experiences. For pi this is a model pattern like `anthropic/claude-sonnet-4-20250514` or whatever you're running as (with an optional `:thinking` suffix).
391
401
 
392
402
  While it runs, periodically tail the output to give the user updates on which iteration it's on and what the scores look like.
393
403
 
394
- This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls Claude to propose improvements based on what failed. It re-evaluates each new description on both train and test, iterating up to 5 times. When it's done, it opens an HTML report in the browser showing the results per iteration and returns JSON with `best_description` — selected by test score rather than train score to avoid overfitting.
404
+ This handles the full optimization loop automatically. It splits the eval set into 60% train and 40% held-out test, evaluates the current description (running each query 3 times to get a reliable trigger rate), then calls pi to propose improvements based on what failed. It re-evaluates each new description on both train and test, iterating up to 5 times. When it's done, it opens an HTML report in the browser showing the results per iteration and returns JSON with `best_description` — selected by test score rather than train score to avoid overfitting.
395
405
 
396
406
  ### How skill triggering works
397
407
 
398
- Understanding the triggering mechanism helps design better eval queries. Skills appear in Claude's `available_skills` list with their name + description, and Claude decides whether to consult a skill based on that description. The important thing to know is that Claude only consults skills for tasks it can't easily handle on its own — simple, one-step queries like "read this PDF" may not trigger a skill even if the description matches perfectly, because Claude can handle them directly with basic tools. Complex, multi-step, or specialized queries reliably trigger skills when the description matches.
408
+ Understanding the triggering mechanism helps design better eval queries. Skills appear in pi's `available_skills` list (in the system prompt) with their name + description, and the agent decides whether to consult a skill based on that description. The important thing to know is that the agent only consults skills for tasks it can't easily handle on its own — simple, one-step queries like "read this PDF" may not trigger a skill even if the description matches perfectly, because the agent can handle them directly with basic tools. Complex, multi-step, or specialized queries reliably trigger skills when the description matches.
399
409
 
400
- This means your eval queries should be substantive enough that Claude would actually benefit from consulting a skill. Simple queries like "read file X" are poor test cases — they won't trigger skills regardless of description quality.
410
+ This means your eval queries should be substantive enough that the agent would actually benefit from consulting a skill. Simple queries like "read file X" are poor test cases — they won't trigger skills regardless of description quality.
401
411
 
402
412
  ### Step 4: Apply the result
403
413
 
@@ -417,45 +427,31 @@ After packaging, direct the user to the resulting `.skill` file path so they can
417
427
 
418
428
  ---
419
429
 
420
- ## Claude.ai-specific instructions
430
+ ## Running without subagents
421
431
 
422
- In Claude.ai, the core workflow is the same (draft → test → review → improve → repeat), but because Claude.ai doesn't have subagents, some mechanics change. Here's what to adapt:
432
+ The core workflow is the same (draft → test → review → improve → repeat), but if your harness has no subagent tool, some mechanics change. Here's what to adapt:
423
433
 
424
434
  **Running test cases**: No subagents means no parallel execution. For each test case, read the skill's SKILL.md, then follow its instructions to accomplish the test prompt yourself. Do them one at a time. This is less rigorous than independent subagents (you wrote the skill and you're also running it, so you have full context), but it's a useful sanity check — and the human review step compensates. Skip the baseline runs — just use the skill to complete the task as requested.
425
435
 
426
- **Reviewing results**: If you can't open a browser (e.g., Claude.ai's VM has no display, or you're on a remote server), skip the browser reviewer entirely. Instead, present results directly in the conversation. For each test case, show the prompt and the output. If the output is a file the user needs to see (like a .docx or .xlsx), save it to the filesystem and tell them where it is so they can download and inspect it. Ask for feedback inline: "How does this look? Anything you'd change?"
436
+ **Reviewing results**: If you can't open a browser (e.g., a remote server with no display), skip the browser reviewer entirely. Instead, present results directly in the conversation. For each test case, show the prompt and the output. If the output is a file the user needs to see (like a .docx or .xlsx), save it to the filesystem and tell them where it is so they can inspect it. Ask for feedback inline: "How does this look? Anything you'd change?"
427
437
 
428
438
  **Benchmarking**: Skip the quantitative benchmarking — it relies on baseline comparisons which aren't meaningful without subagents. Focus on qualitative feedback from the user.
429
439
 
430
- **The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem if you have one.
440
+ **The iteration loop**: Same as before — improve the skill, rerun the test cases, ask for feedback — just without the browser reviewer in the middle. You can still organize results into iteration directories on the filesystem.
431
441
 
432
- **Description optimization**: This section requires the `claude` CLI tool (specifically `claude -p`) which is only available in Claude Code. Skip it if you're on Claude.ai.
442
+ **Description optimization**: This requires the `pi` CLI (specifically `pi -p`) which the eval scripts call via subprocess. It works as long as `pi` is on PATH. If it isn't available in the environment, skip description optimization.
433
443
 
434
444
  **Blind comparison**: Requires subagents. Skip it.
435
445
 
436
- **Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem. On Claude.ai, you can run it and the user can download the resulting `.skill` file.
446
+ **Packaging**: The `package_skill.py` script works anywhere with Python and a filesystem.
437
447
 
438
448
  **Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. In this case:
439
449
  - **Preserve the original name.** Note the skill's directory name and `name` frontmatter field -- use them unchanged. E.g., if the installed skill is `research-helper`, output `research-helper.skill` (not `research-helper-v2`).
440
- - **Copy to a writeable location before editing.** The installed skill path may be read-only. Copy to `/tmp/skill-name/`, edit there, and package from the copy.
450
+ - **Copy to a writeable location before editing.** An installed skill path may be read-only (e.g., a global `~/.pi/agent/skills/` install). Copy to `/tmp/skill-name/`, edit there, and package from the copy.
441
451
  - **If packaging manually, stage in `/tmp/` first**, then copy to the output directory -- direct writes may fail due to permissions.
442
452
 
443
453
  ---
444
454
 
445
- ## Cowork-Specific Instructions
446
-
447
- If you're in Cowork, the main things to know are:
448
-
449
- - You have subagents, so the main workflow (spawn test cases in parallel, run baselines, grade, etc.) all works. (However, if you run into severe problems with timeouts, it's OK to run the test prompts in series rather than parallel.)
450
- - You don't have a browser or display, so when generating the eval viewer, use `--static <output_path>` to write a standalone HTML file instead of starting a server. Then proffer a link that the user can click to open the HTML in their browser.
451
- - For whatever reason, the Cowork setup seems to disincline Claude from generating the eval viewer after running the tests, so just to reiterate: whether you're in Cowork or in Claude Code, after running tests, you should always generate the eval viewer for the human to look at examples before revising the skill yourself and trying to make corrections, using `generate_review.py` (not writing your own boutique html code). Sorry in advance but I'm gonna go all caps here: GENERATE THE EVAL VIEWER *BEFORE* evaluating inputs yourself. You want to get them in front of the human ASAP!
452
- - Feedback works differently: since there's no running server, the viewer's "Submit All Reviews" button will download `feedback.json` as a file. You can then read it from there (you may have to request access first).
453
- - Packaging works — `package_skill.py` just needs Python and a filesystem.
454
- - Description optimization (`run_loop.py` / `run_eval.py`) should work in Cowork just fine since it uses `claude -p` via subprocess, not a browser, but please save it until you've fully finished making the skill and the user agrees it's in good shape.
455
- - **Updating an existing skill**: The user might be asking you to update an existing skill, not create a new one. Follow the update guidance in the claude.ai section above.
456
-
457
- ---
458
-
459
455
  ## Reference files
460
456
 
461
457
  The agents/ directory contains instructions for specialized subagents. Read them when you need to spawn the relevant subagent.
@@ -473,13 +469,13 @@ Repeating one more time the core loop here for emphasis:
473
469
 
474
470
  - Figure out what the skill is about
475
471
  - Draft or edit the skill
476
- - Run claude-with-access-to-the-skill on test prompts
472
+ - Run pi-with-access-to-the-skill on test prompts
477
473
  - With the user, evaluate the outputs:
478
474
  - Create benchmark.json and run `eval-viewer/generate_review.py` to help the user review them
479
475
  - Run quantitative evals
480
476
  - Repeat until you and the user are satisfied
481
477
  - Package the final skill and return it to the user.
482
478
 
483
- Please add steps to your TodoList, if you have such a thing, to make sure you don't forget. If you're in Cowork, please specifically put "Create evals JSON and run `eval-viewer/generate_review.py` so human can review test cases" in your TodoList to make sure it happens.
479
+ Please add steps to your TodoList, if you have such a thing, to make sure you don't forget. Specifically put "Create evals JSON and run `eval-viewer/generate_review.py` so human can review test cases" in your TodoList to make sure it happens.
484
480
 
485
481
  Good luck!
@@ -545,7 +545,7 @@
545
545
  <div class="header">
546
546
  <div>
547
547
  <h1>Eval Review: <span id="skill-name"></span></h1>
548
- <div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.</div>
548
+ <div class="instructions">Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into your pi session.</div>
549
549
  </div>
550
550
  <div class="progress" id="progress"></div>
551
551
  </div>
@@ -634,7 +634,7 @@
634
634
  <div class="done-overlay" id="done-overlay">
635
635
  <div class="done-card">
636
636
  <h2>Review Complete</h2>
637
- <p>Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.</p>
637
+ <p>Your feedback has been saved. Go back to your pi session and tell the agent you're done reviewing.</p>
638
638
  <div class="btn-row">
639
639
  <button onclick="closeDoneDialog()">OK</button>
640
640
  </div>
@@ -225,7 +225,7 @@ Output from Benchmark mode. Located at `benchmarks/<timestamp>/benchmark.json`.
225
225
  "metadata": {
226
226
  "skill_name": "pdf",
227
227
  "skill_path": "/path/to/pdf",
228
- "executor_model": "claude-sonnet-4-20250514",
228
+ "executor_model": "anthropic/claude-sonnet-4-20250514",
229
229
  "analyzer_model": "most-capable-model",
230
230
  "timestamp": "2026-01-15T10:30:00Z",
231
231
  "evals_run": [1, 2, 3],
@@ -148,7 +148,7 @@ def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "")
148
148
  <body>
149
149
  <h1>""" + title_prefix + """Skill Description Optimization</h1>
150
150
  <div class="explainer">
151
- <strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
151
+ <strong>Optimizing your skill's description.</strong> This page updates automatically as pi tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, pi will apply the best-performing description to your skill.
152
152
  </div>
153
153
  """]
154
154
 
@@ -2,13 +2,12 @@
2
2
  """Improve a skill description based on eval results.
3
3
 
4
4
  Takes eval results (from run_eval.py) and generates an improved description
5
- by calling `claude -p` as a subprocess (same auth pattern as run_eval.py
6
- uses the session's Claude Code auth, no separate ANTHROPIC_API_KEY needed).
5
+ by calling `pi -p` as a subprocess (uses the session's pi auth/config, no
6
+ separate API key needed).
7
7
  """
8
8
 
9
9
  import argparse
10
10
  import json
11
- import os
12
11
  import re
13
12
  import subprocess
14
13
  import sys
@@ -17,32 +16,26 @@ from pathlib import Path
17
16
  from scripts.utils import parse_skill_md
18
17
 
19
18
 
20
- def _call_claude(prompt: str, model: str | None, timeout: int = 300) -> str:
21
- """Run `claude -p` with the prompt on stdin and return the text response.
19
+ def _call_pi(prompt: str, model: str | None, timeout: int = 300) -> str:
20
+ """Run `pi -p` with the prompt on stdin and return the text response.
22
21
 
23
22
  Prompt goes over stdin (not argv) because it embeds the full SKILL.md
24
23
  body and can easily exceed comfortable argv length.
25
24
  """
26
- cmd = ["claude", "-p", "--output-format", "text"]
25
+ cmd = ["pi", "-p", "--mode", "text", "--no-session"]
27
26
  if model:
28
27
  cmd.extend(["--model", model])
29
28
 
30
- # Remove CLAUDECODE env var to allow nesting claude -p inside a
31
- # Claude Code session. The guard is for interactive terminal conflicts;
32
- # programmatic subprocess usage is safe. Same pattern as run_eval.py.
33
- env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
34
-
35
29
  result = subprocess.run(
36
30
  cmd,
37
31
  input=prompt,
38
32
  capture_output=True,
39
33
  text=True,
40
- env=env,
41
34
  timeout=timeout,
42
35
  )
43
36
  if result.returncode != 0:
44
37
  raise RuntimeError(
45
- f"claude -p exited {result.returncode}\nstderr: {result.stderr}"
38
+ f"pi -p exited {result.returncode}\nstderr: {result.stderr}"
46
39
  )
47
40
  return result.stdout
48
41
 
@@ -58,7 +51,7 @@ def improve_description(
58
51
  log_dir: Path | None = None,
59
52
  iteration: int | None = None,
60
53
  ) -> str:
61
- """Call Claude to improve the description based on eval results."""
54
+ """Call pi to improve the description based on eval results."""
62
55
  failed_triggers = [
63
56
  r for r in eval_results["results"]
64
57
  if r["should_trigger"] and not r["pass"]
@@ -68,7 +61,6 @@ def improve_description(
68
61
  if not r["should_trigger"] and not r["pass"]
69
62
  ]
70
63
 
71
- # Build scores summary
72
64
  train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
73
65
  if test_results:
74
66
  test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
@@ -76,9 +68,9 @@ def improve_description(
76
68
  else:
77
69
  scores_summary = f"Train: {train_score}"
78
70
 
79
- prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
71
+ prompt = f"""You are optimizing a skill description for a pi skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that the agent sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
80
72
 
81
- The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
73
+ The description appears in the agent's "available_skills" list. When a user sends a query, the agent decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
82
74
 
83
75
  Here's the current description:
84
76
  <current_description>
@@ -134,14 +126,14 @@ Concretely, your description should not be more than about 100-200 words, even i
134
126
  Here are some tips that we've found to work well in writing these descriptions:
135
127
  - The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
136
128
  - The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
137
- - The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable.
129
+ - The description competes with other skills for the agent's attention — make it distinctive and immediately recognizable.
138
130
  - If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
139
131
 
140
132
  I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
141
133
 
142
134
  Please respond with only the new description text in <new_description> tags, nothing else."""
143
135
 
144
- text = _call_claude(prompt, model)
136
+ text = _call_pi(prompt, model)
145
137
 
146
138
  match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
147
139
  description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
@@ -157,9 +149,8 @@ Please respond with only the new description text in <new_description> tags, not
157
149
 
158
150
  # Safety net: the prompt already states the 1024-char hard limit, but if
159
151
  # the model blew past it anyway, make one fresh single-turn call that
160
- # quotes the too-long version and asks for a shorter rewrite. (The old
161
- # SDK path did this as a true multi-turn; `claude -p` is one-shot, so we
162
- # inline the prior output into the new prompt instead.)
152
+ # quotes the too-long version and asks for a shorter rewrite. (pi -p is
153
+ # one-shot, so we inline the prior output into the new prompt instead.)
163
154
  if len(description) > 1024:
164
155
  shorten_prompt = (
165
156
  f"{prompt}\n\n"
@@ -171,7 +162,7 @@ Please respond with only the new description text in <new_description> tags, not
171
162
  f"important trigger words and intent coverage. Respond with only "
172
163
  f"the new description in <new_description> tags."
173
164
  )
174
- shorten_text = _call_claude(shorten_prompt, model)
165
+ shorten_text = _call_pi(shorten_prompt, model)
175
166
  match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
176
167
  shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
177
168
 
@@ -229,7 +220,6 @@ def main():
229
220
  if args.verbose:
230
221
  print(f"Improved: {new_description}", file=sys.stderr)
231
222
 
232
- # Output as JSON with both the new description and updated history
233
223
  output = {
234
224
  "description": new_description,
235
225
  "history": history + [{