pi-crew 0.10.2 → 0.10.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +249 -0
  2. package/dist/index.mjs +98 -307
  3. package/package.json +2 -1
  4. package/schema.json +11 -0
  5. package/skills/real-test-pi-crew/REPORT-TEMPLATE.md +6 -2
  6. package/skills/real-test-pi-crew/SKILL.md +278 -79
  7. package/src/config/config-merge.ts +11 -1
  8. package/src/config/config-validation.ts +40 -1
  9. package/src/config/config.ts +28 -6
  10. package/src/config/defaults.ts +35 -10
  11. package/src/config/env-vars.ts +27 -2
  12. package/src/config/types.ts +36 -0
  13. package/src/extension/registration/lifecycle-handlers.ts +40 -9
  14. package/src/extension/registration/team-tool.ts +53 -5
  15. package/src/extension/team-tool/doctor.ts +364 -7
  16. package/src/extension/team-tool/handle-settings.ts +19 -0
  17. package/src/extension/team-tool/inspect.ts +10 -2
  18. package/src/extension/team-tool/status.ts +7 -0
  19. package/src/extension/team-tool.ts +35 -2
  20. package/src/hooks/registry.ts +59 -56
  21. package/src/prompt/inbox-poll.ts +90 -0
  22. package/src/prompt/message-tool.ts +166 -0
  23. package/src/prompt/prompt-runtime.ts +201 -18
  24. package/src/prompt/surface-worker.ts +720 -0
  25. package/src/prompt/worker-events-channel.ts +49 -3
  26. package/src/runtime/async-runner.ts +29 -1
  27. package/src/runtime/background-runner.ts +13 -7
  28. package/src/runtime/broker/broker-issuer.ts +27 -2
  29. package/src/runtime/broker/crew-broker-tokens.ts +56 -4
  30. package/src/runtime/broker/crew-broker.ts +261 -41
  31. package/src/runtime/child-pi/child-pi-spawn.ts +23 -9
  32. package/src/runtime/child-pi/child-pi-streams.ts +9 -1
  33. package/src/runtime/child-pi/child-pi.ts +353 -5
  34. package/src/runtime/crew-agent-records.ts +13 -1
  35. package/src/runtime/dispatch-batch.ts +12 -1
  36. package/src/runtime/event-log-tail-source.ts +374 -0
  37. package/src/runtime/finalize-run.ts +4 -0
  38. package/src/runtime/live-session/live-agent-manager.ts +34 -1
  39. package/src/runtime/live-session/live-control-realtime.ts +10 -0
  40. package/src/runtime/live-session/live-session-runtime.ts +47 -27
  41. package/src/runtime/manifest-cache.ts +128 -17
  42. package/src/runtime/model/pi-args.ts +54 -65
  43. package/src/runtime/output/sidechain-output.ts +61 -6
  44. package/src/runtime/process/proc-stat.ts +46 -0
  45. package/src/runtime/process/zombie-scanner.ts +32 -19
  46. package/src/runtime/spawn-policy.ts +27 -41
  47. package/src/runtime/surface/degrade.ts +776 -0
  48. package/src/runtime/surface/herdr-provider.ts +546 -0
  49. package/src/runtime/surface/launch-script.ts +172 -0
  50. package/src/runtime/surface/resolve-surface.ts +274 -0
  51. package/src/runtime/surface/surface-provider.ts +129 -0
  52. package/src/runtime/surface/surface-spawn.ts +475 -0
  53. package/src/runtime/surface/tmux-provider.ts +400 -0
  54. package/src/runtime/task-runner/child-executor.ts +47 -0
  55. package/src/runtime/task-runner/post-execution.ts +57 -2
  56. package/src/runtime/task-runner/prompt-builder.ts +1 -0
  57. package/src/runtime/task-runner/retrieval-orchestrator.ts +191 -56
  58. package/src/runtime/task-runner/state-helpers.ts +54 -30
  59. package/src/runtime/task-runner.ts +4 -2
  60. package/src/runtime/team-runner.ts +101 -0
  61. package/src/schema/config-schema.ts +24 -0
  62. package/src/state/atomic-write.ts +219 -40
  63. package/src/state/coordination/locks.ts +7 -5
  64. package/src/state/coordination/mailbox.ts +56 -10
  65. package/src/state/event-log/cursor.ts +413 -23
  66. package/src/state/event-log/event-log.ts +120 -113
  67. package/src/state/event-log/sequence-cache.ts +21 -3
  68. package/src/state/stores/state-store.ts +98 -6
  69. package/src/state/types.ts +51 -0
  70. package/src/ui/inline-panel/agent-pane.ts +3 -0
  71. package/src/ui/render-diff.ts +16 -8
  72. package/src/ui/run-dashboard.ts +87 -42
  73. package/src/ui/run-event-bus.ts +10 -1
  74. package/src/ui/run-snapshot-cache.ts +83 -35
  75. package/src/ui/transcript-cache.ts +101 -13
  76. package/src/ui/transcript-viewer.ts +92 -24
  77. package/src/ui/widget/index.ts +32 -8
  78. package/src/utils/visual.ts +43 -0
  79. package/src/worktree/worktree-manager.ts +65 -4
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: real-test-pi-crew
3
- description: "End-to-end verification for pi-crew changes: fast critical tests, 3-path kill-switch proof, bundle md5 sync, live TUI probing, smoke team runs, and a live feature-action battery (team tool + subagent tools)."
3
+ description: "End-to-end verification for pi-crew changes: fast critical tests, 3-path kill-switch proof, bundle md5 sync, live TUI probing, smoke team runs, a live feature-action battery (team tool + subagent tools), and a surface-mode battery (workers in real tmux/herdr panes, degrade-to-headless)."
4
4
  origin: pi-crew
5
5
  triggers:
6
6
  - "test the change"
@@ -22,32 +22,44 @@ triggers:
22
22
  - "schema fix"
23
23
  - "feature battery"
24
24
  - "full features of pi-crew"
25
- - "tier 1 / tier 2 / tier 3 / tier 4 / tier 5 / tier 6 / tier 7 / tier 8 / tier 9"
25
+ - "surface test"
26
+ - "surface mode"
27
+ - "pane test"
28
+ - "herdr test"
29
+ - "degrade test"
30
+ - "worker in pane"
31
+ - "message tool test"
32
+ - "delegate tool test"
33
+ - "ask tool test"
34
+ - "nested agent test"
35
+ - "tier 1 / tier 2 / tier 3 / tier 4 / tier 5 / tier 6 / tier 7 / tier 8 / tier 9 / tier 10"
26
36
  ---
27
37
 
28
38
  # real-test-pi-crew
29
39
 
30
40
  End-to-end verification discipline for pi-crew changes. Distilled from the broker Phase-4 rollout (commits `1cb2dca` → `d599578` → `612e18b` → `4186284`, July 2026). The pain this skill prevents: shipping code that compiles + unit-tests-green but breaks in the user's live Pi session, or hangs the verifier worker.
31
41
 
32
- **When to use**: after any change to `src/runtime/crew-broker*.ts`, `src/ui/`, `src/config/`, `src/extension/registration/lifecycle-handlers.ts`, `src/runtime/child-pi-spawn.ts`, `src/runtime/plan-templates.ts`, `src/schema/team-tool-schema.ts` (or any `Type.Unsafe({...})` schema definition), `src/extension/registration/team-tool.ts`, `workflows/*.workflow.md`, or before any commit touching these paths. Schema changes additionally require Tier 9 (feature battery) because the team tool's TypeBox schema is validated by pi-ai BEFORE the handler runs — a too-strict or malformed schema breaks every action silently.
42
+ **When to use**: after any change to `src/runtime/broker/*.ts` (broker + tokens + issuer), `src/ui/`, `src/config/`, `src/extension/registration/lifecycle-handlers.ts`, `src/runtime/child-pi/*.ts` (worker spawn/kill/steering), `src/runtime/surface/*.ts` (MuxSurface providers, degrade, launch script), `src/prompt/*.ts` (worker-side tools: ask / message / delegate / surface-worker recorder), `src/runtime/goal-workflow/plan-templates.ts`, `src/runtime/team-runner.ts` or `src/runtime/task-runner/**` (scheduler / execution — Tier 7 smoke), `src/state/**` (durable state — Tier 7 + 9a events/status), `src/runtime/live-session/**` + `src/runtime/custom-tools/*` (live-session mode + worker custom tools), `src/schema/team-tool-schema.ts` (or any `Type.Unsafe({...})` schema definition), `src/extension/registration/team-tool.ts`, `workflows/*.workflow.md`, or before any commit touching these paths. Schema changes additionally require Tier 9 (feature battery) because the team tool's TypeBox schema is validated by pi-ai BEFORE the handler runs — a too-strict or malformed schema breaks every action silently. Surface changes additionally require Tier 10 (surface-mode battery) because surface is fail-closed: every failure degrades to headless and the run still goes green — only pane-level evidence proves the panes engaged.
43
+
44
+ > **Path map (2026-08-26 reorg + A1)**: `src/runtime/crew-broker*.ts` → `src/runtime/broker/`; `src/runtime/child-pi*.ts` → `src/runtime/child-pi/`; `src/runtime/plan-templates.ts` (flat) → `src/runtime/goal-workflow/plan-templates.ts`; NEW dirs `src/runtime/surface/` and `src/prompt/`. Test files moved with them (`test/unit/crew-broker-*.test.ts` → `test/unit/runtime/broker/`, `test/unit/keybinding-map.parity.test.ts` → `test/unit/ui/`, ...).
33
45
 
34
46
  ## Core principle: disk ≠ live Pi
35
47
 
36
48
  Two locations hold pi-crew state:
37
49
 
38
- 1. **Source** (`src/`, `test/`, `package.json`, `workflows/`, `src/runtime/plan-templates.ts`) — git-tracked, `git diff` shows it.
50
+ 1. **Source** (`src/`, `test/`, `package.json`, `workflows/`, `src/runtime/goal-workflow/plan-templates.ts`) — git-tracked, `git diff` shows it.
39
51
  2. **Bundle** (`dist/index.mjs`) — pre-built, loaded by Pi at **extension cold-start only**.
40
52
 
41
- The 3-way resolution order for `dist/index.mjs` (per `index.ts:5-22`):
53
+ The 3-way resolution order for `dist/index.mjs` (per `index.ts:1-25`):
42
54
  ```
43
55
  1. dist/index.mjs (pre-built bundle) if present ← DEFAULT since the v0.9.17 bundle-as-default rollout
44
56
  2. Inline strip-types loading — fallback when bundle missing
45
57
  OR PI_CREW_USE_BUNDLE=0
46
58
  ```
47
59
 
48
- > **Note on version pins**: this skill mentions specific versions (v0.9.17, v0.9.46, v0.9.47) as anchors for *when a behavior was introduced*, not as a constraint on which version the skill applies to. The verification discipline (Tiers 1–8) applies to every pi-crew release. Verify the version pin is still accurate via `git log --oneline -- index.ts` and `git log --oneline -- src/ui/run-dashboard.ts`.
60
+ > **Note on version pins**: this skill mentions specific versions (v0.9.17, v0.9.46, v0.9.47) as anchors for *when a behavior was introduced*, not as a constraint on which version the skill applies to. The verification discipline (Tiers 1–10) applies to every pi-crew release. Verify the version pin is still accurate via `git log --oneline -- index.ts` and `git log --oneline -- src/ui/run-dashboard.ts`.
49
61
 
50
- **Workflow files are runtime data** — `workflows/*.workflow.md` and task prompt strings inside `src/runtime/plan-templates.ts` are loaded per-call, NOT bundled. Edits take effect immediately, no rebuild needed.
62
+ **Workflow files are runtime data** — `workflows/*.workflow.md` and task prompt strings inside `src/runtime/goal-workflow/plan-templates.ts` are loaded per-call, NOT bundled. Edits take effect immediately, no rebuild needed.
51
63
 
52
64
  **The most common silent-failure mode**: edit `src/`, run `npm test` (pass!), rebuild bundle (good md5!), but the session still has the old code because Pi wasn't `/quit`-ed + reopened.
53
65
 
@@ -61,7 +73,8 @@ Before running any tier, verify these are available:
61
73
  | `npm` | All tiers | `npm --version` |
62
74
  | `bash` | All tiers | `echo $BASH_VERSION` |
63
75
  | `md5sum` | Tiers 3, 4, 8 | `which md5sum` (or `md5` on macOS) |
64
- | `tmux` | Tier 5 | `which tmux` (optional — Tier 6 is the fallback) |
76
+ | `tmux` | Tier 5, 10 | `which tmux` (optional — Tier 6 is the fallback) |
77
+ | `herdr` | Tier 10c | `which herdr` (optional — only when pi itself runs inside a herdr pane) |
65
78
  | `python3` | Tier 6 | `python3 --version` (optional — Tier 5 is the fallback) |
66
79
  | `pi` in PATH | Tiers 5, 6 | `which pi` (must be installed via `npx pi install .`) |
67
80
  | `git` | Reference lookups | `git log --oneline -1` should work |
@@ -110,11 +123,11 @@ To add Tier 1 to CI as a fast-feedback gate (under 30s):
110
123
 
111
124
  ---
112
125
 
113
- ## Tier 1 — Critical unit tests (~25s, 101 tests, the only suite you need for broker/UI changes)
126
+ ## Tier 1 — Critical unit tests (~21s, 102 tests, the only suite you need for broker/UI changes)
114
127
 
115
128
  **What**: run the curated 14-file fast subset.
116
129
 
117
- **Why this exists**: full `npm run test:unit` runs 642 files, >4 minutes. Verifier worker timeout is 300s → worker killed mid-run, run = "hang". The fix (introduced in commit `1cb2dca`) splits out a `test:critical` subset covering exactly what changed in the broker/UI work.
130
+ **Why this exists**: full `npm run test:unit` runs 810 files (was 642 at skill-writing time — it keeps growing), several minutes. Verifier worker response timeout would kill the worker mid-run → run = "hang". The fix (introduced in commit `1cb2dca`) splits out a `test:critical` subset covering exactly what changed in the broker/UI work.
118
131
 
119
132
  **How**:
120
133
 
@@ -122,19 +135,19 @@ To add Tier 1 to CI as a fast-feedback gate (under 30s):
122
135
  time npm run test:critical
123
136
  ```
124
137
 
125
- Expected output: `# tests 101 # pass 101 # fail 0 # duration_ms ~26000`. (Count was 97 at v0.9.46; 101 since the model-routing merge — verify with the actual run; the skill's hard-coded numbers drift between releases.)
138
+ Expected output: `# tests 102 # pass 102 # fail 0 # duration_ms ~21000`. (Count was 97 at v0.9.46, 101 at v0.9.66, **102 since the waitMethodsEnabled flip** — verify with the actual run; the skill's hard-coded numbers drift between releases.)
126
139
 
127
140
  **References**:
128
141
 
129
142
  | What | Where |
130
143
  |---|---|
131
- | Script definition | `package.json:67` — list of 14 files passed to `node scripts/test-runner.mjs` |
144
+ | Script definition | `package.json:85` — list of 14 files passed to `node scripts/test-runner.mjs` |
132
145
  | Introduced in commit | `1cb2dca fix(verifier): use test:critical instead of test:unit to avoid worker timeout` |
133
146
  | Runner wrapper | `scripts/test-runner.mjs` — injects `--test-force-exit`, forwards to `tsx --test` |
134
- | The 14 files | broker: `crew-broker-{handshake,stale-socket,feature-flag,server-gate,client-fallback,mailbox-observer,close-during-reconnect,steer-dedup,symlink-steering}.test.ts`; UI: `keybinding-map.parity.test.ts`, `pi-tui-dispatch-probe.test.ts`, `session-utils-extract.test.ts`; config: `config-schema-sync.test.ts`, `child-pi-env-spread.test.ts` |
135
- | Failure mode that motivates it | Worker timeout in `src/runtime/child-pi-constants.ts:23` (`RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs` = 300000); verifier LLM ran `npm test` and got killed at 300s with exit 143 (SIGTERM) |
147
+ | The 14 files | broker: `test/unit/runtime/broker/crew-broker-{handshake,stale-socket,feature-flag,server-gate,client-fallback,mailbox-observer,close-during-reconnect,steer-dedup,symlink-steering}.test.ts`; UI: `test/unit/ui/keybinding-map.parity.test.ts`, `test/unit/ui/pi-tui-dispatch-probe.test.ts`; utils: `test/unit/utils/session-utils-extract.test.ts`; config: `test/unit/config/config-schema-sync.test.ts`; spawn env: `test/unit/runtime/child-pi/child-pi-env-spread.test.ts` |
148
+ | Failure mode that motivates it | Worker timeout in `src/runtime/child-pi/child-pi-constants.ts:23` (`RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs` — 300000 at the time, now 600000); verifier LLM ran `npm test` and got killed with exit 143 (SIGTERM) |
136
149
 
137
- **Run after**: any edit to `src/runtime/crew-broker*.ts`, `src/ui/`, `src/config/`, `src/extension/registration/lifecycle-handlers.ts`, or `src/runtime/child-pi-spawn.ts`.
150
+ **Run after**: any edit to `src/runtime/broker/*.ts`, `src/ui/`, `src/config/`, `src/extension/registration/lifecycle-handlers.ts`, or `src/runtime/child-pi/*.ts`.
138
151
 
139
152
  ---
140
153
 
@@ -142,7 +155,7 @@ Expected output: `# tests 101 # pass 101 # fail 0 # duration_ms ~26000`. (Count
142
155
 
143
156
  **What**: prove all three precedence paths in `effectiveEnabled()` still resolve correctly.
144
157
 
145
- **Why**: any change to `DEFAULT_BROKER` (in `src/config/defaults.ts:169`) or `effectiveEnabled()` (in `src/extension/registration/lifecycle-handlers.ts:819-833`) can silently break the precedence chain. The chain:
158
+ **Why**: any change to `DEFAULT_BROKER` (in `src/config/defaults.ts:191`) or `effectiveEnabled()` (in `src/extension/registration/lifecycle-handlers.ts:1026-1039`) can silently break the precedence chain. The chain:
146
159
 
147
160
  ```
148
161
  PI_CREW_BROKER=0 → disabled (env always wins)
@@ -168,11 +181,11 @@ All three must show `# pass 101 # fail 0`. Measured times in this session (2026-
168
181
 
169
182
  | What | Where |
170
183
  |---|---|
171
- | `DEFAULT_BROKER` constant | `src/config/defaults.ts:169-173` (Phase 4: `enabled: true`) |
172
- | Precedence function | `src/extension/registration/lifecycle-handlers.ts:819-833` (`return cfg?.enabled !== false;` at line 828) |
173
- | `resolveBrokerEnvOverride` | `src/config/defaults.ts:186-193` |
174
- | Env-precedence unit tests | `test/unit/crew-broker-feature-flag.test.ts:31` (default-on assertion), `:54-110` (env=1/env=0/unset/arbitrary cases at lines 54, 66, 78, 90, 103) |
175
- | Controller-gate tests | `test/unit/crew-broker-server-gate.test.ts:78` (env kill switch under default-on), `:143` (env=1 with no config) |
184
+ | `DEFAULT_BROKER` constant | `src/config/defaults.ts:191` (Phase 4: `enabled: true`; `waitMethodsEnabled: true` at `:205` since the 2026-08-26 ask flip) |
185
+ | Precedence function | `src/extension/registration/lifecycle-handlers.ts:1026-1039` (`return cfg?.enabled !== false;`) |
186
+ | `resolveBrokerEnvOverride` | `src/config/defaults.ts:252` |
187
+ | Env-precedence unit tests | `test/unit/runtime/broker/crew-broker-feature-flag.test.ts:31` (default-on assertion), `:54-110` (env=1/env=0/unset/arbitrary cases at lines 54, 66, 78, 90, 103) |
188
+ | Controller-gate tests | `test/unit/runtime/broker/crew-broker-server-gate.test.ts:78` (env kill switch under default-on), `:143` (env=1 with no config) |
176
189
  | Decision doc | `docs/decisions/2026-07-22-broker-phase4-gated-on.md` |
177
190
  | Superseded doc | `docs/decisions/2026-07-21-broker-phase4-default-on.md` (marked SUPERSEDED in commit `4186284`) |
178
191
  | Default flip commit | `612e18b feat(broker): Phase 4 gated ON — flip broker.enabled default to true` |
@@ -200,9 +213,9 @@ Compare the printed md5 against what the user's Pi session loaded. If they diffe
200
213
  | `typecheck` script | `package.json` `"typecheck"` — runs `tsc --noEmit && node --experimental-strip-types -e "await import('./index.ts'); ..."` |
201
214
  | `build:bundle` script | `package.json` `"build:bundle"` — runs `node scripts/build-bundle.mjs` |
202
215
  | Bundle builder | `scripts/build-bundle.mjs` (esbuild-based, bundles `index.bundle.ts` → `dist/index.mjs`) |
203
- | Bundle resolution rule | `index.ts:5-22` (entrypoint docstring); also `scripts/build-bundle.mjs:14-20` (entrypoint preference); **symlink is live for source files but the bundled `dist/index.mjs` is loaded** |
216
+ | Bundle resolution rule | `index.ts:1-25` (entrypoint docstring); also `scripts/build-bundle.mjs:14-20` (entrypoint preference); **symlink is live for source files but the bundled `dist/index.mjs` is loaded** |
204
217
  | Postinstall hook | `scripts/postinstall.mjs:43` — best-effort bundle rebuild; falls back to strip-types if esbuild missing |
205
- | Bundle md5 after Phase-4 commit | `1cc4d55e18add7b9a036c569143320b6` (~2.78 MB at the time; **check current**: `md5sum dist/index.mjs`. As of v0.9.66 I-batch 2026-08-11: `16e29d053bd370e24f40df147dadcb79` ~2.81 MB) |
218
+ | Bundle md5 anchors | `1cc4d55e18add7b9a036c569143320b6` (Phase-4 flip, ~2.78 MB) → `16e29d053bd370e24f40df147dadcb79` (v0.9.66, 2026-08-11) → `9b557ac106b82e1ee33d39dd0d6c7dd7` (post-MuxSurface-A1 main, 2026-08-27). **Always check current**: `md5sum dist/index.mjs` |
206
219
 
207
220
  ---
208
221
 
@@ -212,7 +225,7 @@ Compare the printed md5 against what the user's Pi session loaded. If they diffe
212
225
 
213
226
  **The immediate-vs-rebuild rule** (which edits take effect without a rebuild):
214
227
  - `workflows/*.workflow.md` edits → **immediate**, no rebuild, no restart
215
- - `src/runtime/plan-templates.ts` `taskTemplate` strings → **immediate**, runtime data
228
+ - `src/runtime/goal-workflow/plan-templates.ts` `taskTemplate` strings → **immediate**, runtime data
216
229
  - Everything else (`src/` edits, `package.json`) → must `npm run build:bundle` THEN user `/quit` + reopen Pi
217
230
 
218
231
  **How to verify in this session**:
@@ -234,7 +247,7 @@ tmux -S /tmp/sock new-session -d -x 160 -y 50 -s pi \
234
247
 
235
248
  | What | Where |
236
249
  |---|---|
237
- | Bundle resolution | `index.ts:5-22` — "dist/index.mjs (pre-built bundle) if present AND not explicitly disabled — DEFAULT since v0.9.17" |
250
+ | Bundle resolution | `index.ts:1-25` — "dist/index.mjs (pre-built bundle) if present AND not explicitly disabled — DEFAULT since v0.9.17" |
238
251
  | Bundle size impact after Phase-4 flip | `docs/decisions/2026-07-22-broker-phase4-gated-on.md` §Verification: "2.78 MB before and after the flip; the broker code was already in the bundle; only the default boolean changed" |
239
252
  | Symlink confirmation | **The symlink lives in the CONSUMING project, not inside pi-crew itself.** From the pi-crew repo, check the parent: `readlink ../node_modules/pi-crew` (returns `../pi-crew` for dev clones). For global installs: `readlink "$(npm root -g)"/pi-crew`. Pattern is always `<consumer>/node_modules/pi-crew → <pi-crew-repo>`. |
240
253
 
@@ -281,7 +294,7 @@ tmux capture-pane -t pi -p > /tmp/screen-after-up.txt
281
294
  |---|---|
282
295
  | `keyOf()` helper | `src/ui/key-utils.ts:37-42` (import + type alias at lines 16-18) |
283
296
  | Dispatch path | `src/ui/keybinding-map.ts` (migrated to `matchesKey()` in commit `f05a10d`) |
284
- | Golden snapshot test | `test/unit/keybinding-map.parity.test.ts` — 7 `it()` blocks asserting parity against a generated golden snapshot; BINDINGS table has 27 entries (`src/ui/keybinding-map.ts:132-180`) |
297
+ | Golden snapshot test | `test/unit/ui/keybinding-map.parity.test.ts` — 8 `it()` blocks asserting parity against a generated golden snapshot; `DEFAULT_BINDINGS` table has 31 action entries (`src/ui/keybinding-map.ts:147-211`; user-overridable via the `keybindings` config section / `PI_CREW_KEYBINDINGS` env) |
285
298
  | Live probe test | `test/unit/pi-tui-dispatch-probe.test.ts` — direct probe of dispatch (3 tests) |
286
299
  | Probe commit | `84944f7 test(probe): add invalidate() to control object so typecheck passes` |
287
300
  | Tab/Space bind | `src/ui/run-dashboard.ts` + commit `15a0ffe fix(ui): also bind Tab/Space/Enter/S to select in dashboard dispatch` |
@@ -345,9 +358,9 @@ else:
345
358
 
346
359
  ## Tier 7 — Smoke team run (verifier prompt doesn't hang)
347
360
 
348
- **What**: prove the verifier worker completes within `RESPONSE_TIMEOUT_MS` (300s).
361
+ **What**: prove the verifier worker completes within `RESPONSE_TIMEOUT_MS` (**600s since the stuck-worker hardening — was 300s when this skill was distilled; `DEFAULT_CHILD_PI.responseTimeoutMs = 10 * 60_000`**).
349
362
 
350
- **Why this is its own tier**: `test:critical` covers unit-level invariants, but the verifier LLM is a separate failure mode — it reads the verifier prompt from `src/runtime/plan-templates.ts:143, 190` (taskTemplate strings) or from `workflows/*.workflow.md:24, 30, 31` (workflow verifier sections), then decides which bash command to run. If the prompt says "Run tests" without specifying which, the LLM runs `npm test` and the worker hangs at 300s with exit 143.
363
+ **Why this is its own tier**: `test:critical` covers unit-level invariants, but the verifier LLM is a separate failure mode — it reads the verifier prompt from `src/runtime/goal-workflow/plan-templates.ts:144, 147` (taskTemplate strings) or from `workflows/*.workflow.md` (workflow verifier sections), then decides which bash command to run. If the prompt says "Run tests" without specifying which, the LLM runs `npm test` (810+ files) and the worker gets killed by the response timeout with exit 143.
351
364
 
352
365
  **How** (from parent Pi session — `team` is a tool, not a shell command):
353
366
 
@@ -358,13 +371,13 @@ team:
358
371
  action: run # run | status | events | cancel | retry | ...
359
372
  team: fast-fix # team (a role-set): default / fast-fix / implementation / parallel-research / research / review
360
373
  workflow: fast-fix # workflow (a phase DAG): default / fast-fix / plan-execute / implementation / review / research / parallel-research / pipeline / chain
361
- goal: "Smoke-verify <X>. Run `npm run test:critical && npx tsc --noEmit` once, cache output, report exact pass/fail counts + total time. Confirm verifier completes without hang (must be <300s)."
374
+ goal: "Smoke-verify <X>. Run `npm run test:critical && npx tsc --noEmit` once, cache output, report exact pass/fail counts + total time. Confirm verifier completes without hang (must be <600s)."
362
375
  async: false # synchronous: wait for completion before returning
363
376
  ```
364
377
 
365
378
  The `team` tool is described in the agent's system prompt. Use `team action='status' <runId>` to inspect mid-run, `team action='events' <runId> <limit>` for the event log, `team action='cancel' <runId>` to abort.
366
379
 
367
- **Real measured outcomes from this session**:
380
+ **Real measured outcomes from this session** (July 2026, under the old 300s timeout — wall-clock shape still representative):
368
381
 
369
382
  | Run ID | Goal | Result | Wall-clock |
370
383
  |---|---|---|---|
@@ -376,18 +389,18 @@ The `team` tool is described in the agent's system prompt. Use `team action='sta
376
389
 
377
390
  | What | Where |
378
391
  |---|---|
379
- | `verificationCommand` for plan-templates | `src/runtime/plan-templates.ts:146, 193` — both templates now `npm run test:critical && npx tsc --noEmit` |
380
- | `taskTemplate` for verifier | `src/runtime/plan-templates.ts:143, 190` — explicit "Do NOT run `npm test`" + "<2 min" budget |
381
- | Workflow verifier prompts | `workflows/fast-fix.workflow.md:24`, `workflows/default.workflow.md:31`, `workflows/plan-execute.workflow.md:30`, `workflows/review.workflow.md:31` |
392
+ | `verificationCommand` for plan-templates | `src/runtime/goal-workflow/plan-templates.ts:147, 151` — both templates now `npm run test:critical && npx tsc --noEmit` |
393
+ | `taskTemplate` for verifier | `src/runtime/goal-workflow/plan-templates.ts:144` — explicit "Do NOT run `npm test`" + "<2 min" budget |
394
+ | Workflow verifier prompts | `workflows/fast-fix.workflow.md:24`, `workflows/plan-execute.workflow.md:30`, `workflows/review.workflow.md:31` — all three pin `test:critical`; `workflows/default.workflow.md:39` uses generic wording ("FAST targeted checks only, never the full suite") |
382
395
  | Verifier fix commit (plan-templates) | `1cb2dca fix(verifier): use test:critical instead of test:unit to avoid worker timeout` |
383
396
  | Verifier fix commit (workflows) | `d599578 fix(workflows): specify fast test:critical command in verifier prompts` |
384
- | Watchdog constant | `src/runtime/child-pi-constants.ts:23` — `RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs` |
385
- | Cache directive | `Run FAST checks ONCE (cache output to .crew/cache/)` — anti-re-run safeguard baked into all 4 workflow verifier prompts |
397
+ | Watchdog constant | `src/runtime/child-pi/child-pi-constants.ts:23` — `RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs` = **600_000** (`src/config/defaults.ts:26`; env override `PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS`, see `child-pi.ts:692-697`) |
398
+ | Cache directive | `Run FAST checks ONCE (cache output to .crew/cache/)` — anti-re-run safeguard baked into the verifier prompts |
386
399
  | Decision doc | `docs/decisions/2026-07-22-broker-phase4-gated-on.md` §Verification (mentions the smoke run `team_20260722100811_9bf95bebff2b052a`) |
387
400
 
388
401
  **Two known failure modes for verifier**:
389
402
 
390
- 1. **Verifier LLM runs `npm test`** (full unit + integration suite, >4 min) instead of `npm run test:critical`. Symptom: worker killed with exit 143 after exactly 300s. Fix: rewrite the verifier prompt to specify the exact fast command AND include "Do NOT run `npm test` or `npm run test:unit`".
403
+ 1. **Verifier LLM runs `npm test`** (full unit + integration suite, >4 min) instead of `npm run test:critical`. Symptom: worker killed with exit 143 at the response timeout (300s historically — the measured runs below predate the bump to 600s). Fix: rewrite the verifier prompt to specify the exact fast command AND include "Do NOT run `npm test` or `npm run test:unit`".
391
404
  2. **Verifier LLM improvises** with a clean-cache `npm test` run anyway. The cache directive ("cache to `.crew/cache/`", "do NOT re-run") catches this — the second worker that observes a cached log should not re-run.
392
405
 
393
406
  ---
@@ -407,7 +420,7 @@ md5sum dist/index.mjs
407
420
  readlink ../node_modules/pi-crew/dist/index.mjs 2>/dev/null \
408
421
  || readlink "$(npm root -g)"/pi-crew/dist/index.mjs \
409
422
  || md5sum "$(npm root -g)"/pi-crew/dist/index.mjs
410
- # (the consuming project loads pi-crew via this symlink — see index.ts:5-22)
423
+ # (the consuming project loads pi-crew via this symlink — see index.ts:1-25)
411
424
  ```
412
425
 
413
426
  If the two md5s match → session is on the latest code. If not → user must `/quit` + reopen Pi.
@@ -418,7 +431,7 @@ If the two md5s match → session is on the latest code. If not → user must `/
418
431
 
419
432
  | What | Where |
420
433
  |---|---|
421
- | Symlink path | `index.ts:5-22` — **the symlink lives in the CONSUMING project** (parent dir or global prefix), not inside pi-crew itself. From the repo: `readlink ../node_modules/pi-crew` (dev) or `readlink "$(npm root -g)"/pi-crew` (global). Verify with `readlink` + `npm root -g`. |
434
+ | Symlink path | `index.ts:1-25` — **the symlink lives in the CONSUMING project** (parent dir or global prefix), not inside pi-crew itself. From the repo: `readlink ../node_modules/pi-crew` (dev) or `readlink "$(npm root -g)"/pi-crew` (global). Verify with `readlink` + `npm root -g`. |
422
435
  | Session load model | Same file: "dist/index.mjs (pre-built bundle) if present — DEFAULT since v0.9.17" |
423
436
 
424
437
  ---
@@ -427,7 +440,7 @@ If the two md5s match → session is on the latest code. If not → user must `/
427
440
 
428
441
  **What**: drive the team tool + subagent tools through a spread of actions from the parent Pi session to prove the full surface works end-to-end, not just one smoke run.
429
442
 
430
- **Why this exists**: Tier 7 proves one team run completes. But pi-crew has ~50 `team` actions plus 4 subagent tools (`Agent`, `crew_agent`, `get_subagent_result`, `crew_agent_steer`), dispatched through several code paths (sync run, async run, chain, parallel, direct subagent). A schema or registration regression can break *some* paths while others still pass. The battery catches path-specific breakage.
443
+ **Why this exists**: Tier 7 proves one team run completes. But pi-crew has **55 `team` actions across 5 domain dispatchers** (`RUN` 10: run/parallel/plan/plans/orchestrate/resume/retry/wait/steer/goal · `STATUS` 16: status/list/get/events/artifacts/summary/graph/search/health/worktrees/checkpoint/cache/explain/onboard/recommend/help · `CONTROL` 7 · `MANAGE` 16 · `AUTOMATE` 6 — `src/schema/team-tool-schema.ts:391-437`) plus the subagent tools (`Agent`, `crew_agent`, `get_subagent_result`, `steer_subagent` — with `crew_agent_result`/`crew_agent_steer` aliases), the worker-side tools (`ask`, `delegate`, `message`), and the `team-settings` config surface, dispatched through several code paths (sync run, async run, chain, parallel, direct subagent). A schema or registration regression can break *some* paths while others still pass. The battery catches path-specific breakage.
431
444
 
432
445
  **When required**: any change to `src/schema/team-tool-schema.ts`, `src/extension/registration/team-tool.ts`, `src/extension/team-tool/*.ts` (handler dispatch), or the subagent-tool registration. Optional but cheap for any change — the read-only actions are free.
433
446
 
@@ -437,20 +450,31 @@ If the two md5s match → session is on the latest code. If not → user must `/
437
450
  - `team action='list'` — teams/workflows/agents
438
451
  - `team action='recommend' goal='...'` — planner routing
439
452
  - `team action='health'` — run-state scan
440
- - `team action='doctor' focus='zombies'` — orphan subagent scan (read-only)
453
+ - `team action='doctor' focus='zombies'` — orphan subagent + orphan surface-pane scan (read-only)
441
454
  - `team action='status' runId='<recent>' details=false` — compact
442
455
  - `team action='events' runId='<recent>'` — full event lifecycle
443
456
  - `team action='summary' runId='<recent>'` — cost/by-role report
444
457
  - `team action='get' resource='workflow' team='implementation'` — resource inspect
445
458
  - `team action='explain' runId='<recent>'` — markdown render
446
459
  - `team action='worktrees' runId='<recent>'` — workspace listing
460
+ - `team action='graph' runId='<recent>'` — task-graph render (newer action)
461
+ - `team action='search' query='...'` — event/artifact search (newer action)
462
+ - `team-settings` (slash) or `team action='settings' config={args:'get runtime.surface.mode'}` — config surface incl. the surface/nesting keys
447
463
  2. **9b. Spawn paths** (cost tokens — one probe each is enough):
448
464
  - `team action='run'` sync (fast-fix, trivial goal) — proves sync run + child-pi spawn + provider-extension loading
449
465
  - `team action='run' async=true` — proves background dispatch
450
466
  - `team action='run' chain='"A" -> "B"'` — proves sequential handoff (chain runner). **Omit `workflow`** — passing `workflow:'chain'` forwards it to each step and fails fast (~58ms silent; issue #44).
467
+ - `team action='orchestrate'` / `action='plan'` / `action='plans'` — planning surface without execution (cheap middle ground between 9a read-only and full spawn)
451
468
  - `Agent` direct subagent — proves the direct-subagent tool
452
469
  - `crew_agent` `run_in_background=true` then `get_subagent_result` — proves background subagent lifecycle
453
- 3. **Acceptance**: every action returns without `Unknown type` / `Validation failed for tool team` / empty error text; every spawn path completes with `consistency=1` and the expected probe token in the agent output.
470
+ - `steer_subagent` while a background subagent runs — proves live steering (timing-sensitive; was listed under 9c, but it is the canonical name now — `crew_agent_steer` is the alias)
471
+ 3. **9b-W. Worker-tool paths** (cost tokens — proven via goal text that instructs the worker to call the tool; one probe each):
472
+ - **ask round-trip**: goal says "use the `ask` tool to ask the parent <question>, wait for the reply". Proves `wait.request` → park → `team action='respond'` → pickup. **The gate `broker.waitMethodsEnabled` defaulted to `false` until 2026-08-26 (`ceb9a68d` flipped it) — ask slept silently for weeks while every wait.request was rejected `policy-disabled`.** If a worker "answers its own question" instead of asking, the gate or the prompt guidance regressed. Rejections are never silent: a `policy.action` event lands in `events.jsonl`.
473
+ - **message notify**: goal says "use the `message` tool to notify the parent when done". Proves `msg.send` (non-blocking) + the broker `from`-override (anti-spoof) + the wake pattern on the orchestrator session. Rate-limit 10 msg/60s per worker — a burst probe should hit the limit, not hang.
474
+ - **message DM/group**: goal says "DM task `<sibling taskId>` / send to group `x`" — proves `to:` routing + inbox pickup (delivered as fenced `<inbox-message>` DATA, not instructions).
475
+ - **delegate nesting**: goal says "use the `delegate` tool to spawn a child agent". Proves the role gate is open for every role (D8, default-on), the depth cap (`nesting.maxDepth: 4` — a depth-5 attempt must reject with the structured policy message + `delegate.rejected` event, never silently), and the nested-slot budget. Kill switch: `nesting.enabled: false` in **user** config only (sensitive — project config cannot flip it).
476
+ - **full loadout sanity** (D5): in any 9b run, have the worker report its loaded extensions/skills/tools. Workers are FULL pi sessions by default — no `--no-extensions`, no `--tools` allowlist, `--no-skills` only when the agent frontmatter says `inheritSkills: false`. A worker missing MCP tools/skills means the loadout policy regressed (see Anti-patterns, armed-role row).
477
+ 4. **Acceptance**: every action returns without `Unknown type` / `Validation failed for tool team` / empty error text; every spawn path completes with `consistency=1` and the expected probe token in the agent output.
454
478
 
455
479
  **Real measured outcome** (this session, after the v0.9.57 schema fix): 9a (15 team actions) + 9b (4 subagent tools / 3 run paths) exercised; all green; the two silent-failure modes that motivated this tier (`Unknown type` from `Type.Unsafe` without Kind, and `Validation failed for tool team` from empty-string-strict schema) were caught ONLY by this battery — Tier 1-8 all passed while the team tool was broken live. The session also surfaced the unauthorized-agent-edit anti-pattern (a chain-run agent edited `chain-runner.ts` mid-smoke) — see Anti-patterns.
456
480
 
@@ -466,7 +490,7 @@ If the two md5s match → session is on the latest code. If not → user must `/
466
490
  - `team action='invalidate' runId='...'` — cache invalidation
467
491
  - `team action='resume' runId='...'` / `retry` — resume a completed/failed run
468
492
  - `team action='respond' taskId='...' message='...'` — mailbox reply (needs a waiting task)
469
- - subagent steering: `crew_agent run_in_background=true` a long task (e.g. `sleep 60`), then `crew_agent_steer` while it runs, then `get_subagent_result` — proves the steer arrived (timing-sensitive; assert the agent's output reflects the steer)
493
+ - subagent steering (full procedure): `crew_agent run_in_background=true` a long task (e.g. `sleep 60`), then `steer_subagent` (alias `crew_agent_steer`) while it runs, then `get_subagent_result` — proves the steer arrived (timing-sensitive; assert the agent's output reflects the steer; the quick one-shot version lives in 9b)
470
494
 
471
495
  **9d. Destructive** (⚠️ **requires explicit user confirmation** per the delegation policy — never run unprompted):
472
496
  - `team action='prune' keep=<N>` — delete old finished runs
@@ -491,28 +515,110 @@ If the two md5s match → session is on the latest code. If not → user must `/
491
515
 
492
516
  ---
493
517
 
518
+ ## Tier 10 — Surface-mode battery (MuxSurface A1, workers in real panes)
519
+
520
+ **What**: prove workers can live in REAL multiplexer panes (tmux/herdr) — pane spawn, in-pane boot via launch script, auto-exit, degrade-to-headless on failure, doctor orphan cleanup — without breaking the headless default.
521
+
522
+ **Why this is its own tier**: surface is **fail-closed by design**. Every failure (no mux binary, forced-mode detect fail, depth > `maxDepth`, pane cap reached, `visibleAgents` empty, `mode: off`) degrades to headless and the run **still goes green**. A green run therefore proves NOTHING about panes — only pane-level evidence does (manifest `surface.panes`, `tmux list-panes`, the E2E sentinel). This is the exact inverse of Tier 9's silent schema failures: there the tool errors, here everything looks healthy. **NOTE (2026-08-27): async runs are NO LONGER hard-gated headless** — surface now follows env + `runtime.surface.*` config, not run-mode; so an async run with a live mux still engages panes.
523
+
524
+ **The #1 silent no-op**: `runtime.surface.visibleAgents` defaults to `[]` — surface is visible to NOBODY until opted in (spec §8.1, A1 default). A test that sets `mode: "auto"` (already the default) and expects panes will pass green with zero panes created. **Always set `visibleAgents` (exact agent/role names, or `["*"]`) when testing surface.** Configure via `team-settings set runtime.surface.visibleAgents '["*"]'` (slash) or `team action='settings' config={args:"set runtime.surface.visibleAgents [\"*\"]"}`.
525
+
526
+ **Config surface** (`src/config/types.ts:94`, manageable via team-settings — `src/extension/team-tool/handle-settings.ts:23-24`):
527
+ - `runtime.surface.mode`: `"auto"` (default — detect tmux/herdr, use panes when present) | `"tmux"` / `"herdr"` (force; detect fail → headless + warning event, **never a throw**) | `"off"`
528
+ - `runtime.surface.visibleAgents`: exact-match agent/role names, `["*"]` = all. Default `[]` = nobody.
529
+
530
+ **When required**: any change to `src/runtime/surface/**` (providers, resolve, spawn, degrade, launch script), `src/prompt/surface-worker.ts` (recorder + auto-exit + parent-guard), the surface branch of `src/runtime/child-pi/child-pi.ts`, surface fields in doctor, or the surface config keys.
531
+
532
+ ### 10a. E2E suites (real tmux + real herdr, no mocks)
533
+
534
+ Hai suite sinh đôi, mỗi backend một file — tmux tự skip khi `CI=1` hoặc `$TMUX` unset (chạy từ TRONG tmux); herdr tự skip khi CI, đang trong tmux, hoặc socket herdr không tồn tại:
535
+
536
+ ```bash
537
+ # tmux — from a shell inside tmux (or spawn a dedicated session):
538
+ tmux new-session -d -s crew-e2e "cd ${PWD} && \
539
+ node --experimental-strip-types --test --test-concurrency=1 --test-timeout=120000 \
540
+ test/system/surface-tmux.e2e.test.ts 2>&1 | tee /tmp/surface-e2e.log"
541
+
542
+ # herdr — chạy khi herdr server sống và KHÔNG trong tmux (test tạo pane thật
543
+ # trong herdr của user ~4s rồi tự dọn — pane sẽ hiện lên màn hình):
544
+ node --experimental-strip-types --test --test-concurrency=1 --test-timeout=120000 \
545
+ test/system/surface-herdr.e2e.test.ts
546
+ ```
547
+
548
+ Mỗi suite 3 test (cùng kịch bản, provider khác nhau):
549
+ 1. **spawn + self-close**: pane thật được tạo, launch script boot worker trong pane (sentinel mang pane id + PID của worker), pane tự đóng khi task xong (auto-exit qua `ctx.shutdown()`), run hoàn thành.
550
+ 2. **kill-pane giữa chừng → degrade**: pane bị giết → `classifyOnExit` (2s) → cause-group lockout → re-dispatch headless → run vẫn `done`. Đây là proof "không chết khi multiplexer chết" — điều kiện nền của toàn bộ thiết kế.
551
+ 3. **doctor orphan cleanup**: liệt kê + đóng pane mồ côi thật (từ terminal-run manifests), report chứa pane id.
552
+
553
+ Acceptance: 3/3 cho mỗi suite khi điều kiện backend thỏa; skip vì thiếu mux là **correct-by-design**, không phải fail — nhưng cũng không tính là "Tier 10 pass" cho backend đó (xem Done-criteria).
554
+
555
+ **Bài học wire herdr (3 bug thật chỉ E2E mới bắt được, fix `01af9a78` 2026-08-27)**: herdr 0.8.2 không push `pane.closed` cho process exit tự nhiên (chỉ `pane.exited`) — provider phải subscribe cả hai; frame `\n\n` khiến server đóng subscription; `attach` null khiến doctor không bao giờ đóng orphan herdr. Unit test fake socket KHÔNG bao giờ bắt được loại này — luôn chạy E2E thật khi đụng wire provider.
556
+
557
+ ### 10b. Live surface run (từ parent Pi session)
558
+
559
+ ```text
560
+ 1. team-settings set runtime.surface.visibleAgents '["*"]' # hoặc agent cụ thể, vd '["executor"]'
561
+ 2. team action='run' team='fast-fix' goal='<trivial>' async=false
562
+ 3. DẠNG KIỂM TRA (shell):
563
+ tmux list-panes -a -F '#{pane_id} #{pane_title} #{pane_pid}' | grep <taskId>
564
+ # pane title mang taskId; pane_pid là shell chạy launch script
565
+ 4. Sau khi run xong: pane đã tự đóng (auto-exit); không còn pane mang taskId
566
+ 5. team action='status' / manifest: surface.panes ghi nhận provider + pane ids
567
+ 6. Dọn dẹp: team-settings set runtime.surface.visibleAgents '[]'
568
+ ```
569
+
570
+ Evidence cần thu: pane id + title từ `list-panes` TRONG lúc run, và pane biến mất sau run. **Đừng lấy `manifest.surface.panes` làm evidence engage** — map này được `releaseSurfacePane` xóa ngay khi pane đóng, nên một run engage THÀNH CÔNG cũng kết thúc với `panes: {}`. Evidence đúng sau run: `events.jsonl` có `worker.surface_spawned` + `worker.surface_closed` (kèm paneId) và KHÔNG có `surface.degraded`; `manifest.surface.provider` + `workerPids` non-empty (chỉ nhánh surface mới ghi `workerPids` qua `notifyWorkerStarted`). Nếu đã set `visibleAgents` mà không thấy surface_spawned: đọc `worker.surface_gate_blocked` (từ `d668e166`) — nó cho biết gate nào chặn và vì sao (`{gate, reason, env}`). Không có các event đó → surface không engage được dù run xanh.
571
+
572
+ ### 10c. herdr path (chỉ khi pi chạy trong herdr pane)
573
+
574
+ herdr chỉ được detect khi **chính pi session đang chạy trong một herdr pane** (design decision — không đoán mò qua socket nếu pi không thuộc herd). Socket API newline-JSON qua `~/.config/herdr/herdr.sock` (herdr 0.8.2+): 1 request = 1 connection, envelope `{"event":...}` (underscore), `pane.read` cần source `"visible"`. Nếu không có herdr: 10c skip với lý do "not in herdr pane" — chấp nhận được, miễn ghi rõ trong report.
575
+
576
+ ### Surface failure modes → symptom map
577
+
578
+ | Symptom | Likely cause | Recovery |
579
+ |---|---|---|
580
+ | Run xanh nhưng không pane nào xuất hiện | `visibleAgents` còn `[]` (default) — silent no-op | Set `visibleAgents`; re-run. Từ `d668e166`: nếu đã opt-in mà vẫn headless, `events.jsonl` có `worker.surface_gate_blocked` mang `{gate, reason, env}` (chỉ phát khi visibleAgents non-empty — default runs im lặng) |
581
+ | `mode: "tmux"` nhưng vẫn headless | tmux binary/socket detect fail → degrade có chủ đích | `tmux info`; kiểm tra `$TMUX`; đọc warning event trong `events.jsonl` (không bao giờ im lặng) |
582
+ | Worker thứ 7 trở đi headless | Pane cap `MAX_SURFACE_WORKERS = 6` (`src/runtime/surface/resolve-surface.ts`) — hardcoded A1 | By design; config cap là A2 defer |
583
+ | Surface worker chết liên tục → quay lại headless | Degrade lockout: cause-group lockout + spawn-fail streak 3 | Đọc `events.jsonl` (degrade.classify events); fix gốc nhân (thường là launch script env) |
584
+ | Pane ở lại sau crash host | Orphan pane — doctor chưa quét | `team action='doctor' focus='zombies'` liệt kê + đóng; sweepLaunchScripts dọn script TTL |
585
+ | herdr không được detect | pi không chạy trong herdr pane | By design; chạy pi trong herdr pane rồi thử lại |
586
+ | herdr worker xong việc nhưng host treo tới deadline 600s | Provider thiếu subscribe `pane.exited` (herdr 0.8.2 không push `pane.closed` cho exit tự nhiên) — đã fix `01af9a78` | Chạy 10a herdr suite; đọc subscription wiring trong `herdr-provider.ts` |
587
+ | herdr subscription im lặng / mux-dead ngay lập tức | Frame `\n\n` (tự nối newline trên wrapper đã nối sẵn) — server coi empty line là malformed | Xem unit "wire framing" trong `herdr-provider.test.ts`; đừng thêm `\n` ở tầng provider |
588
+ | Async run tưởng "luôn headless" | **KHÔNG còn** — nhưng 2 lớp phải cùng mở: (1) hard-gate async bỏ 2026-08-27; (2) `BACKGROUND_RUNNER_ENV_ALLOWLIST` từng strip `TMUX`/`HERDR_*` khỏi detached runner → async vẫn gate `no-mux` dù host trong mux (battery 2026-08-30 Finding 2, fix `f0a41a16` thêm đủ mux env vào allow-list) | Test với live mux + `visibleAgents` set: async run PHẢI có `worker.surface_spawned` (verified live `team_20260830144901`: 3/3 panes, tab riêng, tab đóng khi run end); nếu chỉ thấy `no-mux` → kiểm allow-list trước khi nghi gate |
589
+
590
+ **Cảnh báo an toàn**: KHÔNG dùng `tmux kill-server` để "test degrade" trên máy user — nó giết toàn bộ session của user. Dùng `kill-pane` trên pane của run thử nghiệm (như E2E test #2 làm), hoặc chạy trong tmux server riêng (`tmux -S /tmp/crew-sock`).
591
+
592
+ ---
593
+
494
594
  ## Anti-patterns (the cost is real, observed in this session)
495
595
 
496
596
  | Anti-pattern | Cost | Where fixed | Reference |
497
597
  |---|---|---|---|
498
- | `npm test` in verifier prompt | 300s worker timeout, run = "hang" | `1cb2dca` | `src/runtime/plan-templates.ts:143, 190` + 4 workflow files |
499
- | `npm run test:unit` for in-loop verify | >4 min, same hang | `1cb2dca` | `package.json:67` (`test:critical` script) |
500
- | Default-off assumption in tests | Break when default flips | `612e18b` | `test/unit/crew-broker-feature-flag.test.ts:31` (`DEFAULT_BROKER.enabled === true`) |
501
- | Test using real `loadConfig()` to mock config | Flaky when env / disk config changes | `612e18b` | `test/unit/crew-broker-server-gate.test.ts:78` (use `brokerEnv: "0"` instead of `flagOn: false`) |
502
- | Source edit seen immediately | No, requires bundle rebuild + reload | n/a (permanent) | `index.ts:5-22` — bundle resolution rules |
598
+ | `npm test` in verifier prompt | worker killed at the response timeout (300s then, 600s now), run = "hang" | `1cb2dca` | verifier `taskTemplate`/`verificationCommand` in `src/runtime/goal-workflow/plan-templates.ts` (now `:144, 147, 151`) + workflow files |
599
+ | `npm run test:unit` for in-loop verify | >4 min, same hang | `1cb2dca` | `package.json:85` (`test:critical` script) |
600
+ | Default-off assumption in tests | Break when default flips | `612e18b` | `test/unit/runtime/broker/crew-broker-feature-flag.test.ts:31` (`DEFAULT_BROKER.enabled === true`) |
601
+ | Test using real `loadConfig()` to mock config | Flaky when env / disk config changes | `612e18b` | `test/unit/runtime/broker/crew-broker-server-gate.test.ts:78` (use `brokerEnv: "0"` instead of `flagOn: false`) |
602
+ | Source edit seen immediately | No, requires bundle rebuild + reload | n/a (permanent) | `index.ts:1-25` — bundle resolution rules |
503
603
  | Skip disabled-path proof | `effectiveEnabled()` regression slips through | n/a (permanent) | Tier 2 above |
504
- | `npm run test:unit` against 642 files | >4 min; mis-judges verifier runtime | n/a (permanent) | Tier 1 above |
604
+ | `npm run test:unit` against the full suite (810 files now, 642 then) | several minutes; mis-judges verifier runtime | n/a (permanent) | Tier 1 above |
505
605
  | Skip typecheck | TS errors slip past `test:critical` (which uses `--test-timeout=30000`) | n/a (permanent) | Tier 3 above |
506
606
  | Run `pi` from a stale bundle | Session shows old behavior despite src/ edits | n/a (permanent) | `scripts/check-bundle-staleness.mjs` — CI gate |
507
607
  | Test by reading code | Proves nothing about runtime | n/a (permanent) | All tiers above |
508
- | `makeFakeCtx({ flagOn: false })` without `brokerEnv: "0"` | `makeFakeCtx` deletes `PI_CREW_BROKER` env if `brokerEnv` is undefined | `612e18b` (test fix) | `test/unit/crew-broker-server-gate.test.ts:78` — pass `brokerEnv: "0"` to preserve env |
608
+ | `makeFakeCtx({ flagOn: false })` without `brokerEnv: "0"` | `makeFakeCtx` deletes `PI_CREW_BROKER` env if `brokerEnv` is undefined | `612e18b` (test fix) | `test/unit/runtime/broker/crew-broker-server-gate.test.ts:78` — pass `brokerEnv: "0"` to preserve env |
509
609
  | Trust green CI on one OS | macOS/Windows regressions slip through | n/a (permanent) | `.crew/knowledge.md` — "CI runs 3 OSes ... A flake on one OS IS a real bug" |
510
- | Trusting a team-run agent not to edit the repo under test | Agents spawned by `team`/`Agent`/`crew_agent` inherit the session cwd and have `edit`/`write` tools — a proactive LLM (observed with deepseek) will make **unauthorized source edits** to pi-crew during a trivial smoke run (e.g. "improving" `chain-runner.ts` while parsing a chain string). The edit can be correct + green-tested yet still be unintended scope creep that silently lands in your commit. | n/a (permanent) | After EVERY team/subagent run: `git status` and verify each changed file was authored by you. Diff + review any surprise change before staging. Consider `workspaceMode: 'worktree'` for parallel/risky runs to isolate mutations. |
511
- | **Armed-role tool-surface bug (found live 2026-08-11)**: an opt-in tool (e.g. `scratchpad`) is armed via `ROLE_TOOL_CONFIGS[role].scratchpad=true` AND env `PI_CREW_SCRATCHPAD=1`, but NEVER appears in the worker surface. Root cause: `resolveToolPolicy` (`src/agents/agent-config.ts:165`) falls back `roleConfig.tools ?? agent.tools` when the role has no `tools` allowlist, and the builtin `agents/{executor,verifier,test-engineer}.md` frontmatter `tools:` did not list `scratchpad` → pi got `--tools read,grep,find,ls,bash,edit,write` and **hard-filtered** scratchpad. Env was correct; the tool was silently dropped by the `--tools` allowlist. Reproduce: `pi -p --tools read,grep,find,ls,bash,edit,write "list tools"` → no scratchpad; with scratchpad added → present. | `f753be30` | **Fix**: keep armed-role `agents/*.md` frontmatter `tools:` lists in sync with `ROLE_TOOL_CONFIGS` (QW17 pins the pinned roles; add the new tool to BOTH frontmatter AND role config for pinned roles, frontmatter-only for vacuous roles). A smoke run that claims the tool is "not available" in the worker is a REAL signal — verify the worker's actual `--tools` allowlist, not just env vars. |
610
+ | Trusting a team-run agent not to edit the repo under test | Agents spawned by `team`/`Agent`/`crew_agent` inherit the session cwd and have `edit`/`write` tools — a proactive LLM (observed with deepseek) will make **unauthorized source edits** to pi-crew during a trivial smoke run (e.g. "improving" `chain-runner.ts` while parsing a chain string). The edit can be correct + green-tested yet still be unintended scope creep that silently lands in your commit. **Sharpened by D5 (2026-08-26)**: workers used to be tool-allowlisted (`read,grep,bash,...` by role); since full-loadout default EVERY worker has `edit`/`write` + extensions, so this risk now applies to ANY role, not just armed ones. | n/a (permanent) | After EVERY team/subagent run: `git status` and verify each changed file was authored by you. Diff + review any surprise change before staging. Consider `workspaceMode: 'worktree'` for parallel/risky runs to isolate mutations. |
611
+ | **Armed-role tool-surface bug (found live 2026-08-11; INVERTED by D5 2026-08-26)**: originally an opt-in tool (e.g. `scratchpad`) armed via `ROLE_TOOL_CONFIGS[role]` + env never appeared in the worker surface — the builtin `agents/*.md` frontmatter `tools:` allowlist hard-filtered it via `--tools`. **Since D5 (`bcb9dd5d`, spec v0.7 §10) workers are FULL pi sessions by default: no `--no-extensions`, no `--tools`, no `--no-skills` unless the agent frontmatter declares them (`src/runtime/model/pi-args.ts:283-330`) — so the default failure mode flipped.** Now a tool missing from a worker means either (a) the agent's frontmatter declares a restrictive `tools:` list (opt-in) that doesn't include it, or (b) `inheritSkills: false` / SEC-1 declaration-strip on a dynamic/project agent source. Control tools (`ask`, `delegate`) are auto-added to any declared list. Reproduce: `pi -p --tools read,bash "list tools"` → restricted; plain `pi -p` → full set. | `f753be30` → `bcb9dd5d` | **Fix**: for agents that OPT IN to restrictions, keep `agents/*.md` frontmatter `tools:` in sync with `ROLE_TOOL_CONFIGS` (add new tools to BOTH for pinned roles). A worker claiming a tool is "not available" is a REAL signal — check the worker's actual argv (`--tools` present?) and frontmatter, not just env vars. |
612
+ | **Surface test that never engages surface**: `runtime.surface.visibleAgents` defaults to `[]` (nobody). A test setting only `mode:"auto"` (already default) passes green with ZERO panes — surface's fail-closed degrade makes the headless path indistinguishable from success in the run result. | n/a (process) | Always set `visibleAgents` when testing surface, and require pane-level evidence (events `worker.surface_spawned`/`worker.surface_closed`, `tmux list-panes` during the run, sentinel PID). See Tier 10. |
613
+ | **Reading `manifest.surface.panes == {}` at run END as "zero panes engaged"** (observed 2026-08-27, full-10tier report): `releaseSurfacePane` deletes the pane entry the moment the pane closes, so a FULLY SUCCESSFUL surface run also ends with `panes:{}` — the report flipped a live herdr engagement (pane `w6:pW`, `worker.surface_spawned` seq 99) into "by-design headless, gate short-circuited". Same trap, other direction: the executor worker re-derived the gate trace from ITS OWN env (`PI_CREW_DEPTH=1` — the CORRECT and EXPECTED depth for a tier-1 worker) instead of the HOST env the gate actually reads (`child-pi.ts` passes `depthEnv ?? process.env`), concluding "headless" while literally running inside a herdr pane (`PI_CREW_SURFACE_PANE=w6:pW` sat unread in its own env). | n/a (process) | Engage-evidence = `events.jsonl` (`worker.surface_spawned` + `worker.surface_closed`, no `surface.degraded`) + `manifest.surface.provider`/`workerPids` (only the surface branch writes `workerPids`). A worker's self-report of "which path taken" is a HYPOTHESIS — workers cannot see the host's gate inputs; trust events over worker prose. |
614
+ | **Reporting "session is loading the latest code" from FILE-md5 equality alone** (disk vs symlink): a live report (2026-08-27) did exactly this — md5 disk = md5 symlink → "Tier 4 PASS" — while the parent pi process had started BEFORE the bundle rebuild and was still running the PRE-A1 bundle in memory. Every downstream anomaly then got misread as a code bug (a false "config parser drops surface" finding + root-cause misread). File equality only proves the FILES match, not what the PROCESS loaded — extension code loads at cold-start only. | n/a (permanent) | Tier 4/8 needs PROCESS-level liveness: after any rebuild, the session must `/quit` + reopen, then prove the new code is live via a behavior probe (e.g. `team action='settings' config={args:'get runtime.surface.visibleAgents'}` must recognize the key; any new run's worker env shows `PI_CREW_MAX_DEPTH=4`). Corroborate with `ps -eo pid,lstart,args | grep pi` — a session started before the rebuild mtime is stale, full stop. |
615
+ | **Assuming `ask`/messaging works because the code exists**: `ask` shipped behind `broker.waitMethodsEnabled` default `false` and slept ~3 weeks — every production wait.request was rejected `policy-disabled` while unit tests stayed green (the broker ctor is fail-closed by design; only the DEFAULT was wrong). Flipped `true` in `ceb9a68d` (2026-08-26) + "never guess, call ask" prompt guidance. | `ceb9a68d` | A worker-tool claim needs a live round-trip probe (Tier 9b-W): wait.request → park → respond → pickup, with the reply visible in the worker transcript. Gate rejections emit `policy.action` events — grep events.jsonl, don't trust silence. |
512
616
  | `Type.Unsafe({ anyOf/type })` schema field **without** `[TypeBox.Kind]` symbol | `Value.Check` throws `Unknown type` the first time a model emits that field (e.g. `skill`, `config`) — every team action returns `isError:true` text `"Unknown type"`. Tier 1-8 stay green because unit tests never send the offending field. | v0.9.57 | `src/schema/team-tool-schema.ts` — `SkillOverride`/`FreeformConfig` switched from `Type.Unsafe` to TypeBox-native `Type.Union`/`Type.Record`. See Tier 9. |
513
617
  | Schema too strict for model-emitted empty strings (`runId:""`, `workspaceMode:""`, `budgetTotal:0`) | pi-ai `validateToolArguments` runs BEFORE the pi-crew handler and rejects `""` against Literal unions / patterns → `Validation failed for tool team` → model loops. | v0.9.57 | `src/schema/team-tool-schema.ts` — added `Literal("")` to unions, `^$|` pattern for runId, `""` to action enum, `0`/Boolean allowances. Handler-side `normalizeTeamParams` drops the empties. |
514
- | Claiming "all 9 tiers pass" while 9c–9f were never run | Overclaim — once reported "9 tiers pass" when only 9a (8/10) + 9b (4/5) had actually run; 9c–9f were skipped. Past runs then become unverifiable ("did it really pass 9 tiers?"). **2026-08-11 repeat**: an initial report said "9c–9f skipped" yet the summary read as full coverage until the gap was called out. | n/a (process) | Fill `REPORT-TEMPLATE.md` per-tier DURING the run. "Tier 9 pass" = 9a AND 9b AND the applicable 9c–9f, each with evidence. Round-up-to-pass is the anti-pattern this row exists to prevent. If 9c–9f are skipped, SAY SO in the verdict and do not phrase it as "all pass". |
618
+ | Claiming "all tiers pass" while 9c–9f (or Tier 10) were never run | Overclaim — once reported "9 tiers pass" when only 9a (8/10) + 9b (4/5) had actually run; 9c–9f were skipped. Past runs then become unverifiable ("did it really pass 9 tiers?"). **2026-08-11 repeat**: an initial report said "9c–9f skipped" yet the summary read as full coverage until the gap was called out. Tier 10 adds the surface variant: a green headless run reported as "surface pass". | n/a (process) | Fill `REPORT-TEMPLATE.md` per-tier DURING the run. "Tier 9 pass" = 9a AND 9b AND the applicable 9c–9f, each with evidence; "Tier 10 pass" = pane-level evidence, not a green run. Round-up-to-pass is the anti-pattern this row exists to prevent. If tiers/sub-tiers are skipped, SAY SO in the verdict and do not phrase it as "all pass". |
515
619
  | chain run with `workflow:"chain"` forwarded to steps | Every chain step fails in ~58ms with an EMPTY error string — looks like a parse failure but isn't. `chain-dispatch` forwards `params.workflow` ("chain") into executor overrides; each step then runs the "chain" workflow via the normal `executeTeamRun` path and fails fast + silently. | Open (issue #44) | Omit `workflow` when invoking `action:'run' chain=...` — chain then runs 2/2 success (~308s). See `docs/bugs/chain-workflow-forward-quirk.md`. |
620
+ | **Env allow-list strip mux vars — async surface chết ở tầng env, không phải tầng gate** (battery 2026-08-30 Finding 2): gate async-run đã bỏ nhưng `BACKGROUND_RUNNER_ENV_ALLOWLIST` vẫn strip `TMUX`/`HERDR_*` → detached runner thấy `no-mux` → async headless mãi mãi. Gate telemetry (`asyncRun:true` trong env snapshot) nói đúng — không gate async — nhưng env detection fail vì biến bị cắt trước khi process chào. Unit test allow-list không catch (list "đúng" theo nghĩa cũ); chỉ async run LIVE với mux mới lộ. | `f0a41a16` (2026-08-30) | Mọi env var mà `src/runtime/surface/*` đọc phải có trong `BACKGROUND_RUNNER_ENV_ALLOWLIST` (pin test `test/unit/runtime/core/async-runner.test.ts` "forwards mux env"). Thêm env detection mới → thêm vào allow-list + pin test cùng lúc. |
621
+ | **`set <array-key> []` là no-op** (battery 2026-08-30 Finding 3): `parseStringList` normalize `[]` → `undefined` → patch mất key → `mergeConfig` giữ list cũ trên đĩa; `Effective` hiển thị sai giá trị đã set. `unset` vẫn hoạt động (workaround). | `5a31ccf6` (2026-08-30) | `[]` tường minh là GIÁ TRỊ, không phải unset. Test round-trip: set → get → soi config trên đĩa (test/unit/config/surface-config.test.ts F3 block). |
516
622
 
517
623
  ---
518
624
 
@@ -530,12 +636,17 @@ When a tier fails, the recovery is usually quick. Match the symptom to the cause
530
636
  | Tmux probe: keys not reaching component | Wrong terminal encoding | Check `pi-tui` env; use both `\x1b[A` and `\x1bOA`; check `matchesKey` is wired in the dispatched class |
531
637
  | `pty_probe.py` errors `OSError: [Errno 6] No such device` | Pty already closed | Reduce `--startup-sleep` or check `pi` actually launched |
532
638
  | Smoke team: 04_verify exits with 143 | Verifier ran slow command (typically `npm test`) | Read worker transcript for actual command run; fix the verifier prompt per Tier 7 |
533
- | Smoke team: worker times out at 300s | Either verifier command slow OR LLM thinking cap | Check `RESPONSE_TIMEOUT_MS` (300s); bump only if you verified the command itself finishes <300s |
639
+ | Smoke team: worker times out (exit 143) | Either verifier command slow OR LLM thinking cap | Check `RESPONSE_TIMEOUT_MS` (600s; env override `PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS`); bump only if you verified the command itself finishes under it |
534
640
  | `stale-ctx` error in worker output | Extension ctx is stale after session replacement | This is runtime noise, not a regression; ignore. (Source: `.crew/knowledge.md` "Process Safety" notes) |
535
641
  | Bundle md5 not changing after rebuild | Stale `dist/` cache or esbuild no-op | `rm -rf dist/ && npm run build:bundle`; verify new md5 |
536
642
  | Team tool returns `Unknown type` (isError:true, short text) | `Value.Check` in the handler hit a `Type.Unsafe({...})` schema node with **no `[TypeBox.Kind]` symbol** — only triggered when the model actually sends that field. Tier 1-8 pass; only Tier 9 (feature battery) catches it. | Replace the `Type.Unsafe` with a TypeBox-native constructor (`Type.Union`, `Type.Record`, `Type.Any`). Reproduce with `node --input-type=module -e "import {Value} from '@sinclair/typebox/value'; import {TeamToolParams} from './src/schema/team-tool-schema.ts'; Value.Check(TeamToolParams, {action:'list', skill:'', config:{}})"` — a throw = the bug. |
537
643
  | `Validation failed for tool "team": ... must be equal to constant` | pi-ai `validateToolArguments` (`@earendil-works/pi-ai/dist/utils/validation.js`) rejects model-emitted `""`/`0`/`false` defaults against Literal unions / patterns / minimums — it runs BEFORE the pi-crew handler, so handler-side normalization is too late. | Loosen the schema to accept the unset marker (`Literal("")`, pattern `^$|...`, `Literal(0)`, add `Boolean()` to unions). Verify with the pi-ai validator directly: `import {validateToolArguments} from '@earendil-works/pi-ai'; validateToolArguments({name:'team',parameters:TeamToolParams},{name:'team',arguments:{...fullModelBlob}})`. |
538
644
  | User says "restarted" but the probe still shows the OLD error | Multiple `pi` PIDs open; the user reopened a different terminal than the one the agent runs in; the agent's session never reloaded the bundle. | `ps -eo pid,lstart,tty,args \| grep pi` to list PIDs; match the agent's session log (the `.jsonl` being appended right now) to its PID; have the user reopen THAT session, or move the work into the freshly-opened one. |
645
+ | Surface run green but zero panes created | `runtime.surface.visibleAgents` still `[]` (default — visible to nobody), or the env has no live mux (no `$TMUX` / no herdr socket / `runtime.surface.mode: off`), or depth > `maxDepth`. **NOT async anymore** — async runs are no longer hard-gated headless (2026-08-27): surface now follows env + `runtime.surface.*` config, not run-mode. | Set `visibleAgents` (`team-settings set runtime.surface.visibleAgents '["*"]'`) — note `[]` is a silent no-op, `unset` removes it. If run async, it still engages panes when env has a live mux. Check `worker.surface_gate_blocked` events (mode/depth/cap/role/no-mux) in `events.jsonl`. See Tier 10. |
646
+ | Worker boots in pane then dies instantly / pane flashes | Launch script env broken (missing `PI_CREW_SURFACE_PANE`, wrong cwd) or parent-guard tripped (host PID died / starttime mismatch) | Read the pane's recorder log (`agents/{taskId}/events.jsonl`) + degrade.classify events in run `events.jsonl`; check `PI_CREW_PARENT_PID` propagation in `src/runtime/child-pi/child-pi-spawn.ts` |
647
+ | `ask` tool fast-fails "proceed with best judgment" | `broker.waitMethodsEnabled: false` somewhere (user config can re-close the default) | `team-settings get broker.waitMethodsEnabled`; expect `true` (default since `ceb9a68d`); grep events.jsonl for `policy.action` |
648
+ | `delegate` rejects with a policy message | By design when depth cap hit (`maxDepth: 4`) or `nesting.enabled: false` in USER config (sensitive — project cannot flip) | Check depth in the rejection payload; `delegate.rejected` event in events.jsonl confirms the structured (non-silent) path |
649
+ | herdr provider never engages | pi is not itself running inside a herdr pane (design: no socket guessing) | Run pi inside herdr, then `runtime.surface.mode` auto/`herdr`; verify `~/.config/herdr/herdr.sock` responds |
539
650
 
540
651
  ## Performance budget (per-tier soft limits)
541
652
 
@@ -547,9 +658,12 @@ When a tier fails, the recovery is usually quick. Match the symptom to the cause
547
658
  | 4 (md5 sync check) | <1s | 5s | Disk/symlink issue |
548
659
  | 5 (tmux spawn) | 5s | 15s | tmux server issue |
549
660
  | 6 (pty probe) | 5s | 15s | `pi` not in PATH |
550
- | 7 (smoke team) | 60s (verifier only) | 300s (worker hard limit) | Worker killed by `RESPONSE_TIMEOUT_MS` |
661
+ | 7 (smoke team) | 60s (verifier only) | 600s (worker hard limit) | Worker killed by `RESPONSE_TIMEOUT_MS` |
551
662
  | 8 (final md5 sync) | <1s | 5s | Disk/symlink issue |
552
- | 9 (feature battery) | 30s (read-only batch) + ~120s per spawn probe | 300s per spawn probe (worker hard limit) | Spawn probe hung or returned `Unknown type`/`Validation failed` — a schema or registration regression; see Tier 9 + Failure symptoms |
663
+ | 9 (feature battery) | 30s (read-only batch) + ~120s per spawn probe | 600s per spawn probe (worker hard limit) | Spawn probe hung or returned `Unknown type`/`Validation failed` — a schema or registration regression; see Tier 9 + Failure symptoms |
664
+ | 10a (surface E2E suite) | 90s | 180s | tmux server issue or a real spawn/degrade regression — investigate, don't bump |
665
+ | 10b (live surface run) | ~120s (one fast-fix run) | 600s (worker hard limit) | Pane never engaged (check `visibleAgents`) or auto-exit failed leaving panes open |
666
+ | 10c (herdr path) | ~120s | 600s | herdr socket protocol drift — check `herdr api schema --json` against `src/runtime/surface/herdr-provider.ts` |
553
667
 
554
668
  If a tier runs over the hard limit, **stop and investigate** — don't bump the budget silently. The budget exists precisely so regressions in test runtime (which usually means a regression in test setup/teardown) are caught early.
555
669
 
@@ -624,12 +738,45 @@ The "skill stack" for a typical pi-crew change:
624
738
  4. tier 3 (typecheck + bundle) ← this skill
625
739
  5. tier 5/6 (live TUI) ← this skill, if ui change
626
740
  6. tier 7 (smoke team) ← this skill, if plan/workflow change
627
- 7. commit + push
628
- 8. verify-before-complete ← make the "done" claim with evidence
741
+ 7. tier 9 (feature battery) ← this skill, if schema/tool-surface change
742
+ 8. tier 10 (surface battery) ← this skill, if surface/pane change
743
+ 9. commit + push
744
+ 10. verify-before-complete ← make the "done" claim with evidence
629
745
  ```
630
746
 
631
747
  ---
632
748
 
749
+ ## Feature coverage map (tính năng → tier verify)
750
+
751
+ Use this to answer "đủ full tính năng chưa?" without re-deriving. Every user-facing pi-crew feature, and the cheapest tier that proves it live. If a feature row has no evidence in the report, the battery was not "full" — regardless of how many tiers ran.
752
+
753
+ | Feature | Code entry | Verify via |
754
+ |---|---|---|
755
+ | Team tool — 55 actions / 5 domains | `src/schema/team-tool-schema.ts:391-437`, dispatch in `src/extension/team-tool/` | 9a (read-only) + 9b/9c/9d/9e/9f theo domain |
756
+ | Runtime mode `child-process` (default) | `src/runtime/child-pi/` | 9b sync run + T7 |
757
+ | Runtime mode `scaffold` (dry-run) | `src/runtime/task-runner/pre-execution.ts:176` | 9b `action='plan'`/`'plans'` (preview không spawn) hoặc run với `runtime.mode='scaffold'` |
758
+ | Runtime mode `live-session` (experimental) | `src/runtime/live-session/` | Run với `runtime.mode='live-session'` + irc tool xuất hiện trong worker (`src/runtime/custom-tools/irc-tool.ts`) |
759
+ | Subagent tools (Agent / steer / result) | `src/extension/registration/subagent-tools.ts` | 9b (`Agent`, `crew_agent`+`get_subagent_result`, `steer_subagent`) |
760
+ | Worker tool `ask` (blocking Q→parent) | `src/prompt/prompt-runtime.ts:639`, broker wait.* | 9b-W ask round-trip |
761
+ | Worker tool `message` (notify/DM/group) | `src/prompt/message-tool.ts` | 9b-W message probes |
762
+ | Worker tool `delegate` (nested spawning) | `src/prompt/prompt-runtime.ts:414` | 9b-W delegate + depth-cap reject |
763
+ | Full loadout (D5) | `src/runtime/model/pi-args.ts:283-330` | 9b-W full-loadout sanity |
764
+ | Surface panes tmux/herdr (A1) | `src/runtime/surface/` | T10 (10a E2E + 10b live + 10c herdr) |
765
+ | Broker (mailbox, steer, tokens) | `src/runtime/broker/` | T1/T2 + 9c steer/respond + T10a test #2 |
766
+ | Dashboard + keybindings + overlays | `src/ui/`, commands `src/extension/registration/commands/` | T5/T6 probe + parity golden test |
767
+ | Slash commands (8: run/status/doctor/help/dashboard/settings/init/config) | `commands/{run,status,manage,dashboard}.ts` | T5 send-keys một lệnh `/team-*` |
768
+ | team-settings / config | `src/extension/team-tool/handle-settings.ts` | 9a settings get + 10b set visibleAgents |
769
+ | Worktree isolation | `src/worktree/` | 9a worktrees + 9b run `workspaceMode='worktree'` |
770
+ | Async detached runs + watchdog | `src/runtime/async-runner.ts` | 9b async + 9f (survive host exit: E2E riêng) |
771
+ | Crash recovery / resume | `src/state/`, 9c | 9c resume/retry + checkpoint |
772
+ | Export/import bundles | `src/extension/team-tool/` (import/imports/export) | 9e |
773
+ | Schedule/cron, goal-loop, anchors | AUTOMATE domain | 9f |
774
+ | Doctor / health / zombies + orphan panes | `src/extension/team-tool/doctor.ts` | 9a doctor + T10a test #3 |
775
+ | Model fallback chain | `src/config/types.ts` (modelFallback) | unit tests + 9b sync run (auto-tail chay ngầm) |
776
+ | State perf (fsync coalescing, event-log tail) | `src/state/` | bench `scripts/run-bench.mjs` (b5/b11-b13) — không cần battery live |
777
+
778
+ ---
779
+
633
780
  ## Maintenance
634
781
 
635
782
  The skill mentions specific commits, line numbers, and version pins. As the code evolves, these will drift. Maintenance playbook:
@@ -639,8 +786,10 @@ The skill mentions specific commits, line numbers, and version pins. As the code
639
786
  | Verify line refs after each `src/` commit | Every commit touching the cited file | `git log -p -- src/extension/registration/lifecycle-handlers.ts \| grep effectiveEnabled` — if line moved, update the skill |
640
787
  | Verify commit hashes still exist | Quarterly or before major edits | `git log --oneline -1 <hash>` — if gone, find the equivalent newer commit |
641
788
  | Verify version pins (v0.9.46, etc.) | Each release | `git log --oneline -- src/ui/run-dashboard.ts \| head -5` — confirm diag removal history (e3ee6fe2) still accurate |
642
- | Verify `test:critical` still has 14 files | Each `src/runtime/crew-broker*.ts` edit | `cat package.json \| grep test:critical` — adjust the file list |
789
+ | Verify `test:critical` still has 14 files | Each `src/runtime/broker/*.ts` edit | `grep test:critical package.json` — adjust the file list |
643
790
  | Verify Tier 7 verifier prompts still say `test:critical` | Each workflow file edit | `grep "Run FAST checks" workflows/*.workflow.md` |
791
+ | Verify Tier 10 surface refs | Each `src/runtime/surface/**` edit | `ls test/system/surface-*.e2e.test.ts` + grep `MAX_SURFACE_WORKERS` in resolve-surface.ts — cap/config shape may drift between A1 → A2 |
792
+ | Verify herdr wire details | Each herdr release bump | `herdr api schema --json` vs `src/runtime/surface/herdr-provider.ts` (envelope/pane.read source/1-conn-per-request were verified on herdr 0.8.2) |
644
793
 
645
794
  The skill does NOT need to be updated for every commit — only when the cited lines/files move. Consider it a "living reference" not a "live spec".
646
795
 
@@ -649,7 +798,7 @@ The skill does NOT need to be updated for every commit — only when the cited l
649
798
  ## Quick reference — exact commands
650
799
 
651
800
  ```bash
652
- # Tier 1 (critical unit, ~25s, 101 tests)
801
+ # Tier 1 (critical unit, ~21s, 102 tests)
653
802
  npm run test:critical
654
803
  # Tier 2 (3-path proof, broker changes only)
655
804
  PI_CREW_BROKER=0 npm run test:critical
@@ -675,11 +824,18 @@ md5sum dist/index.mjs
675
824
  md5sum "$(npm root -g)"/pi-crew/dist/index.mjs 2>/dev/null \
676
825
  || md5sum ../node_modules/pi-crew/dist/index.mjs
677
826
  # Tier 9 (feature battery — from parent Pi session, tool calls not shell)
678
- # read-only: team action=list / recommend / health / doctor / status / events / summary / get / explain / worktrees
827
+ # read-only: team action=list / recommend / health / doctor / status / events / summary / get / explain / worktrees / settings
679
828
  # spawn: team action=run (sync) ; team action=run async=true ; team action=run chain='"A" -> "B"'
680
- # Agent (direct) ; crew_agent run_in_background=true + get_subagent_result
829
+ # Agent (direct) ; crew_agent run_in_background=true + get_subagent_result ; steer_subagent
830
+ # worker tools (goal-text probes): ask round-trip ; message notify/DM/group ; delegate nesting (depth-cap reject)
681
831
  # reproduce the two silent schema failures:
682
832
  # node --input-type=module -e "import {Value} from '@sinclair/typebox/value'; import {TeamToolParams} from './src/schema/team-tool-schema.ts'; Value.Check(TeamToolParams, {action:'list', skill:'', config:{}})" # throws 'Unknown type' = Type.Unsafe-without-Kind bug
833
+ # Tier 10 (surface battery)
834
+ # team-settings set runtime.surface.visibleAgents '["*"]' # opt-in — default [] engages NOTHING
835
+ tmux list-panes -a -F '#{pane_id} #{pane_title} #{pane_pid}' # during run: pane per taskId
836
+ # E2E suite (must run inside tmux):
837
+ node --experimental-strip-types --test --test-concurrency=1 --test-timeout=120000 test/system/surface-tmux.e2e.test.ts
838
+ # doctor orphan panes: team action='doctor' focus='zombies'
683
839
  ```
684
840
 
685
841
  ---
@@ -688,17 +844,18 @@ md5sum "$(npm root -g)"/pi-crew/dist/index.mjs 2>/dev/null \
688
844
 
689
845
  Before claiming "tested":
690
846
 
691
- - [ ] Tier 1: `test:critical` fresh-run, all pass (<25s). Count varies by release — was 97 at v0.9.46, **101 since the model-routing merge (v0.9.66)**; record the actual count in the report.
847
+ - [ ] Tier 1: `test:critical` fresh-run, all pass (<25s). Count varies by release — was 97 at v0.9.46, 101 at v0.9.66, **102 since the waitMethodsEnabled flip**; record the actual count in the report.
692
848
  - [ ] Tier 2: 3-path proof all pass — **required if you touched `src/config/defaults.ts` or `src/extension/registration/lifecycle-handlers.ts`**
693
849
  - [ ] Tier 3: `npm run typecheck` exit 0, `npm run build:bundle` exit 0
694
850
  - [ ] Tier 4: bundle md5 matches what the session loaded (or user has `/quit`-ed + reopened)
695
851
  - [ ] Tier 5/6: live TUI smoke for any `src/ui/` change — keystroke reached `handleInput`
696
- - [ ] Tier 7: smoke team run for any `src/runtime/plan-templates.ts` or `workflows/*.workflow.md` change — completed, no hang, verifier output under 60s
852
+ - [ ] Tier 7: smoke team run for any `src/runtime/goal-workflow/plan-templates.ts` or `workflows/*.workflow.md` change — completed, no hang, verifier output under 60s
697
853
  - [ ] Tier 8: final md5 sync check passed
698
854
  - [ ] Tier 9: feature battery — **required if you touched `src/schema/team-tool-schema.ts`, `src/extension/registration/team-tool.ts`, any `Type.Unsafe({...})` schema, or any armed-role tool list (`agents/*.md` / `src/config/role-tools.ts`)**. 9a read-only batch all return clean; one probe per 9b spawn path (sync / async / chain / `Agent` / `crew_agent`+`get_subagent_result`) completes with `consistency=1`. Run 9c–9f only when the change touches their code path; **at least one full 9c/9e/9f sweep per release is recommended so the battery stays proven** (see `real-test-2026-08-11-scratchpad-I-batch.md`); 9d (destructive) requires explicit user confirmation. **After every run: `git status` to catch unauthorized agent edits.**
699
855
  - [ ] **Output report**: save `docs/real-test/reports/real-test-<YYYY-MM-DD>-<slug>.md` from `skills/real-test-pi-crew/REPORT-TEMPLATE.md`, filled DURING the run with per-tier evidence (counts/md5/runId) — not reconstructed from memory afterward. This is what makes past runs verifiable instead of trust-the-summary.
856
+ - [ ] Tier 10: surface battery — **required if you touched `src/runtime/surface/**`, `src/prompt/surface-worker.ts`, the surface branch of `src/runtime/child-pi/child-pi.ts`, or the surface config keys**. 10a E2E 3/3 per backend available (tmux trong tmux; herdr ngoài tmux + socket sống — skip vì thiếu mux là correct-by-design nhưng KHÔNG tính pass cho backend đó); 10b live run với session ĐÃ reload bundle mới (xem Anti-patterns "file-md5 only") + `visibleAgents` set + pane-level evidence (pane id/title during run, `worker.surface_spawned`/`worker.surface_closed` events, pane auto-closed after — KHÔNG dùng `manifest.surface.panes` làm evidence engage, xem Anti-patterns "panes == {}"); 10c herdr live chỉ khi pi chạy trong herdr pane (skip kèm lý do nếu không).
700
857
 
701
- **"All 9 tiers pass" is a claim that needs per-row evidence.** Tier 9 means 9a **and** 9b **and** whichever of 9c–9f applies to the change — not "9a passed, therefore 9 passed". If any required item above is unchecked or lacks concrete evidence (a number, an md5, a runId), the answer to "is it tested?" is **no** — say so explicitly instead of rounding up to "pass".
858
+ **"All tiers pass" is a claim that needs per-row evidence.** Tier 9 means 9a **and** 9b **and** whichever of 9c–9f applies to the change — not "9a passed, therefore 9 passed". Tier 10 means pane-level evidence exists, not "run went green" (surface fail-closes to headless on every failure, so green proves nothing). If any required item above is unchecked or lacks concrete evidence (a number, an md5, a runId, a pane id), the answer to "is it tested?" is **no** — say so explicitly instead of rounding up to "pass".
702
859
 
703
860
  ---
704
861
 
@@ -710,25 +867,49 @@ Decision docs:
710
867
  - `docs/decisions/2026-07-21-broker-windows-perms.md` — Windows named-pipe perms + Phase-4 update note
711
868
 
712
869
  Source files (critical paths):
713
- - `src/config/defaults.ts:155-187` — `DEFAULT_BROKER` + `resolveBrokerEnvOverride`
714
- - `src/extension/registration/lifecycle-handlers.ts:819-833` — `effectiveEnabled()` (precedence)
715
- - `src/runtime/child-pi-constants.ts:23` — `RESPONSE_TIMEOUT_MS = 300_000`
716
- - `src/runtime/plan-templates.ts:143, 146, 190, 193` — verifier `taskTemplate` + `verificationCommand`
717
- - `src/runtime/crew-broker.ts` — broker server (per-connection gate, NDJSON framing)
718
- - `src/runtime/crew-broker-client.ts` — client (`isEventFrame()` distinguishes event vs response frames)
719
- - `src/runtime/crew-broker-tokens.ts` — `BrokerTokenRegistry` with `timingSafeEqual`
720
- - `src/runtime/broker-issuer.ts` — per-run broker issuer (env injection at spawn)
721
- - `src/runtime/crew-broker-child.ts` — child-side broker client wiring
870
+ - `src/config/defaults.ts:191` — `DEFAULT_BROKER` (`:205` `waitMethodsEnabled: true`), `:221` `DEFAULT_NESTING`, `:252` `resolveBrokerEnvOverride`
871
+ - `src/extension/registration/lifecycle-handlers.ts:1026-1039` — `effectiveEnabled()` (precedence)
872
+ - `src/runtime/child-pi/child-pi-constants.ts:23` — `RESPONSE_TIMEOUT_MS = 300_000`
873
+ - `src/runtime/goal-workflow/plan-templates.ts:144, 147, 151` — verifier `taskTemplate` + `verificationCommand`
874
+ - `src/runtime/broker/crew-broker.ts` — broker server (per-connection gate, NDJSON framing)
875
+ - `src/runtime/broker/crew-broker-client.ts` — client (`isEventFrame()` distinguishes event vs response frames)
876
+ - `src/runtime/broker/crew-broker-tokens.ts` — `BrokerTokenRegistry` with `timingSafeEqual`, secret-based revocation
877
+ - `src/runtime/broker/broker-issuer.ts` — per-run broker issuer (env injection at spawn)
878
+ - `src/runtime/broker/crew-broker-child.ts` — child-side broker client wiring
722
879
  - `src/ui/key-utils.ts:37-42` — `keyOf()` using pi-tui `matchesKey()`
723
880
  - `src/ui/keybinding-map.ts` — dispatch using `matchesKey()` (commit `f05a10d`)
881
+ - `src/runtime/model/pi-args.ts:283-330` — D5 loadout: `--tools`/`--no-skills` ONLY khi agent frontmatter khai báo; `DEFAULT_MAX_CREW_DEPTH = 4`
882
+ - `src/extension/registration/subagent-tools.ts` — `Agent` (:70), `get_subagent_result` (:359), `steer_subagent` (:475, alias `crew_agent*`)
883
+ - `src/runtime/live-session/live-session-runtime.ts` + `src/runtime/custom-tools/irc-tool.ts` — live-session mode + peer-to-peer irc (experimental)
884
+
885
+ Surface files (Tier 10 critical paths):
886
+ - `src/runtime/surface/surface-provider.ts` — SurfaceProvider interface (spec §4)
887
+ - `src/runtime/surface/resolve-surface.ts` — fail-closed detect matrix (spec §3), `MAX_SURFACE_WORKERS = 6`
888
+ - `src/runtime/surface/tmux-provider.ts` / `herdr-provider.ts` — pane lifecycle per backend (herdr: 1 req = 1 conn, only-in-pane detect)
889
+ - `src/runtime/surface/surface-spawn.ts` — prepareSurfaceSpawn + waitForSurfaceExit (env `PI_CREW_SURFACE`, `PI_CREW_SURFACE_PANE`, `PI_CREW_AUTO_EXIT`, `PI_CREW_PARENT_PID`)
890
+ - `src/runtime/surface/degrade.ts` — classifyOnExit 2s, cause-group lockout, spawn-fail streak 3, headless resume
891
+ - `src/runtime/surface/launch-script.ts` — 0600 script builder + TTL sweep + depth guard
892
+ - `src/prompt/surface-worker.ts` — recorder (seq-seeded), auto-exit via `ctx.shutdown()`, parent-guard `/proc` starttime
893
+ - `src/extension/team-tool/doctor.ts:522+` — T12 orphan surface-pane cleanup + surface telemetry
894
+
895
+ Worker-tool files (Tier 9b-W):
896
+ - `src/prompt/prompt-runtime.ts:414, 639, 1053-1059` — `delegate` / `ask` registration (+ `message` via `src/prompt/message-tool.ts`)
897
+ - `src/prompt/message-tool.ts` + `inbox-poll.ts` — message tool (rate-limit 10/60s, `from` broker override), inbox pickup fences messages as DATA
898
+ - `src/prompt/worker-events-channel.ts` — `emitTerminal()` bypasses rate-limit
899
+ - `src/config/types.ts:94` — `runtime.surface` config shape; `src/extension/team-tool/handle-settings.ts:23-24` — team-settings keys
724
900
 
725
901
  Test files (the 14 in `test:critical`):
726
- - `test/unit/crew-broker-{handshake,stale-socket,feature-flag,server-gate,client-fallback,mailbox-observer,close-during-reconnect,steer-dedup,symlink-steering}.test.ts`
727
- - `test/unit/keybinding-map.parity.test.ts`
728
- - `test/unit/pi-tui-dispatch-probe.test.ts`
729
- - `test/unit/session-utils-extract.test.ts`
730
- - `test/unit/config-schema-sync.test.ts`
731
- - `test/unit/child-pi-env-spread.test.ts`
902
+ - `test/unit/runtime/broker/crew-broker-{handshake,stale-socket,feature-flag,server-gate,client-fallback,mailbox-observer,close-during-reconnect,steer-dedup,symlink-steering}.test.ts`
903
+ - `test/unit/ui/keybinding-map.parity.test.ts`
904
+ - `test/unit/ui/pi-tui-dispatch-probe.test.ts`
905
+ - `test/unit/utils/session-utils-extract.test.ts`
906
+ - `test/unit/config/config-schema-sync.test.ts`
907
+ - `test/unit/runtime/child-pi/child-pi-env-spread.test.ts`
908
+
909
+ Surface tests (Tier 10):
910
+ - `test/system/surface-tmux.e2e.test.ts` — 3 E2E tests, gated `CI || ! $TMUX` (spawn/self-close, kill-pane→degrade→headless resume, doctor orphan cleanup)
911
+ - `test/unit/runtime/surface/` — resolve-surface, degrade, prepare-surface-spawn, surface-spawn unit tests
912
+ - `test/unit/config/surface-config.test.ts` — config shape + team-settings keys
732
913
 
733
914
  Integration tests (Tier 1 covers none — these are for full E2E):
734
915
  - `test/integration/crew-broker-msg.test.ts` — 5 tests (Phases 1)
@@ -746,6 +927,24 @@ Commits (chronological, the patterns they introduced):
746
927
  - `612e18b` — Phase 4 default-on flip (code + decision doc)
747
928
  - `4186284` — mark default-off doc SUPERSEDED + index update
748
929
 
930
+ MuxSurface A1 wave (2026-08-26/27, branch `feature/mux-surface-a1` → main at `ec1ba5d3`):
931
+ - `ceb9a68d` — ask gate flip: `waitMethodsEnabled` default `true` + never-guess guidance
932
+ - `bcb9dd5d` — D5 loadout: worker = full pi session by default (restriction opt-in via frontmatter)
933
+ - `de671c5d` — D8 nesting: `delegate` tool for every role, depth cap 4
934
+ - `f843e14a` / `49ca2468` / `fcb68713` — D9 `message` tool + broker from-override + wake pattern
935
+ - `a77127fd` — `runtime.surface` config + team-settings keys
936
+ - `b2851e98` → `04d86582` / `1854a532` — SurfaceProvider interface + tmux/herdr providers
937
+ - `c29a1370` / `c2ba6f2d` / `9c5ad869` — launch script + spawn branch + recorder/auto-exit/parent-guard
938
+ - `5b7b8033` — EventLogTailSource (host tails per-agent event log)
939
+ - `2eb6cfb4` / `7065cb9d` / `69803eb7` — broker token revocation (stale-token + secret-based check)
940
+ - `df861630` — degrade flow: classify timeout, cause-group lockout, spawn-fail lockout, headless resume
941
+ - `f0586a74` — doctor zombie surface fields + orphan pane cleanup
942
+ - `7340305b` / `ec1ba5d3` — ADR + spec errata + test matrix; herdr race synthetic-exit fix
943
+
944
+ Spec + ADR for the surface feature:
945
+ - `docs/superpowers/specs/2026-08-26-mux-surface-design.md` — spec v0.7.1 (D1-D9, §12 data contracts, §13 sequences, §14 A1/A2 scope)
946
+ - `docs/decisions/2026-08-26-mux-surface-a1.md` — ADR (process ownership, A2 defer list, D7 errata)
947
+
749
948
  Real team runs (Tier 7 outcomes):
750
949
  - `team_20260722083504_cae04a2804a24d79` — full-implementation, 3/4 phases done, 04_verify hung (root cause investigation)
751
950
  - `team_20260722095143_2e58fce2ce91af19` — first fast-fix smoke, 3/3 PASS (after `test:critical` introduced)