pi-crew 0.11.2 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +44 -1
  2. package/dist/index.mjs +491 -493
  3. package/package.json +5 -2
  4. package/skills/real-test-pi-crew/SKILL.md +19 -11
  5. package/src/extension/async-notifier.ts +32 -8
  6. package/src/extension/crew-input-router.ts +9 -2
  7. package/src/extension/plan-orchestrate.ts +19 -3
  8. package/src/extension/registration/commands/shared.ts +11 -10
  9. package/src/extension/registration/foreground-run-controller.ts +2 -2
  10. package/src/extension/registration/lazy-configurers.ts +4 -1
  11. package/src/extension/registration/lifecycle-handlers.ts +16 -10
  12. package/src/extension/registration/lifecycle.ts +2 -0
  13. package/src/extension/registration/observability.ts +27 -18
  14. package/src/extension/registration/team-tool.ts +9 -2
  15. package/src/extension/registration/wire-cross-extension.ts +2 -0
  16. package/src/extension/team-manager-command.ts +3 -2
  17. package/src/extension/team-tool/goal-wrap.ts +7 -0
  18. package/src/extension/team-tool/goal.ts +12 -0
  19. package/src/extension/team-tool/parallel-dispatch.ts +16 -1
  20. package/src/extension/team-tool/run-intent.ts +1 -1
  21. package/src/extension/team-tool/run.ts +1 -1
  22. package/src/runtime/async-runner.ts +36 -0
  23. package/src/runtime/broker/crew-broker.ts +61 -62
  24. package/src/runtime/broker/delegate/delegate-event.ts +12 -8
  25. package/src/runtime/broker/protocol/connection-state.ts +10 -0
  26. package/src/runtime/broker/protocol/lock-busy.ts +76 -0
  27. package/src/runtime/delegate-spawn.ts +16 -2
  28. package/src/runtime/deterministic-ast.ts +14 -2
  29. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
  30. package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
  31. package/src/runtime/goal-workflow/goal-loop-runner.ts +32 -0
  32. package/src/runtime/group-join.ts +11 -3
  33. package/src/runtime/live-session/live-session-runtime.ts +1 -1
  34. package/src/runtime/model/runtime-warmup.ts +2 -0
  35. package/src/runtime/output/output-validator.ts +10 -5
  36. package/src/runtime/plan-approval.ts +14 -1
  37. package/src/runtime/recovery/crash-recovery.ts +75 -52
  38. package/src/runtime/subagent-manager.ts +43 -13
  39. package/src/runtime/task-runner/child-executor.ts +17 -6
  40. package/src/runtime/task-runner/post-execution.ts +44 -14
  41. package/src/runtime/task-runner/prompt-builder.ts +8 -0
  42. package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
  43. package/src/runtime/task-runner.ts +7 -0
  44. package/src/runtime/team-runner.ts +4 -1
  45. package/src/runtime/verification/verification-worktree.ts +3 -0
  46. package/src/state/coordination/mailbox.ts +7 -2
  47. package/src/state/stores/state-store.ts +8 -1
  48. package/src/ui/dashboard-panes/agents-pane.ts +11 -8
  49. package/src/ui/inline-panel/index.ts +1 -1
  50. package/src/ui/render-diff.ts +19 -6
  51. package/src/ui/run-dashboard.ts +11 -1
  52. package/src/ui/run-snapshot-cache.ts +9 -3
  53. package/src/ui/widget/task-list.ts +8 -5
  54. package/src/ui/widget/widget-model.ts +3 -1
  55. package/tsconfig.json +2 -0
  56. package/src/ui/card-colors.ts +0 -168
  57. package/src/utils/completion-dedupe.ts +0 -63
  58. package/src/utils/resolve-shell.ts +0 -36
@@ -16,6 +16,7 @@
16
16
 
17
17
  import { spawnBackgroundTeamRun } from "../../runtime/async-runner.ts";
18
18
  import { GoalStore } from "../../runtime/goal-workflow/goal-state-store.ts";
19
+ import { captureRunModelContext } from "../../runtime/model/session-model.ts";
19
20
  import { snapshotManifests } from "../../runtime/verification/verification-integrity.ts";
20
21
  import { isWorkspaceBusy } from "../../runtime/workspace-lock.ts";
21
22
  import type { TeamToolParamsValue } from "../../schema/team-tool-schema.ts";
@@ -173,6 +174,12 @@ async function handleStart(input: GoalSubActionInput): Promise<ReturnType<typeof
173
174
  // runs/<goalId>/ → "Run not found" → silent death. (Review #2 F1.)
174
175
  const paths = createRunPaths(cwd, goalId);
175
176
  const now2 = new Date().toISOString();
177
+ // Finding #1 (battery 2026-09-29): the goal-loop manifest is loaded by the
178
+ // DETACHED background-runner (no ExtensionContext). Persist the parent
179
+ // session's model routing so goal-loop turns restore it instead of falling
180
+ // back to the default chain (minimax 402 incident family). goalState keeps
181
+ // workerModel/evaluatorModel as the explicit per-role picks.
182
+ const modelContext = captureRunModelContext(ctx, typeof params.model === "string" ? params.model : undefined);
176
183
  const goalLoopManifest: TeamRunManifest = {
177
184
  schemaVersion: 1,
178
185
  runId: goalId, // paths.runId === goalId by construction
@@ -192,6 +199,7 @@ async function handleStart(input: GoalSubActionInput): Promise<ReturnType<typeof
192
199
  artifacts: [],
193
200
  ownerSessionId,
194
201
  runKind: "goal-loop",
202
+ ...(modelContext ? { modelContext } : {}),
195
203
  };
196
204
  await saveRunManifestAsync(goalLoopManifest);
197
205
  await appendEventAsync(paths.eventsPath, {
@@ -449,6 +457,9 @@ async function handleResume(input: GoalSubActionInput): Promise<ReturnType<typeo
449
457
  // Re-spawn the background loop. The loop checks goal.state === "running" before each
450
458
  // turn; since we just set it to running, it proceeds.
451
459
  try {
460
+ // Finding #1: re-capture at resume time (same battery) — the resuming
461
+ // session's live model is authoritative for subsequent turns.
462
+ const modelContext = captureRunModelContext(ctx, typeof params.model === "string" ? params.model : undefined);
452
463
  const manifest: TeamRunManifest = {
453
464
  schemaVersion: 1,
454
465
  runId: goalId,
@@ -468,6 +479,7 @@ async function handleResume(input: GoalSubActionInput): Promise<ReturnType<typeo
468
479
  artifacts: [],
469
480
  ownerSessionId: existing.ownerSessionId,
470
481
  runKind: "goal-loop",
482
+ ...(modelContext ? { modelContext } : {}),
471
483
  };
472
484
  const spawned = await spawnBackgroundTeamRun(manifest);
473
485
  return result(
@@ -10,6 +10,7 @@ import { discoverAgents } from "../../agents/discover-agents.ts";
10
10
  import { loadConfig } from "../../config/config.ts";
11
11
  import { spawnBackgroundTeamRun } from "../../runtime/async-runner.ts";
12
12
  import { resolveCrewRuntime } from "../../runtime/model/runtime-resolver.ts";
13
+ import { captureRunModelContext } from "../../runtime/model/session-model.ts";
13
14
  import type { TeamToolParamsValue } from "../../schema/team-tool-schema.ts";
14
15
  import { appendEventAsync } from "../../state/event-log/event-log.ts";
15
16
  import { createRunManifest } from "../../state/stores/state-store.ts";
@@ -66,13 +67,23 @@ export async function handleParallel(params: TeamToolParamsValue, ctx: TeamConte
66
67
 
67
68
  const runtime = await resolveCrewRuntime(config.config);
68
69
 
70
+ // Finding #1 (full battery 2026-09-29): parallel-dispatched runs re-enter
71
+ // through the detached background-runner, which has no ExtensionContext.
72
+ // Without a persisted modelContext, restoredModelRouting() returns {} and
73
+ // every worker silently falls back to the DEFAULT chain (live: 3 runs died
74
+ // on minimax/MiniMax-M3 402 while the parent session ran zai/glm-5.3).
75
+ // Captured ONCE here (in the parent session's process) so resolveParentModel()
76
+ // sees the live session model; undefined when nothing is worth persisting
77
+ // (older manifests stay byte-identical).
78
+ const modelContext = captureRunModelContext(ctx, typeof params.model === "string" ? params.model : undefined);
79
+
69
80
  const launched: Array<{ runId: string; goal: string; agent: string }> = [];
70
81
  const errors: Array<{ goal: string; error: string }> = [];
71
82
 
72
83
  // C1: Enforce concurrency limit with batched spawning
73
84
  for (let batchStart = 0; batchStart < tasksParam.length; batchStart += concurrency) {
74
85
  const batch = tasksParam.slice(batchStart, batchStart + concurrency);
75
- const batchPromises = batch.map((task) => spawnSingleTask(task, ctx, allAgentsList, team, workflow, runtime));
86
+ const batchPromises = batch.map((task) => spawnSingleTask(task, ctx, allAgentsList, team, workflow, runtime, modelContext));
76
87
  const batchResults = await Promise.allSettled(batchPromises);
77
88
  for (const res of batchResults) {
78
89
  if (res.status === "fulfilled" && res.value.ok) {
@@ -121,6 +132,7 @@ async function spawnSingleTask(
121
132
  team: TeamConfig,
122
133
  workflow: WorkflowConfig,
123
134
  runtime: { available: boolean; kind: string },
135
+ modelContext: ReturnType<typeof captureRunModelContext>,
124
136
  ): Promise<SpawnResult> {
125
137
  try {
126
138
  const taskRec = task as Record<string, unknown>;
@@ -163,6 +175,9 @@ async function spawnSingleTask(
163
175
  // skipped and any session can cancel/retry parallel-dispatched runs.
164
176
  ownerSessionId: ctx.sessionId,
165
177
  runKind: "team-run",
178
+ // Finding #1: persist the parent session's model routing so the detached
179
+ // runner restores it instead of default-chaining (see handleParallel).
180
+ modelContext,
166
181
  });
167
182
 
168
183
  await appendEventAsync(created.manifest.eventsPath, {
@@ -73,7 +73,7 @@ const MAX_ANALYSIS_BYTES = 100_000;
73
73
  var crewInitPromise: Promise<typeof import("../../state/crew-init.ts")> | undefined;
74
74
  function loadCrewInit(): Promise<typeof import("../../state/crew-init.ts")> {
75
75
  if (!crewInitPromise) {
76
- crewInitPromise = import("../../state/crew-init.ts");
76
+ crewInitPromise = import("../../state/crew-init.ts"); // LAZY: defer crew-init until first run intent
77
77
  }
78
78
  return crewInitPromise;
79
79
  }
@@ -61,7 +61,7 @@ import type { RuntimeResolutionState, TeamRunManifest, TeamTaskState } from "../
61
61
  var crewInitPromise: Promise<typeof import("../../state/crew-init.ts")> | undefined;
62
62
  function loadCrewInit(): Promise<typeof import("../../state/crew-init.ts")> {
63
63
  if (!crewInitPromise) {
64
- crewInitPromise = import("../../state/crew-init.ts");
64
+ crewInitPromise = import("../../state/crew-init.ts"); // LAZY: defer crew-init until first run intent
65
65
  }
66
66
  return crewInitPromise;
67
67
  }
@@ -240,6 +240,16 @@ export const BACKGROUND_RUNNER_ENV_ALLOWLIST: string[] = [
240
240
  "HERDR_SOCKET_PATH",
241
241
  "HERDR_WORKSPACE_ID",
242
242
  "HERDR_PING_TIMEOUT_MS",
243
+ // Model-selection control vars (Finding #1, full battery 2026-09-29): the
244
+ // detached runner resolves subagent model routing from env (PI_CREW_MODEL →
245
+ // resolveDefaultSubagentModel; PI_CREW_MODEL_FALLBACK_ORDER →
246
+ // resolveModelFallbackPolicy) for manifests without a persisted modelContext
247
+ // (older manifests) and as a user override channel. Stripping them made
248
+ // every async run silently fall back to the DEFAULT chain (live: 3 parallel
249
+ // runs died on minimax/MiniMax-M3 402 while the parent ran zai/glm-5.3).
250
+ // PI_CREW_MODEL_REQUIRE_CREDENTIALS is NOT here — see forwardModelControlEnv.
251
+ "PI_CREW_MODEL",
252
+ "PI_CREW_MODEL_FALLBACK_ORDER",
243
253
  ];
244
254
 
245
255
  /**
@@ -256,6 +266,29 @@ export function buildBackgroundRunnerEnv(env: NodeJS.ProcessEnv): NodeJS.Process
256
266
  return { ...env, PI_CREW_ASYNC_RUN: "1" };
257
267
  }
258
268
 
269
+ /**
270
+ * Model-selection control vars that CANNOT live on
271
+ * {@link BACKGROUND_RUNNER_ENV_ALLOWLIST}: isSecretKey() matches the
272
+ * "_CREDENTIALS" suffix of PI_CREW_MODEL_REQUIRE_CREDENTIALS (keyword
273
+ * "credential"), and sanitizeEnvSecrets THROWS on secret-looking allowlist
274
+ * entries that are absent from the spawning env — so allowlisting it would
275
+ * break EVERY async spawn whenever the flag is unset. It is a declared
276
+ * control flag (config/env-vars.ts, read by resolveModelFallbackPolicy),
277
+ * never a secret, so we forward it explicitly when set instead.
278
+ * Exported pure so tests pin the contract (async-runner.test.ts).
279
+ */
280
+ export const MODEL_CONTROL_ENV_EXPLICIT: readonly string[] = ["PI_CREW_MODEL_REQUIRE_CREDENTIALS"];
281
+
282
+ /** Copy the explicitly-forwarded model control vars (set ones only). */
283
+ export function forwardModelControlEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
284
+ const forwarded: NodeJS.ProcessEnv = {};
285
+ for (const key of MODEL_CONTROL_ENV_EXPLICIT) {
286
+ const value = env[key];
287
+ if (value !== undefined) forwarded[key] = value;
288
+ }
289
+ return forwarded;
290
+ }
291
+
259
292
  /** F4 v2: the ONE line written to the background-runner's stdin carrying
260
293
  * PER-TASK compound broker tokens (see stdin-handshake.ts for why compound —
261
294
  * ADR-0 item 6 rejects bare-runId tokens for wait.*). Kept pure + exported so
@@ -377,6 +410,9 @@ export async function spawnBackgroundTeamRun(manifest: TeamRunManifest): Promise
377
410
  const childEnv = buildBackgroundRunnerEnv({
378
411
  ...(peerDepDir ? { ...filteredEnv, [PEER_DEP_DIR_ENV]: peerDepDir } : filteredEnv),
379
412
  PI_CREW_BACKGROUND_RUNNER_ENTRY: "1",
413
+ // Finding #1: model control vars that cannot ride the allowlist (see
414
+ // forwardModelControlEnv). Copied from the ORIGINAL env, post-sanitize.
415
+ ...forwardModelControlEnv(process.env),
380
416
  });
381
417
 
382
418
  const loader = resolveTypeScriptLoader();
@@ -28,7 +28,7 @@ import { withRunLockSync } from "../../state/coordination/locks.ts";
28
28
  import { appendMailboxMessageAsync, type MailboxMessage, registerMailboxAppendObserver } from "../../state/coordination/mailbox.ts";
29
29
  import { appendEventAsync } from "../../state/event-log/event-log.ts";
30
30
  import { loadRunManifestById, saveRunManifest, saveRunTasks } from "../../state/stores/state-store.ts";
31
- import type { TeamTaskState } from "../../state/types.ts";
31
+ import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
32
32
  import { runEventBus } from "../../ui/run-event-bus.ts";
33
33
  import { logInternalError } from "../../utils/internal-error.ts";
34
34
  import { BrokerError, encodeBrokerFrame, MAX_BROKER_FRAME_BYTES, NdjsonDecoder } from "../../utils/ndjson.ts";
@@ -40,11 +40,12 @@ import { resolveCrewMaxDepth } from "../model/pi-args.ts";
40
40
  import { NestedSlotBudget } from "../scheduling/nested-slots.ts";
41
41
  import { evaluateDelegateAdmission } from "../spawn-policy.ts";
42
42
  import { type BrokerToken, BrokerTokenRegistry } from "./crew-broker-tokens.ts";
43
- import { recordDelegateEvent } from "./delegate/delegate-event.ts";
43
+ import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent } from "./delegate/delegate-event.ts";
44
44
  import { promoteShadowToRunning } from "./delegate/shadow-lifecycle.ts";
45
45
  import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
46
46
  import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
47
47
  import { handleEventsSince } from "./protocol/events-replay.ts";
48
+ import { DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, type LockBusyOutcome, withRunLockBusyRetry } from "./protocol/lock-busy.ts";
48
49
  import { loadRunForHello } from "./protocol/manifest-loader.ts";
49
50
  import { handleMsgInbox } from "./protocol/msg-inbox.ts";
50
51
  import {
@@ -104,6 +105,7 @@ export class CrewBroker {
104
105
  | "grandchildSpawner"
105
106
  | "modelCatalog"
106
107
  | "serializeOnPathOverlap"
108
+ | "lockBusyRetryDelaysMs"
107
109
  >;
108
110
  private readonly tokens = new BrokerTokenRegistry();
109
111
  /** Task 10 (mux-surface A1 §5.2): taskId → the compound token most
@@ -1332,28 +1334,31 @@ export class CrewBroker {
1332
1334
  }
1333
1335
 
1334
1336
  // WP-2/R2: wait.request / wait.resolve (ADR-0 2026-08-17-waiting-producer-ask)
1337
+ // waitAuthError: imported directly from protocol/wait-auth.ts (M4/WI-4.1).
1335
1338
 
1336
- // waitAuthError: protocol/wait-auth.ts (M4/WI-4.1).
1337
- private waitAuthError(conn: ServerConnection): { code: string; message: string } | null {
1338
- return waitAuthError(conn);
1339
- }
1340
-
1341
- // (The "ADR item 7" doc that used to dangle here documents
1342
- // recordWaitPolicyRejection — see wait-auth.ts, where it belongs.
1343
- // Removed 2026-09-10, review F6.)
1344
1339
  // T3/R5 (ADR-5): delegate.request — governed-nesting admission + background
1345
1340
  // grandchild spawn with durable mailbox delivery (WP-5 step 5).
1346
- // getDelegateNestedSlots: inlined at 4 call sites (5-line method; M4/WI-4.1).
1341
+
1342
+ /** ADR-5 §5: lazy singleton for the nested-slot budget — first use
1343
+ * constructs it from the broker options (4 call sites previously
1344
+ * inlined this IIFE). */
1345
+ private nestedSlotBudget(): NestedSlotBudget {
1346
+ if (!this.nestedSlots)
1347
+ this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
1348
+ return this.nestedSlots;
1349
+ }
1350
+
1351
+ /** RR-023 F4: bounded busy-retry around a run-locked RMW (see
1352
+ * protocol/lock-busy.ts) — options override or the default schedule.
1353
+ * A busy-exhausted result is the handler's cue to answer a typed `busy`
1354
+ * frame (connection survives) instead of letting the throw kill it. */
1355
+ private runLockBusyRetry<T>(manifest: TeamRunManifest, fn: () => T): Promise<LockBusyOutcome<T>> {
1356
+ return withRunLockBusyRetry(manifest, this.options.lockBusyRetryDelaysMs ?? DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, fn);
1357
+ }
1347
1358
 
1348
1359
  private recordDelegateEvent(
1349
- manifest: { eventsPath: string; runId: string },
1350
- type:
1351
- | "delegate.requested"
1352
- | "delegate.admitted"
1353
- | "delegate.rejected"
1354
- | "delegate.completed"
1355
- | "delegate.timed_out"
1356
- | "delegate.rolled_up",
1360
+ manifest: DelegateEventTarget,
1361
+ type: DelegateEventType,
1357
1362
  taskId: string,
1358
1363
  data: Record<string, unknown>,
1359
1364
  ): void {
@@ -1434,7 +1439,8 @@ export class CrewBroker {
1434
1439
  this.recordDelegateEvent(loaded.manifest, "delegate.requested", parentTaskId, { subId, role: requested.role ?? "explorer" });
1435
1440
  // Admission: full spawn-policy matrix (ADR-5 §2-§7), parent state from
1436
1441
  // the RECORD under the run lock — never the worker's env/self-report.
1437
- const admissionOutcome = withRunLockSync(loaded.manifest, () => {
1442
+ // RR-023 F4: run-locked via bounded busy-retry (protocol/lock-busy.ts).
1443
+ const admissionAttempt = await this.runLockBusyRetry(loaded.manifest, () => {
1438
1444
  const fresh = loadRunManifestById(loaded.manifest.cwd, runId);
1439
1445
  if (!fresh) return { code: "no-manifest" as const, message: `run '${runId}' not found` };
1440
1446
  const task = fresh.tasks.find((t) => t.id === parentTaskId);
@@ -1474,11 +1480,7 @@ export class CrewBroker {
1474
1480
  ...(task.depth !== undefined ? { depth: task.depth } : {}),
1475
1481
  ...(task.allocation !== undefined ? { allocation: task.allocation } : {}),
1476
1482
  },
1477
- slots: (() => {
1478
- if (!this.nestedSlots)
1479
- this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
1480
- return this.nestedSlots;
1481
- })().snapshot(),
1483
+ slots: this.nestedSlotBudget().snapshot(),
1482
1484
  requested,
1483
1485
  ...(effectiveCatalog !== undefined ? { modelCatalog: effectiveCatalog } : {}),
1484
1486
  // ADR-5 §12: the delegate surface is an escalation — trusted only by the
@@ -1498,28 +1500,11 @@ export class CrewBroker {
1498
1500
  return { code: "policy-denied" as const, message: decision.message ?? decision.reason ?? "delegate denied" };
1499
1501
  }
1500
1502
  // Slot acquisition INSIDE the lock (no reserve-then-race refund window).
1501
- if (
1502
- !(() => {
1503
- if (!this.nestedSlots)
1504
- this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
1505
- return this.nestedSlots;
1506
- })().tryAcquire(subId)
1507
- ) {
1503
+ if (!this.nestedSlotBudget().tryAcquire(subId)) {
1508
1504
  this.recordDelegateEvent(fresh.manifest, "delegate.rejected", parentTaskId, { subId, reason: "slots-exhausted" });
1509
1505
  return {
1510
1506
  code: "policy-denied" as const,
1511
- message: `delegate rejected: nested spawn budget exhausted; ${
1512
- (
1513
- () => {
1514
- if (!this.nestedSlots)
1515
- this.nestedSlots = new NestedSlotBudget(
1516
- this.options.globalWorkerSemaphore ?? 4,
1517
- this.options.nestingMaxSlots,
1518
- );
1519
- return this.nestedSlots;
1520
- }
1521
- )().statusLine
1522
- }`,
1507
+ message: `delegate rejected: nested spawn budget exhausted; ${this.nestedSlotBudget().statusLine}`,
1523
1508
  };
1524
1509
  }
1525
1510
  // Reserve the requested budget pessimistically (ADR-5 §5): tokensSpent
@@ -1562,13 +1547,23 @@ export class CrewBroker {
1562
1547
  startedAt: new Date().toISOString(),
1563
1548
  } satisfies TeamTaskState,
1564
1549
  ]);
1565
- return { code: "ok" as const, decision, reserved, executionCwd };
1550
+ // RR-023 F3: thread the parent's model out of the locked read (gc fallback).
1551
+ return { code: "ok" as const, decision, reserved, executionCwd, parentModel: task.model };
1566
1552
  });
1553
+ // RR-023 F4 (finding #4): run.lock held by a live cross-process holder —
1554
+ // degrade to a typed busy frame AND a delegate.rejected event (never
1555
+ // silent); the connection survives (ping still answers).
1556
+ if (!admissionAttempt.ok) {
1557
+ this.recordDelegateEvent(loaded.manifest, "delegate.rejected", parentTaskId, { subId, reason: "run-lock-busy" });
1558
+ this.sendError(conn, id, "busy", admissionAttempt.message);
1559
+ return;
1560
+ }
1561
+ const admissionOutcome = admissionAttempt.value;
1567
1562
  if (admissionOutcome.code !== "ok") {
1568
1563
  this.sendError(conn, id, admissionOutcome.code, admissionOutcome.message);
1569
1564
  return;
1570
1565
  }
1571
- const { decision, reserved, executionCwd } = admissionOutcome;
1566
+ const { decision, reserved, executionCwd, parentModel } = admissionOutcome;
1572
1567
  this.recordDelegateEvent(loaded.manifest, "delegate.admitted", parentTaskId, {
1573
1568
  subId,
1574
1569
  childDepth: decision.childDepth,
@@ -1584,6 +1579,9 @@ export class CrewBroker {
1584
1579
  : undefined;
1585
1580
  // Return IMMEDIATELY (principle 7): the tool self-polls the mailbox.
1586
1581
  this.sendResult(conn, id, { ok: true, grandchildTaskRef: subId, childDepth: decision.childDepth, timeoutSec: decision.timeoutSec });
1582
+ // RR-023 F3 (finding #3, run team_20260929041427): model-less gc spawn fell
1583
+ // through to the pi GLOBAL default — inherit the parent task's model.
1584
+ const gcModel = requested.model ?? parentModel;
1587
1585
  // Background grandchild lifecycle (ADR-5 §1/§6).
1588
1586
  const spawner = this.options.grandchildSpawner ?? spawnDelegateGrandchild;
1589
1587
  void (async () => {
@@ -1598,7 +1596,7 @@ export class CrewBroker {
1598
1596
  eventsPath: loaded.manifest.eventsPath,
1599
1597
  role: requested.role ?? "explorer",
1600
1598
  ...(grandchildCreds ? { brokerSpawn: grandchildCreds } : {}),
1601
- ...(requested.model !== undefined ? { model: requested.model } : {}),
1599
+ ...(gcModel !== undefined ? { model: gcModel } : {}),
1602
1600
  ...(requested.maxTurns !== undefined ? { maxTurns: requested.maxTurns } : {}),
1603
1601
  timeoutSec: decision.timeoutSec ?? 900,
1604
1602
  depthOverride: decision.childDepth ?? 2,
@@ -1711,11 +1709,7 @@ export class CrewBroker {
1711
1709
  }
1712
1710
  }
1713
1711
  } finally {
1714
- (() => {
1715
- if (!this.nestedSlots)
1716
- this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
1717
- return this.nestedSlots;
1718
- })().release(subId);
1712
+ this.nestedSlotBudget().release(subId);
1719
1713
  }
1720
1714
  this.recordDelegateEvent(loaded.manifest, outcome.timedOut ? "delegate.timed_out" : "delegate.completed", parentTaskId, {
1721
1715
  subId,
@@ -1724,9 +1718,7 @@ export class CrewBroker {
1724
1718
  })();
1725
1719
  }
1726
1720
 
1727
- private recordWaitPolicyRejection(manifest: { eventsPath: string; runId: string }, taskId: string, method: string): void {
1728
- recordWaitPolicyRejection(manifest, taskId, method);
1729
- }
1721
+ // recordWaitPolicyRejection: imported directly from protocol/wait-auth.ts.
1730
1722
 
1731
1723
  /** WP-2/R2 step 4: park the calling task while its `ask` tool awaits a
1732
1724
  * leader answer. Park = task.status "waiting" + task.waiting marker +
@@ -1741,7 +1733,7 @@ export class CrewBroker {
1741
1733
  this.sendError(conn, id, "auth", "not authed");
1742
1734
  return;
1743
1735
  }
1744
- const authErr = this.waitAuthError(conn);
1736
+ const authErr = waitAuthError(conn);
1745
1737
  if (authErr) {
1746
1738
  this.sendError(conn, id, authErr.code, authErr.message);
1747
1739
  return;
@@ -1778,7 +1770,7 @@ export class CrewBroker {
1778
1770
  // Capability gate (ADR item 7): fail-closed, NEVER silent — every
1779
1771
  // rejection leaves a policy.action trace in the run's events.jsonl.
1780
1772
  if (this.options.waitMethodsEnabled !== true) {
1781
- this.recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.request");
1773
+ recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.request");
1782
1774
  this.sendError(
1783
1775
  conn,
1784
1776
  id,
@@ -1796,7 +1788,8 @@ export class CrewBroker {
1796
1788
  const deadline = Date.now() + clampSec * 1000;
1797
1789
  const runId = conn.runId;
1798
1790
  const taskId = conn.taskId;
1799
- const outcome = withRunLockSync(loaded.manifest, () => {
1791
+ // RR-023 F4: run-locked via bounded busy-retry (protocol/lock-busy.ts).
1792
+ const outcome = await this.runLockBusyRetry(loaded.manifest, () => {
1800
1793
  // Fresh reload INSIDE the lock (respond.ts:42-43 discipline).
1801
1794
  const fresh = loadRunManifestById(loaded.manifest.cwd, runId);
1802
1795
  if (!fresh) return { code: "no-manifest" as const, message: `run '${runId}' not found` };
@@ -1829,8 +1822,14 @@ export class CrewBroker {
1829
1822
  saveRunManifest(updatedManifest);
1830
1823
  return { code: "ok" as const, message: "" };
1831
1824
  });
1832
- if (outcome.code !== "ok") {
1833
- this.sendError(conn, id, outcome.code, outcome.message);
1825
+ // RR-023 F4 (finding #4): run.lock held by a live cross-process holder
1826
+ // (e.g. the detached runner) — typed busy frame, connection survives.
1827
+ if (!outcome.ok) {
1828
+ this.sendError(conn, id, "busy", outcome.message);
1829
+ return;
1830
+ }
1831
+ if (outcome.value.code !== "ok") {
1832
+ this.sendError(conn, id, outcome.value.code, outcome.value.message);
1834
1833
  return;
1835
1834
  }
1836
1835
  // Events AFTER the run lock is released (the event-log lock is a
@@ -1895,7 +1894,7 @@ export class CrewBroker {
1895
1894
  this.sendError(conn, id, "auth", "not authed");
1896
1895
  return;
1897
1896
  }
1898
- const authErr = this.waitAuthError(conn);
1897
+ const authErr = waitAuthError(conn);
1899
1898
  if (authErr) {
1900
1899
  this.sendError(conn, id, authErr.code, authErr.message);
1901
1900
  return;
@@ -1928,7 +1927,7 @@ export class CrewBroker {
1928
1927
  return;
1929
1928
  }
1930
1929
  if (this.options.waitMethodsEnabled !== true) {
1931
- this.recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.resolve");
1930
+ recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.resolve");
1932
1931
  this.sendError(
1933
1932
  conn,
1934
1933
  id,
@@ -13,15 +13,19 @@ import { logInternalError } from "../../../utils/internal-error.ts";
13
13
  /** Fire-and-forget async append; an append failure is logged, never thrown
14
14
  * (broker handlers must not block the event loop on the sync event-log lock).
15
15
  * See `crew-broker.delegate.event` event scope conventions in ADR-5 §10. */
16
+ export type DelegateEventType =
17
+ | "delegate.requested"
18
+ | "delegate.admitted"
19
+ | "delegate.rejected"
20
+ | "delegate.completed"
21
+ | "delegate.timed_out"
22
+ | "delegate.rolled_up";
23
+
24
+ export type DelegateEventTarget = { eventsPath: string; runId: string };
25
+
16
26
  export function recordDelegateEvent(
17
- manifest: { eventsPath: string; runId: string },
18
- type:
19
- | "delegate.requested"
20
- | "delegate.admitted"
21
- | "delegate.rejected"
22
- | "delegate.completed"
23
- | "delegate.timed_out"
24
- | "delegate.rolled_up",
27
+ manifest: DelegateEventTarget,
28
+ type: DelegateEventType,
25
29
  taskId: string,
26
30
  data: Record<string, unknown>,
27
31
  ): void {
@@ -55,6 +55,16 @@ export interface CrewBrokerOptions {
55
55
  nestingTrustedEscalation?: boolean;
56
56
  /** Global worker semaphore size, used to size the nested-slot budget. */
57
57
  globalWorkerSemaphore?: number;
58
+ /** RR-023 F4 (2026-09-29 battery, finding #4): bounded retry schedule
59
+ * (ms) for run.lock contention in the wait.request / delegate.request
60
+ * handlers. A live cross-process holder (e.g. the detached runner
61
+ * persisting task state) makes acquireLockWithRetry throw; instead of
62
+ * letting that escape handleData → closeConnection, the handler retries
63
+ * on this schedule and then answers a typed `busy` error frame with the
64
+ * connection SURVIVING (plus a delegate.rejected event on the delegate
65
+ * path — never silent). Default DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS
66
+ * (50/100/200/400/800). See protocol/lock-busy.ts. */
67
+ lockBusyRetryDelaysMs?: readonly number[];
58
68
  /** Test seam / alternative spawner for delegate grandchildren. Production
59
69
  * uses spawnDelegateGrandchild (direct runChildPi call-site, ADR-5 §2). */
60
70
  grandchildSpawner?: (input: GrandchildSpawnInput) => Promise<GrandchildSpawnResult>;
@@ -0,0 +1,76 @@
1
+ /**
2
+ * lock-busy.ts — RR-023 F4 (2026-09-29 full battery, finding #4).
3
+ *
4
+ * Broker-handler degradation for run.lock contention. Root cause: the
5
+ * wait.request / delegate.request handlers run their state RMW inside
6
+ * withRunLockSync; acquireLockWithRetry (locks.ts, canSteal=false) throws
7
+ * IMMEDIATELY when a live cross-process holder (e.g. the detached runner
8
+ * persisting task state) owns run.lock, and that throw escaped handleData →
9
+ * closeConnection — workers saw an untyped `code=close` socket death
10
+ * (evidence: team_20260929041005 ask-probe / team_20260929041020
11
+ * delegate-probe, both dead within ~5ms; the SYNC foreground run was the
12
+ * zero-contention control).
13
+ *
14
+ * Fix layer is the broker ONLY — locks.ts steal semantics are the v0.9.26
15
+ * lock-family invariant and stay untouched (a live holder is never stolen).
16
+ * Precedent: protocol/request-parsers.ts (M4/WI-4.1 extraction discipline).
17
+ */
18
+
19
+ import { withRunLockSync } from "../../../state/coordination/locks.ts";
20
+ import type { TeamRunManifest } from "../../../state/types.ts";
21
+
22
+ /** Default bounded retry schedule for run.lock contention (ms). Five
23
+ * back-off attempts (~1.55s total) absorb a transient holder — e.g. the
24
+ * detached runner's state flush — without parking the request. */
25
+ export const DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [50, 100, 200, 400, 800];
26
+
27
+ /** Match the exact identity locks.ts throws for a LIVE holder on the
28
+ * never-steal path (sync acquireLockWithRetry, its async twin, and the
29
+ * lockCtx fallthrough): `Run '<basename>' is locked by another operation.`
30
+ * Classification is by message identity — locks.ts exports no error class
31
+ * (plain Error), and src/state must not be edited from the broker layer. */
32
+ const RUN_LOCK_BUSY_MESSAGE = /^Run '[^']+' is locked by another operation\.$/;
33
+
34
+ export function isRunLockBusyError(error: unknown): boolean {
35
+ return error instanceof Error && RUN_LOCK_BUSY_MESSAGE.test(error.message);
36
+ }
37
+
38
+ /** Result of a run-locked RMW under the busy-retry policy: either the body's
39
+ * value, or the typed busy degradation (message derived from the underlying
40
+ * lock error — it names run.lock, so it matches /run\.lock|busy/i). */
41
+ export type LockBusyOutcome<T> = { ok: true; value: T } | { ok: false; message: string };
42
+
43
+ /** Run `fn` under withRunLockSync with a bounded busy-retry.
44
+ *
45
+ * Retry safety: the busy throw is raised by the lock ACQUIRE before `fn`
46
+ * runs (locks.ts), and in the broker's wait/delegate bodies every nested
47
+ * state write is lockCtx-reentrant (same-context nesting never re-acquires),
48
+ * so a retry never re-executes a partially-run body. Any NON-busy error
49
+ * propagates unchanged (previous handler behavior for real faults).
50
+ * Waits use async sleep — the broker event loop keeps serving other
51
+ * connections (and pings) between attempts. */
52
+ export async function withRunLockBusyRetry<T>(
53
+ manifest: TeamRunManifest,
54
+ delaysMs: readonly number[],
55
+ fn: () => T,
56
+ ): Promise<LockBusyOutcome<T>> {
57
+ for (let attempt = 0; ; attempt++) {
58
+ try {
59
+ return { ok: true, value: withRunLockSync(manifest, fn) };
60
+ } catch (error) {
61
+ if (!isRunLockBusyError(error)) throw error;
62
+ const delay = delaysMs[attempt];
63
+ if (delay === undefined) {
64
+ return {
65
+ ok: false,
66
+ message: `run.lock busy (bounded retry budget exhausted): ${(error as Error).message}`,
67
+ };
68
+ }
69
+ await sleep(delay);
70
+ }
71
+ }
72
+ }
73
+
74
+ function sleep(ms: number): Promise<void> {
75
+ return new Promise((resolve) => setTimeout(resolve, ms));
76
+ }
@@ -95,6 +95,21 @@ export function grandchildArtifactsRoot(cwd: string, runId: string, parentTaskId
95
95
  return path.join(cwd, ".crew", "artifacts", runId, parentTaskId, "nested", subId);
96
96
  }
97
97
 
98
+ /** RR-023 F3 (2026-09-29 battery finding #3): derive the relayed grandchild
99
+ * result text from a ChildPiRunResult — pure so it is unit-testable without
100
+ * spawning. Same precedence as the original inline expression at the capture
101
+ * site: rawFinalText → stdout → stderr (failure only).
102
+ * Diagnosability guard: exit 0 with ZERO assistant text (run
103
+ * team_20260929041427: the model-less gc fell through to the pi global default
104
+ * and returned empty completions) used to fence an EMPTY mailbox body —
105
+ * indistinguishable from a relay bug. An explicit marker keeps the relay
106
+ * honest; the real fix (parent-model inheritance) lives in the broker. */
107
+ export function grandchildResultText(result: ChildPiRunResult): string {
108
+ const ok = result.exitCode === 0;
109
+ const text = (result.rawFinalText ?? result.stdout ?? "").trim() || (ok ? "" : result.stderr.trim());
110
+ return ok && text === "" ? "[grandchild exited 0 with no assistant output]" : text;
111
+ }
112
+
98
113
  export async function spawnDelegateGrandchild(input: GrandchildSpawnInput): Promise<GrandchildSpawnResult> {
99
114
  // RR-012 F03: single artifacts formula — grandchildArtifactsRoot(input.cwd, …).
100
115
  // input.cwd is the PARENT TASK cwd threaded by the broker (the admitted
@@ -137,8 +152,7 @@ export async function spawnDelegateGrandchild(input: GrandchildSpawnInput): Prom
137
152
  onSpawn: input.onSpawn,
138
153
  });
139
154
  const ok = result.exitCode === 0;
140
- const text = (result.rawFinalText ?? result.stdout ?? "").trim() || (ok ? "" : result.stderr.trim());
141
- return { ok, resultText: text, ...(usageTokens !== undefined ? { usageTokens } : {}) };
155
+ return { ok, resultText: grandchildResultText(result), ...(usageTokens !== undefined ? { usageTokens } : {}) };
142
156
  } catch (error) {
143
157
  const timedOut = abort.signal.aborted;
144
158
  return {
@@ -17,9 +17,21 @@
17
17
  * parse error downstream. We don't double-report parse errors.
18
18
  */
19
19
 
20
- import { parse } from "acorn";
20
+ import { createRequire } from "node:module";
21
21
  import { getCrewEnv } from "../config/env-vars.ts";
22
22
 
23
+ // RR-021 WI-2.1: acorn parses ONLY when the determinism check actually runs —
24
+ // every dynamic-workflow-runner import was paying acorn's load cost up front.
25
+ // Lazy sync accessor (model: ui/syntax-highlight.ts); acorn stays external in
26
+ // the bundle and resolves through this shim on first use.
27
+ const require = createRequire(import.meta.url);
28
+ type AcornModule = typeof import("acorn");
29
+ let cachedAcorn: AcornModule | undefined;
30
+ function acorn(): AcornModule {
31
+ if (!cachedAcorn) cachedAcorn = require("acorn") as AcornModule;
32
+ return cachedAcorn;
33
+ }
34
+
23
35
  const NONDETERMINISM_ERROR =
24
36
  "Workflow scripts must be deterministic: Date.now()/Math.random()/new Date() are unavailable. These introduce non-reproducible behavior across runs. Use ctx.vars for cached state, or pass a fixed seed via ctx.setArgs(). To bypass this check (escape hatch), set PI_CREW_DWF_SKIP_DETERMINISM_CHECK=1.";
25
37
 
@@ -38,7 +50,7 @@ export class DeterminismError extends Error {
38
50
  export function assertDeterministicScript(script: string): void {
39
51
  let ast: AstNode;
40
52
  try {
41
- ast = parse(script, {
53
+ ast = acorn().parse(script, {
42
54
  ecmaVersion: "latest",
43
55
  sourceType: "module",
44
56
  allowAwaitOutsideFunction: true,