pi-crew 0.11.2 → 0.11.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -1
- package/README.md +1 -1
- package/dist/index.mjs +492 -493
- package/docs/actions-reference.md +22 -0
- package/index.ts +9 -5
- package/package.json +12 -9
- package/scripts/analyze-run.mjs +6 -3
- package/skills/real-test-pi-crew/SKILL.md +20 -12
- package/src/extension/async-notifier.ts +32 -8
- package/src/extension/crew-input-router.ts +9 -2
- package/src/extension/plan-orchestrate.ts +19 -3
- package/src/extension/registration/commands/shared.ts +11 -10
- package/src/extension/registration/foreground-run-controller.ts +2 -2
- package/src/extension/registration/lazy-configurers.ts +4 -1
- package/src/extension/registration/lifecycle-handlers.ts +16 -10
- package/src/extension/registration/lifecycle.ts +2 -0
- package/src/extension/registration/observability.ts +27 -18
- package/src/extension/registration/team-tool.ts +9 -2
- package/src/extension/registration/wire-cross-extension.ts +2 -0
- package/src/extension/team-manager-command.ts +3 -2
- package/src/extension/team-tool/dispatch/index.ts +4 -3
- package/src/extension/team-tool/dispatch/status.ts +7 -3
- package/src/extension/team-tool/goal-wrap.ts +7 -0
- package/src/extension/team-tool/goal.ts +12 -0
- package/src/extension/team-tool/inspect.ts +90 -0
- package/src/extension/team-tool/parallel-dispatch.ts +16 -1
- package/src/extension/team-tool/run-intent.ts +1 -1
- package/src/extension/team-tool/run.ts +1 -1
- package/src/runtime/async-runner.ts +36 -0
- package/src/runtime/broker/crew-broker.ts +61 -62
- package/src/runtime/broker/delegate/delegate-event.ts +12 -8
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/lock-busy.ts +76 -0
- package/src/runtime/delegate-spawn.ts +16 -2
- package/src/runtime/deterministic-ast.ts +14 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
- package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
- package/src/runtime/goal-workflow/goal-loop-runner.ts +32 -0
- package/src/runtime/group-join.ts +11 -3
- package/src/runtime/live-session/live-session-runtime.ts +1 -1
- package/src/runtime/model/runtime-warmup.ts +2 -0
- package/src/runtime/output/output-validator.ts +10 -5
- package/src/runtime/plan-approval.ts +14 -1
- package/src/runtime/recovery/crash-recovery.ts +80 -52
- package/src/runtime/stale-reconciler.ts +8 -0
- package/src/runtime/subagent-manager.ts +43 -13
- package/src/runtime/task-output-context.ts +11 -1
- package/src/runtime/task-runner/child-executor.ts +17 -6
- package/src/runtime/task-runner/post-execution.ts +46 -14
- package/src/runtime/task-runner/pre-execution.ts +29 -21
- package/src/runtime/task-runner/prompt-builder.ts +8 -0
- package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
- package/src/runtime/task-runner.ts +7 -0
- package/src/runtime/team-runner.ts +4 -1
- package/src/runtime/verification/verification-worktree.ts +3 -0
- package/src/schema/team-tool-schema.ts +2 -1
- package/src/state/coordination/mailbox.ts +7 -2
- package/src/state/stores/state-store.ts +8 -1
- package/src/ui/dashboard-panes/agents-pane.ts +11 -8
- package/src/ui/inline-panel/index.ts +1 -1
- package/src/ui/render-diff.ts +19 -6
- package/src/ui/run-dashboard.ts +11 -1
- package/src/ui/run-snapshot-cache.ts +9 -3
- package/src/ui/widget/task-list.ts +8 -5
- package/src/ui/widget/widget-model.ts +3 -1
- package/tsconfig.json +2 -0
- package/src/ui/card-colors.ts +0 -168
- package/src/utils/completion-dedupe.ts +0 -63
- package/src/utils/resolve-shell.ts +0 -36
|
@@ -28,7 +28,7 @@ import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
|
28
28
|
import { appendMailboxMessageAsync, type MailboxMessage, registerMailboxAppendObserver } from "../../state/coordination/mailbox.ts";
|
|
29
29
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
30
30
|
import { loadRunManifestById, saveRunManifest, saveRunTasks } from "../../state/stores/state-store.ts";
|
|
31
|
-
import type { TeamTaskState } from "../../state/types.ts";
|
|
31
|
+
import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
32
32
|
import { runEventBus } from "../../ui/run-event-bus.ts";
|
|
33
33
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
34
34
|
import { BrokerError, encodeBrokerFrame, MAX_BROKER_FRAME_BYTES, NdjsonDecoder } from "../../utils/ndjson.ts";
|
|
@@ -40,11 +40,12 @@ import { resolveCrewMaxDepth } from "../model/pi-args.ts";
|
|
|
40
40
|
import { NestedSlotBudget } from "../scheduling/nested-slots.ts";
|
|
41
41
|
import { evaluateDelegateAdmission } from "../spawn-policy.ts";
|
|
42
42
|
import { type BrokerToken, BrokerTokenRegistry } from "./crew-broker-tokens.ts";
|
|
43
|
-
import { recordDelegateEvent } from "./delegate/delegate-event.ts";
|
|
43
|
+
import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent } from "./delegate/delegate-event.ts";
|
|
44
44
|
import { promoteShadowToRunning } from "./delegate/shadow-lifecycle.ts";
|
|
45
45
|
import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
|
|
46
46
|
import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
|
|
47
47
|
import { handleEventsSince } from "./protocol/events-replay.ts";
|
|
48
|
+
import { DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, type LockBusyOutcome, withRunLockBusyRetry } from "./protocol/lock-busy.ts";
|
|
48
49
|
import { loadRunForHello } from "./protocol/manifest-loader.ts";
|
|
49
50
|
import { handleMsgInbox } from "./protocol/msg-inbox.ts";
|
|
50
51
|
import {
|
|
@@ -104,6 +105,7 @@ export class CrewBroker {
|
|
|
104
105
|
| "grandchildSpawner"
|
|
105
106
|
| "modelCatalog"
|
|
106
107
|
| "serializeOnPathOverlap"
|
|
108
|
+
| "lockBusyRetryDelaysMs"
|
|
107
109
|
>;
|
|
108
110
|
private readonly tokens = new BrokerTokenRegistry();
|
|
109
111
|
/** Task 10 (mux-surface A1 §5.2): taskId → the compound token most
|
|
@@ -1332,28 +1334,31 @@ export class CrewBroker {
|
|
|
1332
1334
|
}
|
|
1333
1335
|
|
|
1334
1336
|
// WP-2/R2: wait.request / wait.resolve (ADR-0 2026-08-17-waiting-producer-ask)
|
|
1337
|
+
// waitAuthError: imported directly from protocol/wait-auth.ts (M4/WI-4.1).
|
|
1335
1338
|
|
|
1336
|
-
// waitAuthError: protocol/wait-auth.ts (M4/WI-4.1).
|
|
1337
|
-
private waitAuthError(conn: ServerConnection): { code: string; message: string } | null {
|
|
1338
|
-
return waitAuthError(conn);
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
|
-
// (The "ADR item 7" doc that used to dangle here documents
|
|
1342
|
-
// recordWaitPolicyRejection — see wait-auth.ts, where it belongs.
|
|
1343
|
-
// Removed 2026-09-10, review F6.)
|
|
1344
1339
|
// T3/R5 (ADR-5): delegate.request — governed-nesting admission + background
|
|
1345
1340
|
// grandchild spawn with durable mailbox delivery (WP-5 step 5).
|
|
1346
|
-
|
|
1341
|
+
|
|
1342
|
+
/** ADR-5 §5: lazy singleton for the nested-slot budget — first use
|
|
1343
|
+
* constructs it from the broker options (4 call sites previously
|
|
1344
|
+
* inlined this IIFE). */
|
|
1345
|
+
private nestedSlotBudget(): NestedSlotBudget {
|
|
1346
|
+
if (!this.nestedSlots)
|
|
1347
|
+
this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
|
|
1348
|
+
return this.nestedSlots;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
/** RR-023 F4: bounded busy-retry around a run-locked RMW (see
|
|
1352
|
+
* protocol/lock-busy.ts) — options override or the default schedule.
|
|
1353
|
+
* A busy-exhausted result is the handler's cue to answer a typed `busy`
|
|
1354
|
+
* frame (connection survives) instead of letting the throw kill it. */
|
|
1355
|
+
private runLockBusyRetry<T>(manifest: TeamRunManifest, fn: () => T): Promise<LockBusyOutcome<T>> {
|
|
1356
|
+
return withRunLockBusyRetry(manifest, this.options.lockBusyRetryDelaysMs ?? DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, fn);
|
|
1357
|
+
}
|
|
1347
1358
|
|
|
1348
1359
|
private recordDelegateEvent(
|
|
1349
|
-
manifest:
|
|
1350
|
-
type:
|
|
1351
|
-
| "delegate.requested"
|
|
1352
|
-
| "delegate.admitted"
|
|
1353
|
-
| "delegate.rejected"
|
|
1354
|
-
| "delegate.completed"
|
|
1355
|
-
| "delegate.timed_out"
|
|
1356
|
-
| "delegate.rolled_up",
|
|
1360
|
+
manifest: DelegateEventTarget,
|
|
1361
|
+
type: DelegateEventType,
|
|
1357
1362
|
taskId: string,
|
|
1358
1363
|
data: Record<string, unknown>,
|
|
1359
1364
|
): void {
|
|
@@ -1434,7 +1439,8 @@ export class CrewBroker {
|
|
|
1434
1439
|
this.recordDelegateEvent(loaded.manifest, "delegate.requested", parentTaskId, { subId, role: requested.role ?? "explorer" });
|
|
1435
1440
|
// Admission: full spawn-policy matrix (ADR-5 §2-§7), parent state from
|
|
1436
1441
|
// the RECORD under the run lock — never the worker's env/self-report.
|
|
1437
|
-
|
|
1442
|
+
// RR-023 F4: run-locked via bounded busy-retry (protocol/lock-busy.ts).
|
|
1443
|
+
const admissionAttempt = await this.runLockBusyRetry(loaded.manifest, () => {
|
|
1438
1444
|
const fresh = loadRunManifestById(loaded.manifest.cwd, runId);
|
|
1439
1445
|
if (!fresh) return { code: "no-manifest" as const, message: `run '${runId}' not found` };
|
|
1440
1446
|
const task = fresh.tasks.find((t) => t.id === parentTaskId);
|
|
@@ -1474,11 +1480,7 @@ export class CrewBroker {
|
|
|
1474
1480
|
...(task.depth !== undefined ? { depth: task.depth } : {}),
|
|
1475
1481
|
...(task.allocation !== undefined ? { allocation: task.allocation } : {}),
|
|
1476
1482
|
},
|
|
1477
|
-
slots: (()
|
|
1478
|
-
if (!this.nestedSlots)
|
|
1479
|
-
this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
|
|
1480
|
-
return this.nestedSlots;
|
|
1481
|
-
})().snapshot(),
|
|
1483
|
+
slots: this.nestedSlotBudget().snapshot(),
|
|
1482
1484
|
requested,
|
|
1483
1485
|
...(effectiveCatalog !== undefined ? { modelCatalog: effectiveCatalog } : {}),
|
|
1484
1486
|
// ADR-5 §12: the delegate surface is an escalation — trusted only by the
|
|
@@ -1498,28 +1500,11 @@ export class CrewBroker {
|
|
|
1498
1500
|
return { code: "policy-denied" as const, message: decision.message ?? decision.reason ?? "delegate denied" };
|
|
1499
1501
|
}
|
|
1500
1502
|
// Slot acquisition INSIDE the lock (no reserve-then-race refund window).
|
|
1501
|
-
if (
|
|
1502
|
-
!(() => {
|
|
1503
|
-
if (!this.nestedSlots)
|
|
1504
|
-
this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
|
|
1505
|
-
return this.nestedSlots;
|
|
1506
|
-
})().tryAcquire(subId)
|
|
1507
|
-
) {
|
|
1503
|
+
if (!this.nestedSlotBudget().tryAcquire(subId)) {
|
|
1508
1504
|
this.recordDelegateEvent(fresh.manifest, "delegate.rejected", parentTaskId, { subId, reason: "slots-exhausted" });
|
|
1509
1505
|
return {
|
|
1510
1506
|
code: "policy-denied" as const,
|
|
1511
|
-
message: `delegate rejected: nested spawn budget exhausted; ${
|
|
1512
|
-
(
|
|
1513
|
-
() => {
|
|
1514
|
-
if (!this.nestedSlots)
|
|
1515
|
-
this.nestedSlots = new NestedSlotBudget(
|
|
1516
|
-
this.options.globalWorkerSemaphore ?? 4,
|
|
1517
|
-
this.options.nestingMaxSlots,
|
|
1518
|
-
);
|
|
1519
|
-
return this.nestedSlots;
|
|
1520
|
-
}
|
|
1521
|
-
)().statusLine
|
|
1522
|
-
}`,
|
|
1507
|
+
message: `delegate rejected: nested spawn budget exhausted; ${this.nestedSlotBudget().statusLine}`,
|
|
1523
1508
|
};
|
|
1524
1509
|
}
|
|
1525
1510
|
// Reserve the requested budget pessimistically (ADR-5 §5): tokensSpent
|
|
@@ -1562,13 +1547,23 @@ export class CrewBroker {
|
|
|
1562
1547
|
startedAt: new Date().toISOString(),
|
|
1563
1548
|
} satisfies TeamTaskState,
|
|
1564
1549
|
]);
|
|
1565
|
-
|
|
1550
|
+
// RR-023 F3: thread the parent's model out of the locked read (gc fallback).
|
|
1551
|
+
return { code: "ok" as const, decision, reserved, executionCwd, parentModel: task.model };
|
|
1566
1552
|
});
|
|
1553
|
+
// RR-023 F4 (finding #4): run.lock held by a live cross-process holder —
|
|
1554
|
+
// degrade to a typed busy frame AND a delegate.rejected event (never
|
|
1555
|
+
// silent); the connection survives (ping still answers).
|
|
1556
|
+
if (!admissionAttempt.ok) {
|
|
1557
|
+
this.recordDelegateEvent(loaded.manifest, "delegate.rejected", parentTaskId, { subId, reason: "run-lock-busy" });
|
|
1558
|
+
this.sendError(conn, id, "busy", admissionAttempt.message);
|
|
1559
|
+
return;
|
|
1560
|
+
}
|
|
1561
|
+
const admissionOutcome = admissionAttempt.value;
|
|
1567
1562
|
if (admissionOutcome.code !== "ok") {
|
|
1568
1563
|
this.sendError(conn, id, admissionOutcome.code, admissionOutcome.message);
|
|
1569
1564
|
return;
|
|
1570
1565
|
}
|
|
1571
|
-
const { decision, reserved, executionCwd } = admissionOutcome;
|
|
1566
|
+
const { decision, reserved, executionCwd, parentModel } = admissionOutcome;
|
|
1572
1567
|
this.recordDelegateEvent(loaded.manifest, "delegate.admitted", parentTaskId, {
|
|
1573
1568
|
subId,
|
|
1574
1569
|
childDepth: decision.childDepth,
|
|
@@ -1584,6 +1579,9 @@ export class CrewBroker {
|
|
|
1584
1579
|
: undefined;
|
|
1585
1580
|
// Return IMMEDIATELY (principle 7): the tool self-polls the mailbox.
|
|
1586
1581
|
this.sendResult(conn, id, { ok: true, grandchildTaskRef: subId, childDepth: decision.childDepth, timeoutSec: decision.timeoutSec });
|
|
1582
|
+
// RR-023 F3 (finding #3, run team_20260929041427): model-less gc spawn fell
|
|
1583
|
+
// through to the pi GLOBAL default — inherit the parent task's model.
|
|
1584
|
+
const gcModel = requested.model ?? parentModel;
|
|
1587
1585
|
// Background grandchild lifecycle (ADR-5 §1/§6).
|
|
1588
1586
|
const spawner = this.options.grandchildSpawner ?? spawnDelegateGrandchild;
|
|
1589
1587
|
void (async () => {
|
|
@@ -1598,7 +1596,7 @@ export class CrewBroker {
|
|
|
1598
1596
|
eventsPath: loaded.manifest.eventsPath,
|
|
1599
1597
|
role: requested.role ?? "explorer",
|
|
1600
1598
|
...(grandchildCreds ? { brokerSpawn: grandchildCreds } : {}),
|
|
1601
|
-
...(
|
|
1599
|
+
...(gcModel !== undefined ? { model: gcModel } : {}),
|
|
1602
1600
|
...(requested.maxTurns !== undefined ? { maxTurns: requested.maxTurns } : {}),
|
|
1603
1601
|
timeoutSec: decision.timeoutSec ?? 900,
|
|
1604
1602
|
depthOverride: decision.childDepth ?? 2,
|
|
@@ -1711,11 +1709,7 @@ export class CrewBroker {
|
|
|
1711
1709
|
}
|
|
1712
1710
|
}
|
|
1713
1711
|
} finally {
|
|
1714
|
-
(()
|
|
1715
|
-
if (!this.nestedSlots)
|
|
1716
|
-
this.nestedSlots = new NestedSlotBudget(this.options.globalWorkerSemaphore ?? 4, this.options.nestingMaxSlots);
|
|
1717
|
-
return this.nestedSlots;
|
|
1718
|
-
})().release(subId);
|
|
1712
|
+
this.nestedSlotBudget().release(subId);
|
|
1719
1713
|
}
|
|
1720
1714
|
this.recordDelegateEvent(loaded.manifest, outcome.timedOut ? "delegate.timed_out" : "delegate.completed", parentTaskId, {
|
|
1721
1715
|
subId,
|
|
@@ -1724,9 +1718,7 @@ export class CrewBroker {
|
|
|
1724
1718
|
})();
|
|
1725
1719
|
}
|
|
1726
1720
|
|
|
1727
|
-
|
|
1728
|
-
recordWaitPolicyRejection(manifest, taskId, method);
|
|
1729
|
-
}
|
|
1721
|
+
// recordWaitPolicyRejection: imported directly from protocol/wait-auth.ts.
|
|
1730
1722
|
|
|
1731
1723
|
/** WP-2/R2 step 4: park the calling task while its `ask` tool awaits a
|
|
1732
1724
|
* leader answer. Park = task.status "waiting" + task.waiting marker +
|
|
@@ -1741,7 +1733,7 @@ export class CrewBroker {
|
|
|
1741
1733
|
this.sendError(conn, id, "auth", "not authed");
|
|
1742
1734
|
return;
|
|
1743
1735
|
}
|
|
1744
|
-
const authErr =
|
|
1736
|
+
const authErr = waitAuthError(conn);
|
|
1745
1737
|
if (authErr) {
|
|
1746
1738
|
this.sendError(conn, id, authErr.code, authErr.message);
|
|
1747
1739
|
return;
|
|
@@ -1778,7 +1770,7 @@ export class CrewBroker {
|
|
|
1778
1770
|
// Capability gate (ADR item 7): fail-closed, NEVER silent — every
|
|
1779
1771
|
// rejection leaves a policy.action trace in the run's events.jsonl.
|
|
1780
1772
|
if (this.options.waitMethodsEnabled !== true) {
|
|
1781
|
-
|
|
1773
|
+
recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.request");
|
|
1782
1774
|
this.sendError(
|
|
1783
1775
|
conn,
|
|
1784
1776
|
id,
|
|
@@ -1796,7 +1788,8 @@ export class CrewBroker {
|
|
|
1796
1788
|
const deadline = Date.now() + clampSec * 1000;
|
|
1797
1789
|
const runId = conn.runId;
|
|
1798
1790
|
const taskId = conn.taskId;
|
|
1799
|
-
|
|
1791
|
+
// RR-023 F4: run-locked via bounded busy-retry (protocol/lock-busy.ts).
|
|
1792
|
+
const outcome = await this.runLockBusyRetry(loaded.manifest, () => {
|
|
1800
1793
|
// Fresh reload INSIDE the lock (respond.ts:42-43 discipline).
|
|
1801
1794
|
const fresh = loadRunManifestById(loaded.manifest.cwd, runId);
|
|
1802
1795
|
if (!fresh) return { code: "no-manifest" as const, message: `run '${runId}' not found` };
|
|
@@ -1829,8 +1822,14 @@ export class CrewBroker {
|
|
|
1829
1822
|
saveRunManifest(updatedManifest);
|
|
1830
1823
|
return { code: "ok" as const, message: "" };
|
|
1831
1824
|
});
|
|
1832
|
-
|
|
1833
|
-
|
|
1825
|
+
// RR-023 F4 (finding #4): run.lock held by a live cross-process holder
|
|
1826
|
+
// (e.g. the detached runner) — typed busy frame, connection survives.
|
|
1827
|
+
if (!outcome.ok) {
|
|
1828
|
+
this.sendError(conn, id, "busy", outcome.message);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
if (outcome.value.code !== "ok") {
|
|
1832
|
+
this.sendError(conn, id, outcome.value.code, outcome.value.message);
|
|
1834
1833
|
return;
|
|
1835
1834
|
}
|
|
1836
1835
|
// Events AFTER the run lock is released (the event-log lock is a
|
|
@@ -1895,7 +1894,7 @@ export class CrewBroker {
|
|
|
1895
1894
|
this.sendError(conn, id, "auth", "not authed");
|
|
1896
1895
|
return;
|
|
1897
1896
|
}
|
|
1898
|
-
const authErr =
|
|
1897
|
+
const authErr = waitAuthError(conn);
|
|
1899
1898
|
if (authErr) {
|
|
1900
1899
|
this.sendError(conn, id, authErr.code, authErr.message);
|
|
1901
1900
|
return;
|
|
@@ -1928,7 +1927,7 @@ export class CrewBroker {
|
|
|
1928
1927
|
return;
|
|
1929
1928
|
}
|
|
1930
1929
|
if (this.options.waitMethodsEnabled !== true) {
|
|
1931
|
-
|
|
1930
|
+
recordWaitPolicyRejection(loaded.manifest, conn.taskId, "wait.resolve");
|
|
1932
1931
|
this.sendError(
|
|
1933
1932
|
conn,
|
|
1934
1933
|
id,
|
|
@@ -13,15 +13,19 @@ import { logInternalError } from "../../../utils/internal-error.ts";
|
|
|
13
13
|
/** Fire-and-forget async append; an append failure is logged, never thrown
|
|
14
14
|
* (broker handlers must not block the event loop on the sync event-log lock).
|
|
15
15
|
* See `crew-broker.delegate.event` event scope conventions in ADR-5 §10. */
|
|
16
|
+
export type DelegateEventType =
|
|
17
|
+
| "delegate.requested"
|
|
18
|
+
| "delegate.admitted"
|
|
19
|
+
| "delegate.rejected"
|
|
20
|
+
| "delegate.completed"
|
|
21
|
+
| "delegate.timed_out"
|
|
22
|
+
| "delegate.rolled_up";
|
|
23
|
+
|
|
24
|
+
export type DelegateEventTarget = { eventsPath: string; runId: string };
|
|
25
|
+
|
|
16
26
|
export function recordDelegateEvent(
|
|
17
|
-
manifest:
|
|
18
|
-
type:
|
|
19
|
-
| "delegate.requested"
|
|
20
|
-
| "delegate.admitted"
|
|
21
|
-
| "delegate.rejected"
|
|
22
|
-
| "delegate.completed"
|
|
23
|
-
| "delegate.timed_out"
|
|
24
|
-
| "delegate.rolled_up",
|
|
27
|
+
manifest: DelegateEventTarget,
|
|
28
|
+
type: DelegateEventType,
|
|
25
29
|
taskId: string,
|
|
26
30
|
data: Record<string, unknown>,
|
|
27
31
|
): void {
|
|
@@ -55,6 +55,16 @@ export interface CrewBrokerOptions {
|
|
|
55
55
|
nestingTrustedEscalation?: boolean;
|
|
56
56
|
/** Global worker semaphore size, used to size the nested-slot budget. */
|
|
57
57
|
globalWorkerSemaphore?: number;
|
|
58
|
+
/** RR-023 F4 (2026-09-29 battery, finding #4): bounded retry schedule
|
|
59
|
+
* (ms) for run.lock contention in the wait.request / delegate.request
|
|
60
|
+
* handlers. A live cross-process holder (e.g. the detached runner
|
|
61
|
+
* persisting task state) makes acquireLockWithRetry throw; instead of
|
|
62
|
+
* letting that escape handleData → closeConnection, the handler retries
|
|
63
|
+
* on this schedule and then answers a typed `busy` error frame with the
|
|
64
|
+
* connection SURVIVING (plus a delegate.rejected event on the delegate
|
|
65
|
+
* path — never silent). Default DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS
|
|
66
|
+
* (50/100/200/400/800). See protocol/lock-busy.ts. */
|
|
67
|
+
lockBusyRetryDelaysMs?: readonly number[];
|
|
58
68
|
/** Test seam / alternative spawner for delegate grandchildren. Production
|
|
59
69
|
* uses spawnDelegateGrandchild (direct runChildPi call-site, ADR-5 §2). */
|
|
60
70
|
grandchildSpawner?: (input: GrandchildSpawnInput) => Promise<GrandchildSpawnResult>;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lock-busy.ts — RR-023 F4 (2026-09-29 full battery, finding #4).
|
|
3
|
+
*
|
|
4
|
+
* Broker-handler degradation for run.lock contention. Root cause: the
|
|
5
|
+
* wait.request / delegate.request handlers run their state RMW inside
|
|
6
|
+
* withRunLockSync; acquireLockWithRetry (locks.ts, canSteal=false) throws
|
|
7
|
+
* IMMEDIATELY when a live cross-process holder (e.g. the detached runner
|
|
8
|
+
* persisting task state) owns run.lock, and that throw escaped handleData →
|
|
9
|
+
* closeConnection — workers saw an untyped `code=close` socket death
|
|
10
|
+
* (evidence: team_20260929041005 ask-probe / team_20260929041020
|
|
11
|
+
* delegate-probe, both dead within ~5ms; the SYNC foreground run was the
|
|
12
|
+
* zero-contention control).
|
|
13
|
+
*
|
|
14
|
+
* Fix layer is the broker ONLY — locks.ts steal semantics are the v0.9.26
|
|
15
|
+
* lock-family invariant and stay untouched (a live holder is never stolen).
|
|
16
|
+
* Precedent: protocol/request-parsers.ts (M4/WI-4.1 extraction discipline).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { withRunLockSync } from "../../../state/coordination/locks.ts";
|
|
20
|
+
import type { TeamRunManifest } from "../../../state/types.ts";
|
|
21
|
+
|
|
22
|
+
/** Default bounded retry schedule for run.lock contention (ms). Five
|
|
23
|
+
* back-off attempts (~1.55s total) absorb a transient holder — e.g. the
|
|
24
|
+
* detached runner's state flush — without parking the request. */
|
|
25
|
+
export const DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [50, 100, 200, 400, 800];
|
|
26
|
+
|
|
27
|
+
/** Match the exact identity locks.ts throws for a LIVE holder on the
|
|
28
|
+
* never-steal path (sync acquireLockWithRetry, its async twin, and the
|
|
29
|
+
* lockCtx fallthrough): `Run '<basename>' is locked by another operation.`
|
|
30
|
+
* Classification is by message identity — locks.ts exports no error class
|
|
31
|
+
* (plain Error), and src/state must not be edited from the broker layer. */
|
|
32
|
+
const RUN_LOCK_BUSY_MESSAGE = /^Run '[^']+' is locked by another operation\.$/;
|
|
33
|
+
|
|
34
|
+
export function isRunLockBusyError(error: unknown): boolean {
|
|
35
|
+
return error instanceof Error && RUN_LOCK_BUSY_MESSAGE.test(error.message);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Result of a run-locked RMW under the busy-retry policy: either the body's
|
|
39
|
+
* value, or the typed busy degradation (message derived from the underlying
|
|
40
|
+
* lock error — it names run.lock, so it matches /run\.lock|busy/i). */
|
|
41
|
+
export type LockBusyOutcome<T> = { ok: true; value: T } | { ok: false; message: string };
|
|
42
|
+
|
|
43
|
+
/** Run `fn` under withRunLockSync with a bounded busy-retry.
|
|
44
|
+
*
|
|
45
|
+
* Retry safety: the busy throw is raised by the lock ACQUIRE before `fn`
|
|
46
|
+
* runs (locks.ts), and in the broker's wait/delegate bodies every nested
|
|
47
|
+
* state write is lockCtx-reentrant (same-context nesting never re-acquires),
|
|
48
|
+
* so a retry never re-executes a partially-run body. Any NON-busy error
|
|
49
|
+
* propagates unchanged (previous handler behavior for real faults).
|
|
50
|
+
* Waits use async sleep — the broker event loop keeps serving other
|
|
51
|
+
* connections (and pings) between attempts. */
|
|
52
|
+
export async function withRunLockBusyRetry<T>(
|
|
53
|
+
manifest: TeamRunManifest,
|
|
54
|
+
delaysMs: readonly number[],
|
|
55
|
+
fn: () => T,
|
|
56
|
+
): Promise<LockBusyOutcome<T>> {
|
|
57
|
+
for (let attempt = 0; ; attempt++) {
|
|
58
|
+
try {
|
|
59
|
+
return { ok: true, value: withRunLockSync(manifest, fn) };
|
|
60
|
+
} catch (error) {
|
|
61
|
+
if (!isRunLockBusyError(error)) throw error;
|
|
62
|
+
const delay = delaysMs[attempt];
|
|
63
|
+
if (delay === undefined) {
|
|
64
|
+
return {
|
|
65
|
+
ok: false,
|
|
66
|
+
message: `run.lock busy (bounded retry budget exhausted): ${(error as Error).message}`,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
await sleep(delay);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function sleep(ms: number): Promise<void> {
|
|
75
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
76
|
+
}
|
|
@@ -95,6 +95,21 @@ export function grandchildArtifactsRoot(cwd: string, runId: string, parentTaskId
|
|
|
95
95
|
return path.join(cwd, ".crew", "artifacts", runId, parentTaskId, "nested", subId);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
/** RR-023 F3 (2026-09-29 battery finding #3): derive the relayed grandchild
|
|
99
|
+
* result text from a ChildPiRunResult — pure so it is unit-testable without
|
|
100
|
+
* spawning. Same precedence as the original inline expression at the capture
|
|
101
|
+
* site: rawFinalText → stdout → stderr (failure only).
|
|
102
|
+
* Diagnosability guard: exit 0 with ZERO assistant text (run
|
|
103
|
+
* team_20260929041427: the model-less gc fell through to the pi global default
|
|
104
|
+
* and returned empty completions) used to fence an EMPTY mailbox body —
|
|
105
|
+
* indistinguishable from a relay bug. An explicit marker keeps the relay
|
|
106
|
+
* honest; the real fix (parent-model inheritance) lives in the broker. */
|
|
107
|
+
export function grandchildResultText(result: ChildPiRunResult): string {
|
|
108
|
+
const ok = result.exitCode === 0;
|
|
109
|
+
const text = (result.rawFinalText ?? result.stdout ?? "").trim() || (ok ? "" : result.stderr.trim());
|
|
110
|
+
return ok && text === "" ? "[grandchild exited 0 with no assistant output]" : text;
|
|
111
|
+
}
|
|
112
|
+
|
|
98
113
|
export async function spawnDelegateGrandchild(input: GrandchildSpawnInput): Promise<GrandchildSpawnResult> {
|
|
99
114
|
// RR-012 F03: single artifacts formula — grandchildArtifactsRoot(input.cwd, …).
|
|
100
115
|
// input.cwd is the PARENT TASK cwd threaded by the broker (the admitted
|
|
@@ -137,8 +152,7 @@ export async function spawnDelegateGrandchild(input: GrandchildSpawnInput): Prom
|
|
|
137
152
|
onSpawn: input.onSpawn,
|
|
138
153
|
});
|
|
139
154
|
const ok = result.exitCode === 0;
|
|
140
|
-
|
|
141
|
-
return { ok, resultText: text, ...(usageTokens !== undefined ? { usageTokens } : {}) };
|
|
155
|
+
return { ok, resultText: grandchildResultText(result), ...(usageTokens !== undefined ? { usageTokens } : {}) };
|
|
142
156
|
} catch (error) {
|
|
143
157
|
const timedOut = abort.signal.aborted;
|
|
144
158
|
return {
|
|
@@ -17,9 +17,21 @@
|
|
|
17
17
|
* parse error downstream. We don't double-report parse errors.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import {
|
|
20
|
+
import { createRequire } from "node:module";
|
|
21
21
|
import { getCrewEnv } from "../config/env-vars.ts";
|
|
22
22
|
|
|
23
|
+
// RR-021 WI-2.1: acorn parses ONLY when the determinism check actually runs —
|
|
24
|
+
// every dynamic-workflow-runner import was paying acorn's load cost up front.
|
|
25
|
+
// Lazy sync accessor (model: ui/syntax-highlight.ts); acorn stays external in
|
|
26
|
+
// the bundle and resolves through this shim on first use.
|
|
27
|
+
const require = createRequire(import.meta.url);
|
|
28
|
+
type AcornModule = typeof import("acorn");
|
|
29
|
+
let cachedAcorn: AcornModule | undefined;
|
|
30
|
+
function acorn(): AcornModule {
|
|
31
|
+
if (!cachedAcorn) cachedAcorn = require("acorn") as AcornModule;
|
|
32
|
+
return cachedAcorn;
|
|
33
|
+
}
|
|
34
|
+
|
|
23
35
|
const NONDETERMINISM_ERROR =
|
|
24
36
|
"Workflow scripts must be deterministic: Date.now()/Math.random()/new Date() are unavailable. These introduce non-reproducible behavior across runs. Use ctx.vars for cached state, or pass a fixed seed via ctx.setArgs(). To bypass this check (escape hatch), set PI_CREW_DWF_SKIP_DETERMINISM_CHECK=1.";
|
|
25
37
|
|
|
@@ -38,7 +50,7 @@ export class DeterminismError extends Error {
|
|
|
38
50
|
export function assertDeterministicScript(script: string): void {
|
|
39
51
|
let ast: AstNode;
|
|
40
52
|
try {
|
|
41
|
-
ast = parse(script, {
|
|
53
|
+
ast = acorn().parse(script, {
|
|
42
54
|
ecmaVersion: "latest",
|
|
43
55
|
sourceType: "module",
|
|
44
56
|
allowAwaitOutsideFunction: true,
|
|
@@ -906,67 +906,87 @@ function describeSchemaShape(schema: unknown, depth: number): string {
|
|
|
906
906
|
* reject is checked first because a review can mention both "correctly" (describing existing code)
|
|
907
907
|
* AND "critical bug" (the verdict) — the verdict signal must win.
|
|
908
908
|
*/
|
|
909
|
+
// Precompiled review-outcome signals (module scope — compiled once, not per call).
|
|
910
|
+
// RR-021 WI-1.1: these were previously string literals like "\baccept\b" compiled
|
|
911
|
+
// per call — and "\b" in a TS string literal is a BACKSPACE character (U+0008),
|
|
912
|
+
// so those two signals never matched anything. Regex literals here give real word
|
|
913
|
+
// boundaries. No /g flags → no shared lastIndex state across .test() calls.
|
|
914
|
+
const REJECT_SIGNAL_REGEXES: readonly RegExp[] = [
|
|
915
|
+
/\breject\b/,
|
|
916
|
+
/fundamentally/,
|
|
917
|
+
/completely broken/,
|
|
918
|
+
/totally broken/,
|
|
919
|
+
/critical bug/,
|
|
920
|
+
/critical issue/,
|
|
921
|
+
/critical flaw/,
|
|
922
|
+
/security vulnerability/,
|
|
923
|
+
/does not work/,
|
|
924
|
+
/doesn't work/,
|
|
925
|
+
/will not work/,
|
|
926
|
+
/fails to/,
|
|
927
|
+
/unacceptable/,
|
|
928
|
+
/must not be merged/,
|
|
929
|
+
/do not merge/,
|
|
930
|
+
/wrong approach/,
|
|
931
|
+
/logically incorrect/,
|
|
932
|
+
/incorrectly implements/,
|
|
933
|
+
/returns the opposite/,
|
|
934
|
+
/subtraction instead of addition/,
|
|
935
|
+
/opposite of its intended/,
|
|
936
|
+
];
|
|
937
|
+
const ACCEPT_SIGNAL_REGEXES: readonly RegExp[] = [
|
|
938
|
+
/\baccept\b/,
|
|
939
|
+
/looks good/,
|
|
940
|
+
/well done/,
|
|
941
|
+
/no issues/,
|
|
942
|
+
/no real issues/,
|
|
943
|
+
/no problems/,
|
|
944
|
+
/no concerns/,
|
|
945
|
+
/nothing to change/,
|
|
946
|
+
/ready to merge/,
|
|
947
|
+
/lgtm/,
|
|
948
|
+
/ship it/,
|
|
949
|
+
/correctly implements/,
|
|
950
|
+
/correctly returns/,
|
|
951
|
+
/works as expected/,
|
|
952
|
+
/works correctly/,
|
|
953
|
+
/no bugs/,
|
|
954
|
+
/no defects/,
|
|
955
|
+
/meets all requirements/,
|
|
956
|
+
/all requirements met/,
|
|
957
|
+
/passes all/,
|
|
958
|
+
/is correct/,
|
|
959
|
+
/are correct/,
|
|
960
|
+
/no changes needed/,
|
|
961
|
+
/no changes required/,
|
|
962
|
+
/no further changes/,
|
|
963
|
+
/nothing more to/,
|
|
964
|
+
/complete and correct/,
|
|
965
|
+
/sound implementation/,
|
|
966
|
+
];
|
|
967
|
+
// RR-021 WI-1.1 negation guard: phrases that NEGATE the bare "accept" signal.
|
|
968
|
+
// "I cannot accept this" is a complaint, not approval — the accept hit must be
|
|
969
|
+
// discarded so the outcome falls through to changes_requested (or reject if a
|
|
970
|
+
// reject signal is also present).
|
|
971
|
+
const NEGATED_ACCEPT_REGEXES: readonly RegExp[] = [
|
|
972
|
+
/\bcannot\s+accept\b/,
|
|
973
|
+
/\bcan'?t\s+accept\b/,
|
|
974
|
+
/\bdon'?t\s+accept\b/,
|
|
975
|
+
/\bwon'?t\s+accept\b/,
|
|
976
|
+
/\bdo\s+not\s+accept\b/,
|
|
977
|
+
/\bwill\s+not\s+accept\b/,
|
|
978
|
+
/\bunable\s+to\s+accept\b/,
|
|
979
|
+
/\bnot\s+accept(?:ed|ing|able)?\b/,
|
|
980
|
+
];
|
|
981
|
+
|
|
909
982
|
export function classifyReviewOutcome(prose: string): "accept" | "reject" | "changes_requested" {
|
|
910
983
|
const text = prose.toLowerCase();
|
|
911
|
-
//
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
"fundamentally",
|
|
915
|
-
"completely broken",
|
|
916
|
-
"totally broken",
|
|
917
|
-
"critical bug",
|
|
918
|
-
"critical issue",
|
|
919
|
-
"critical flaw",
|
|
920
|
-
"security vulnerability",
|
|
921
|
-
"does not work",
|
|
922
|
-
"doesn't work",
|
|
923
|
-
"will not work",
|
|
924
|
-
"fails to",
|
|
925
|
-
"unacceptable",
|
|
926
|
-
"must not be merged",
|
|
927
|
-
"do not merge",
|
|
928
|
-
"wrong approach",
|
|
929
|
-
"logically incorrect",
|
|
930
|
-
"incorrectly implements",
|
|
931
|
-
"returns the opposite",
|
|
932
|
-
"subtraction instead of addition",
|
|
933
|
-
"opposite of its intended",
|
|
934
|
-
];
|
|
935
|
-
// Acceptance signals → accept. These indicate explicit approval with no real issues.
|
|
936
|
-
const acceptSignals = [
|
|
937
|
-
"\baccept\b",
|
|
938
|
-
"looks good",
|
|
939
|
-
"well done",
|
|
940
|
-
"no issues",
|
|
941
|
-
"no real issues",
|
|
942
|
-
"no problems",
|
|
943
|
-
"no concerns",
|
|
944
|
-
"nothing to change",
|
|
945
|
-
"ready to merge",
|
|
946
|
-
"lgtm",
|
|
947
|
-
"ship it",
|
|
948
|
-
"correctly implements",
|
|
949
|
-
"correctly returns",
|
|
950
|
-
"works as expected",
|
|
951
|
-
"works correctly",
|
|
952
|
-
"no bugs",
|
|
953
|
-
"no defects",
|
|
954
|
-
"meets all requirements",
|
|
955
|
-
"all requirements met",
|
|
956
|
-
"passes all",
|
|
957
|
-
"is correct",
|
|
958
|
-
"are correct",
|
|
959
|
-
"no changes needed",
|
|
960
|
-
"no changes required",
|
|
961
|
-
"no further changes",
|
|
962
|
-
"nothing more to",
|
|
963
|
-
"complete and correct",
|
|
964
|
-
"sound implementation",
|
|
965
|
-
];
|
|
966
|
-
const hasReject = rejectSignals.some((sig) => new RegExp(sig).test(text));
|
|
967
|
-
const hasAccept = acceptSignals.some((sig) => new RegExp(sig).test(text));
|
|
984
|
+
// Reject-first precedence: a review can mention both "correctly" (describing
|
|
985
|
+
// existing code) AND "critical bug" (the verdict) — the verdict signal must win.
|
|
986
|
+
const hasReject = REJECT_SIGNAL_REGEXES.some((re) => re.test(text));
|
|
968
987
|
if (hasReject) return "reject";
|
|
969
|
-
|
|
988
|
+
const hasAccept = ACCEPT_SIGNAL_REGEXES.some((re) => re.test(text));
|
|
989
|
+
if (hasAccept && !NEGATED_ACCEPT_REGEXES.some((re) => re.test(text))) return "accept";
|
|
970
990
|
return "changes_requested";
|
|
971
991
|
}
|
|
972
992
|
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { readFileSync } from "node:fs";
|
|
21
|
+
import { createRequire } from "node:module";
|
|
21
22
|
import { join } from "node:path";
|
|
22
|
-
import { transformSync } from "esbuild";
|
|
23
23
|
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
24
24
|
import { appendEvent } from "../../state/event-log/event-log.ts";
|
|
25
25
|
import { writeArtifact } from "../../state/stores/artifact-store.ts";
|
|
@@ -33,6 +33,18 @@ import { assertDeterministicScript, isDeterminismCheckEnabled } from "../determi
|
|
|
33
33
|
import { DwfStore } from "../dwf-state-store.ts";
|
|
34
34
|
import { getWorkflowFinalResult, getWorkflowPhaseState, makeWorkflowCtx } from "./dynamic-workflow-context.ts";
|
|
35
35
|
|
|
36
|
+
// RR-021 WI-2.1: esbuild is a heavy native dep (~35ms+ cold load) but is only
|
|
37
|
+
// needed when a .dwf.ts script actually executes. Lazy-load via a synchronous
|
|
38
|
+
// createRequire shim on first use (model: ui/syntax-highlight.ts) — call sites
|
|
39
|
+
// stay sync. esbuild remains external in the bundle and resolves through here.
|
|
40
|
+
const nodeRequire = createRequire(import.meta.url);
|
|
41
|
+
type EsbuildModule = typeof import("esbuild");
|
|
42
|
+
let cachedEsbuild: EsbuildModule | undefined;
|
|
43
|
+
function esbuild(): EsbuildModule {
|
|
44
|
+
if (!cachedEsbuild) cachedEsbuild = nodeRequire("esbuild") as EsbuildModule;
|
|
45
|
+
return cachedEsbuild;
|
|
46
|
+
}
|
|
47
|
+
|
|
36
48
|
export interface RunDynamicWorkflowInput {
|
|
37
49
|
manifest: TeamRunManifest;
|
|
38
50
|
workflow: DynamicWorkflowConfig;
|
|
@@ -119,7 +131,7 @@ async function loadWorkflowModule(scriptPath: string): Promise<DynamicWorkflowSc
|
|
|
119
131
|
// DISC-1: acorn can only parse JavaScript, not TypeScript. Real .dwf.ts files
|
|
120
132
|
// contain type annotations/imports that cause a silent parse error, so the
|
|
121
133
|
// determinism check never runs. Transpile to JS via esbuild first.
|
|
122
|
-
const js = transformSync(scriptSource, { loader: "ts", format: "esm" }).code;
|
|
134
|
+
const js = esbuild().transformSync(scriptSource, { loader: "ts", format: "esm" }).code;
|
|
123
135
|
assertDeterministicScript(js);
|
|
124
136
|
}
|
|
125
137
|
// jiti is the same loader async-runner.ts uses (resolveTypeScriptLoader). We require it
|