pi-legion 3.0.3 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +263 -2
- package/CONTRIBUTING.md +4 -17
- package/README.md +29 -22
- package/agents/worker.md +1 -1
- package/docs/contracts/display-liveness-v1.md +373 -0
- package/docs/contracts/drop-non-pi-kinds-v4.md +198 -0
- package/docs/contracts/esc-abort-v1.md +4 -4
- package/docs/contracts/esc-no-retire-v3.md +2 -2
- package/docs/contracts/launch-admission-queue-v1.md +839 -0
- package/docs/contracts/status-panel-v1.md +174 -0
- package/docs/inprocess-supervisor.md +37 -4
- package/docs/legion-v2-contract.md +18 -2
- package/docs/performance-2026-10-19.md +199 -0
- package/docs/spawn-mode-visibility-v1.md +206 -0
- package/docs/vexillatio-contract.md +3 -1
- package/index.ts +633 -84
- package/package.json +4 -2
- package/skills/pi-legion/SKILL.md +2 -3
- package/src/agents/agents.ts +17 -10
- package/src/agents/overrides.ts +0 -1
- package/src/agents/presets.ts +14 -40
- package/src/agents/settings.ts +26 -77
- package/src/agents/step-model.ts +7 -22
- package/src/api.ts +1 -35
- package/src/extension/event-bus.ts +80 -8
- package/src/extension/legion-commands.ts +43 -17
- package/src/extension/legion-doctor.ts +1 -1
- package/src/extension/legion-stop.ts +39 -13
- package/src/extension/legion-tools.ts +21 -4
- package/src/extension/liveness.ts +200 -0
- package/src/extension/notify.ts +6 -6
- package/src/extension/playbook.ts +6 -6
- package/src/extension/recycle.ts +6 -2
- package/src/extension/runtime.ts +422 -18
- package/src/extension/slash.ts +0 -21
- package/src/extension/spawn-render.ts +183 -0
- package/src/extension/summary.ts +1 -1
- package/src/legion/db.ts +165 -1
- package/src/legion/events.ts +81 -16
- package/src/legion/nodes.ts +13 -2
- package/src/legion/rebuild.ts +7 -1
- package/src/legion/usage.ts +58 -4
- package/src/runs/orchestrator.ts +1165 -62
- package/src/runs/worktree.ts +190 -21
- package/src/shared/ignored-kind.ts +57 -0
- package/src/shared/progress.ts +11 -0
- package/src/shared/session.ts +199 -19
- package/src/shared/types.ts +50 -66
- package/src/supervisor/admission-queue.ts +516 -0
- package/src/supervisor/inprocess-supervisor.ts +108 -33
- package/src/supervisor/proc-memory.ts +67 -0
- package/src/supervisor/routing-supervisor.ts +58 -1
- package/src/tui/live-overlay.ts +14 -0
- package/src/tui/session-messages-file.ts +100 -0
- package/src/tui/status.ts +103 -21
- package/src/tui/tree-view.ts +36 -3
- package/src/tui/viz/layout.ts +54 -3
- package/src/tui/viz/model.ts +67 -14
- package/src/tui/viz/palette.ts +11 -0
- package/src/tui/viz/panels.ts +222 -8
- package/src/tui/viz/railway.ts +28 -25
- package/src/tui/viz/view.ts +110 -23
- package/src/backends/legacy/client.ts +0 -638
- package/src/backends/legacy/cursor-chat.ts +0 -348
- package/src/backends/legacy/runner.ts +0 -101
- package/src/backends/legacy/types.ts +0 -138
- package/src/runs/kind.ts +0 -260
package/src/runs/orchestrator.ts
CHANGED
|
@@ -17,7 +17,7 @@ import type { DatabaseSync } from "../legion/db.ts";
|
|
|
17
17
|
import type { RunStore } from "./store.ts";
|
|
18
18
|
import type { LegionEventBus, LegionProcessEventType } from "../extension/event-bus.ts";
|
|
19
19
|
import { appendEvent } from "../legion/events.ts";
|
|
20
|
-
import { withLaunchBudget, type LegionBudgetLimits } from "../legion/budget.ts";
|
|
20
|
+
import { checkLaunchBudget, withLaunchBudget, type LegionBudgetLimits } from "../legion/budget.ts";
|
|
21
21
|
import { getChildByName, getNode, insertNode, nodeIdForChild, sanitizeNodeSlug, type LegionNode, updateNode } from "../legion/nodes.ts";
|
|
22
22
|
import { upsertUsage } from "../legion/usage.ts";
|
|
23
23
|
import { evaluatePhaseTransition, isLegionPhase, isTerminalPhase } from "../legion/state-machine.ts";
|
|
@@ -30,7 +30,6 @@ import {
|
|
|
30
30
|
} from "../extension/budget.ts";
|
|
31
31
|
import { canUseCachedCollect } from "../extension/recycle.ts";
|
|
32
32
|
import {
|
|
33
|
-
countAssistantMessages,
|
|
34
33
|
attributeParentSessionShutdown,
|
|
35
34
|
deriveOutcome,
|
|
36
35
|
emptyParsedSession,
|
|
@@ -59,10 +58,24 @@ import {
|
|
|
59
58
|
type Usage,
|
|
60
59
|
} from "../shared/types.ts";
|
|
61
60
|
import { RpcSupervisor } from "../supervisor/rpc-supervisor.ts";
|
|
61
|
+
import {
|
|
62
|
+
type AdmissionDimension,
|
|
63
|
+
type AdmissionQueue,
|
|
64
|
+
type AdmissionQueueEntry,
|
|
65
|
+
type AdmissionVerdict,
|
|
66
|
+
acquireAdmissionQueue,
|
|
67
|
+
evaluateInProcessAdmission,
|
|
68
|
+
type InProcessAdmissionCounts,
|
|
69
|
+
peekAdmissionQueue,
|
|
70
|
+
type QueuedLaunch,
|
|
71
|
+
queueTimeoutMessage,
|
|
72
|
+
} from "../supervisor/admission-queue.ts";
|
|
73
|
+
import { type ChildLaunchContext, currentChildContext, runInChildContext } from "../supervisor/child-context.ts";
|
|
74
|
+
import { inProcessDepth, runAtAbsoluteDepth } from "../supervisor/inprocess-depth.ts";
|
|
75
|
+
import { subscribeInProcessSlotReleases } from "../supervisor/routing-supervisor.ts";
|
|
62
76
|
import type { LegionSupervisor, SettleResult, SpawnInput } from "../supervisor/types.ts";
|
|
63
77
|
import { isInProcessSpawn, resolveSpawnMode } from "../supervisor/spawn-mode.ts";
|
|
64
78
|
import type { PendingUIRequest, UIProxy } from "../supervisor/ui-proxy.ts";
|
|
65
|
-
import { nativeModelFor } from "./kind.ts";
|
|
66
79
|
import { applyVerification, type VerifyRunner } from "./acceptance.ts";
|
|
67
80
|
import {
|
|
68
81
|
createChildWorktree,
|
|
@@ -119,6 +132,129 @@ export interface OrchestratorDeps {
|
|
|
119
132
|
*/
|
|
120
133
|
rpcApprove?: boolean;
|
|
121
134
|
onBudgetRefused?: (event: BudgetRefusedEvent) => void;
|
|
135
|
+
/**
|
|
136
|
+
* D6/D10 (launch admission queue): queue bounds. Absent fields fall back to
|
|
137
|
+
* the defaults declared in `admission-queue.ts` (32 pending / 900_000 ms).
|
|
138
|
+
*/
|
|
139
|
+
launchQueue?: { maxPending?: number; timeoutMs?: number };
|
|
140
|
+
/** D9: one user-visible line per queued / admitted / expired / stopped launch. */
|
|
141
|
+
onLaunchQueueNotice?: (line: string) => void;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* One queued launch (contract §4.3, issue #266).
|
|
146
|
+
*
|
|
147
|
+
* `run()` re-enters `launchInternal` with `admissionAlreadyGranted: true`, so the
|
|
148
|
+
* name allocated at enqueue time is reused (D4) and the authoritative gates are
|
|
149
|
+
* re-evaluated at admission time (I-O1).
|
|
150
|
+
*/
|
|
151
|
+
export interface LaunchQueuePayload {
|
|
152
|
+
readonly name: string;
|
|
153
|
+
readonly parentKey: string;
|
|
154
|
+
readonly inProcess: boolean;
|
|
155
|
+
/** ALS depth captured at enqueue time; admission restores it (A18). */
|
|
156
|
+
readonly parentDepth: number;
|
|
157
|
+
/** ALS identity captured at enqueue time; admission restores it (A18). */
|
|
158
|
+
readonly parentContext?: ChildLaunchContext;
|
|
159
|
+
/** Run the real launch. */
|
|
160
|
+
run(): Promise<Handle>;
|
|
161
|
+
/** Settles when the item leaves the queue. */
|
|
162
|
+
readonly settled: Promise<AdmitOutcome>;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** How a queued item left the queue (contract §4.3). */
|
|
166
|
+
export type AdmitOutcome =
|
|
167
|
+
| { readonly kind: "admitted"; readonly waitedMs: number }
|
|
168
|
+
| { readonly kind: "rejected"; readonly reason: "timeout" | "stopped" | "flushed"; readonly message: string };
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* I-O3: what a `queued` record from a *previous* process means. The queue lives
|
|
172
|
+
* in memory, so such a record can never become live again — say so instead of
|
|
173
|
+
* reporting `unknown child`, and let `retire` clean it up.
|
|
174
|
+
*/
|
|
175
|
+
function launchQueueLostMessage(name: string): string {
|
|
176
|
+
return `child '${name}' never started: the launch queue was lost when the previous process exited; re-dispatch it (or relaunch) instead`;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Launch parameters (extracted so the queued replay can reuse them verbatim). */
|
|
180
|
+
export interface LaunchInput {
|
|
181
|
+
agent: AgentConfig;
|
|
182
|
+
task: string;
|
|
183
|
+
name?: string;
|
|
184
|
+
team?: string;
|
|
185
|
+
model?: string;
|
|
186
|
+
/** Provider id pinning `--provider` for an ambiguous bare model id (#123). */
|
|
187
|
+
modelProvider?: string;
|
|
188
|
+
thinking?: string | false;
|
|
189
|
+
worktree?: boolean;
|
|
190
|
+
modelOrigin?: ModelOrigin;
|
|
191
|
+
/** Vexillatio launch: bypass nested-depth gates while recording its parent-side Legion node. */
|
|
192
|
+
utilityDispatch?: boolean;
|
|
193
|
+
/** Mask inherited Legion identity from the spawned child process. */
|
|
194
|
+
stripLegionIdentity?: boolean;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* I-O3: a *live* queued item was asked to do something that needs a spawned
|
|
199
|
+
* child. D4 freezes the payload at enqueue time, so the only way to change a
|
|
200
|
+
* queued launch is to stop it and launch again (§10).
|
|
201
|
+
*/
|
|
202
|
+
function queuedNotStartedMessage(name: string): string {
|
|
203
|
+
return `child '${name}' has not started yet (state=queued); stop it and relaunch to change it`;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Everything `probeAdmission` needs, captured once per launch attempt. */
|
|
207
|
+
interface AdmissionProbeContext {
|
|
208
|
+
readonly inProcess: boolean;
|
|
209
|
+
readonly agent: AgentConfig;
|
|
210
|
+
readonly nodeId?: string;
|
|
211
|
+
readonly ledgerPrevious: LegionNode | null;
|
|
212
|
+
/** Utility nodes are hard leaves; their ledger depth ceiling differs. */
|
|
213
|
+
readonly utility: boolean;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Bookkeeping for one queued item (paired 1:1 with its `queued` ChildRecord).
|
|
218
|
+
*
|
|
219
|
+
* `settled` is the channel D8 §"other errors bubble to the caller" needs: a
|
|
220
|
+
* timeout/stop/flush resolves it with a `rejected` outcome, while a genuine
|
|
221
|
+
* spawn failure rejects it with the original error.
|
|
222
|
+
*/
|
|
223
|
+
interface LaunchGuard {
|
|
224
|
+
/** Resolve the item's `settled` promise exactly once. */
|
|
225
|
+
readonly settle: (outcome: AdmitOutcome) => void;
|
|
226
|
+
/** Leaves the queue (admitted / timeout / stopped / flushed) — what `collect` awaits. */
|
|
227
|
+
readonly settled: Promise<AdmitOutcome>;
|
|
228
|
+
/** Re-run the advisory gate for this item (D2-c attribution, D5 head-of-line skip). */
|
|
229
|
+
readonly probe: () => AdmissionVerdict;
|
|
230
|
+
readonly dimension: AdmissionDimension;
|
|
231
|
+
/** When the item first entered the queue — the requeue path keeps this deadline. */
|
|
232
|
+
readonly enqueuedAt: number;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** The narrow, read-only supervisor surface the probe consumes (I-O1). */
|
|
236
|
+
interface AdmissionProbeSource {
|
|
237
|
+
admissionCounts(name: string): InProcessAdmissionCounts;
|
|
238
|
+
preflightAdmission(name: string): AdmissionVerdict;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function asAdmissionProbe(source: unknown): AdmissionProbeSource | undefined {
|
|
242
|
+
if (!source || typeof source !== "object") return undefined;
|
|
243
|
+
const candidate = source as Partial<AdmissionProbeSource>;
|
|
244
|
+
return typeof candidate.admissionCounts === "function" && typeof candidate.preflightAdmission === "function"
|
|
245
|
+
? (source as AdmissionProbeSource)
|
|
246
|
+
: undefined;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* §4.3: the in-process probe lives on the `InProcessSupervisor` instance. A
|
|
251
|
+
* `RoutingSupervisor` keeps it as `inprocess`; a bare in-process supervisor (or
|
|
252
|
+
* a test double) may expose the members itself. Anything else (RPC, fakes)
|
|
253
|
+
* admits — the probe is advisory only (I-O1).
|
|
254
|
+
*/
|
|
255
|
+
function inProcessProbeOf(supervisor: LegionSupervisor): AdmissionProbeSource | undefined {
|
|
256
|
+
const routing = supervisor as LegionSupervisor & { inprocess?: unknown };
|
|
257
|
+
return asAdmissionProbe(routing.inprocess) ?? asAdmissionProbe(supervisor);
|
|
122
258
|
}
|
|
123
259
|
|
|
124
260
|
export interface CollectResult {
|
|
@@ -149,6 +285,10 @@ const MAX_DEPTH_ENV = "PI_SUBAGENT_MAX_DEPTH";
|
|
|
149
285
|
const UTILITY_CHILD_ENV = "PI_SUBAGENT_UTILITY_CHILD";
|
|
150
286
|
const CHILD_ENV = "PI_SUBAGENT_CHILD";
|
|
151
287
|
const POLL_INTERVAL_MS = 500;
|
|
288
|
+
/** Ceiling for the geometric backoff in `waitOutTransientConnection`. */
|
|
289
|
+
const TRANSIENT_POLL_CAP_MS = 4_000;
|
|
290
|
+
/** Growth sampling step while waiting out one `POLL_INTERVAL_MS` grace window. */
|
|
291
|
+
const ARTIFACT_GROWTH_POLL_MS = 50;
|
|
152
292
|
|
|
153
293
|
function parseDepthCeiling(raw: string | undefined): number | undefined {
|
|
154
294
|
// Genuinely unset / blank → no env ceiling (settings + hard cap still apply).
|
|
@@ -268,6 +408,9 @@ export class Orchestrator {
|
|
|
268
408
|
private readonly onBudgetRefused?: (event: BudgetRefusedEvent) => void;
|
|
269
409
|
private readonly legion?: OrchestratorLegionContext;
|
|
270
410
|
private readonly requestedDepthCap?: number;
|
|
411
|
+
/** D6/D10: queue bounds passed to the process-level queue (first acquisition wins). */
|
|
412
|
+
private readonly launchQueueBounds: { maxPending?: number; timeoutMs?: number };
|
|
413
|
+
private readonly onLaunchQueueNotice?: (line: string) => void;
|
|
271
414
|
private depthWarningEmitted = false;
|
|
272
415
|
private worktreeDeniedWarningEmitted = false;
|
|
273
416
|
private spawned = 0;
|
|
@@ -276,6 +419,34 @@ export class Orchestrator {
|
|
|
276
419
|
private readonly pendingWaits = new Map<string, Promise<SettleResult>>();
|
|
277
420
|
/** Children retired because the parent session is shutting down. */
|
|
278
421
|
private readonly parentShutdownRetire = new Set<string>();
|
|
422
|
+
/**
|
|
423
|
+
* D2-b in-flight reservation: admissions this tick already waved through but
|
|
424
|
+
* which the authoritative gates cannot see yet (the child only lands in the
|
|
425
|
+
* supervisor's `children` map after the first `await` in `spawnChild`).
|
|
426
|
+
* Without it, a same-tick fan-out of 12 probes all read 0 live children and
|
|
427
|
+
* all pass — which is exactly the bug #266 exists to fix.
|
|
428
|
+
*/
|
|
429
|
+
private readonly admitting = new Map<string, { parentKey: string; dimension: AdmissionDimension }>();
|
|
430
|
+
/** Lazily acquired process-level queue for this run directory (§13.6). */
|
|
431
|
+
private admissionQueueRef?: AdmissionQueue<LaunchQueuePayload>;
|
|
432
|
+
/**
|
|
433
|
+
* Per-name admission guard: the `settled` channel for `collect`, the re-probe
|
|
434
|
+
* closure for D2-c/D5, and the original deadline a requeue must honour.
|
|
435
|
+
*/
|
|
436
|
+
private readonly launchGuards = new Map<string, LaunchGuard>();
|
|
437
|
+
/** D2-d-4: at most one unref'd fallback timer while the queue is non-empty. */
|
|
438
|
+
private queueWatchdog?: ReturnType<typeof setTimeout>;
|
|
439
|
+
/** `pumpAdmissions` mutex — a second concurrent pump must never double-admit. */
|
|
440
|
+
private pumping = false;
|
|
441
|
+
/**
|
|
442
|
+
* D2-d-2 subscription, held only while this instance has pending items.
|
|
443
|
+
*
|
|
444
|
+
* `index.ts` creates a fresh Orchestrator per control action, so subscribing
|
|
445
|
+
* eagerly in the constructor would leak one listener per tool call. Subscribing
|
|
446
|
+
* on the first queued item (and dropping it when the queue empties) keeps the
|
|
447
|
+
* listener set proportional to real queue owners.
|
|
448
|
+
*/
|
|
449
|
+
private unsubscribeSlotReleases?: () => void;
|
|
279
450
|
|
|
280
451
|
constructor(deps: OrchestratorDeps) {
|
|
281
452
|
this.supervisor = deps.supervisor ?? new RpcSupervisor();
|
|
@@ -301,6 +472,8 @@ export class Orchestrator {
|
|
|
301
472
|
this.childContext = deps.childContext?.trim() || undefined;
|
|
302
473
|
this.rpcApprove = deps.rpcApprove === true;
|
|
303
474
|
this.onBudgetRefused = deps.onBudgetRefused;
|
|
475
|
+
this.launchQueueBounds = deps.launchQueue ?? {};
|
|
476
|
+
this.onLaunchQueueNotice = deps.onLaunchQueueNotice;
|
|
304
477
|
this.legion = deps.legion;
|
|
305
478
|
}
|
|
306
479
|
|
|
@@ -321,6 +494,11 @@ export class Orchestrator {
|
|
|
321
494
|
for (let index = seen; index < seen + 1000; index += 1) {
|
|
322
495
|
const candidate = makeName(agentName, index);
|
|
323
496
|
if (this.children.has(candidate) || this.supervisor.isAlive(candidate)) continue;
|
|
497
|
+
// #266 D4: an automatically allocated name must not collide with a QUEUED
|
|
498
|
+
// item either. The queue is process-level, so a sibling instance's pending
|
|
499
|
+
// launch is invisible to `this.children` / `isAlive` — allocating its name
|
|
500
|
+
// here would make the new (legitimate) launch fail with NAME_TAKEN.
|
|
501
|
+
if (this.isQueuedHere(candidate)) continue;
|
|
324
502
|
this.nameCounter.set(agentName, index + 1);
|
|
325
503
|
return candidate;
|
|
326
504
|
}
|
|
@@ -557,22 +735,15 @@ export class Orchestrator {
|
|
|
557
735
|
this.onBudgetRefused?.(event);
|
|
558
736
|
}
|
|
559
737
|
|
|
560
|
-
async launch(
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
worktree?: boolean;
|
|
570
|
-
modelOrigin?: ModelOrigin;
|
|
571
|
-
/** Vexillatio launch: bypass nested-depth gates while recording its parent-side Legion node. */
|
|
572
|
-
utilityDispatch?: boolean;
|
|
573
|
-
/** Mask inherited Legion identity from the spawned child process. */
|
|
574
|
-
stripLegionIdentity?: boolean;
|
|
575
|
-
}): Promise<Handle> {
|
|
738
|
+
async launch(
|
|
739
|
+
input: LaunchInput,
|
|
740
|
+
/**
|
|
741
|
+
* Internal replay of an already-admitted launch (issue #266 D2-c / §4.3).
|
|
742
|
+
* Public callers must omit it: admission is decided exactly once, at the
|
|
743
|
+
* frozen insertion point below.
|
|
744
|
+
*/
|
|
745
|
+
internal: { admissionAlreadyGranted?: boolean } = {},
|
|
746
|
+
): Promise<Handle> {
|
|
576
747
|
this.warnDepthCap();
|
|
577
748
|
if (input.agent.kind !== "pi") {
|
|
578
749
|
throw new SubagentError(
|
|
@@ -584,7 +755,12 @@ export class Orchestrator {
|
|
|
584
755
|
const name = this.legion ? this.legionLocalName(generated) : generated;
|
|
585
756
|
const nodeId = this.legion ? nodeIdForChild(this.legion.nodeId, name) : undefined;
|
|
586
757
|
const ledgerPrevious = nodeId ? getNode(this.legion!.db, nodeId) : null;
|
|
587
|
-
|
|
758
|
+
// D3-a: a `queued` record carries a PLACEHOLDER session file, so it must never
|
|
759
|
+
// be mistaken for a resume credential (R8). Only real records resume.
|
|
760
|
+
const priorRecord = this.children.get(name);
|
|
761
|
+
const previous = priorRecord && priorRecord.state !== "queued"
|
|
762
|
+
? priorRecord
|
|
763
|
+
: (ledgerPrevious ? this.childFromNode(name, ledgerPrevious) : undefined);
|
|
588
764
|
if (previous && !this.children.has(name)) this.children.set(name, previous);
|
|
589
765
|
const models = this.planModelCandidates({
|
|
590
766
|
agent: input.agent,
|
|
@@ -653,6 +829,27 @@ export class Orchestrator {
|
|
|
653
829
|
// In-process children use subagents.inprocess.* budgets (never RPC keys).
|
|
654
830
|
if (!input.utilityDispatch && !inProcess) this.assertWithinBudgets(Boolean(previous));
|
|
655
831
|
if (this.supervisor.isAlive(name)) throw new SubagentError(`child is already alive: ${name}`, ErrorCodes.NAME_TAKEN);
|
|
832
|
+
// D4: the name-occupancy check must cover QUEUED records too. `previous`
|
|
833
|
+
// deliberately ignores them (D3-a: its placeholder session path is not a resume
|
|
834
|
+
// credential) and a queued child is not `alive`, so without this clause a
|
|
835
|
+
// second `launch({name})` would enqueue the SAME name twice and only fail
|
|
836
|
+
// later, at admission, with a confusing `child is already alive`.
|
|
837
|
+
//
|
|
838
|
+
// The two cases are NOT the same refusal (I-O3 / §13.3.1):
|
|
839
|
+
// - the item is still in THIS process's queue → `NAME_TAKEN` (a live launch is
|
|
840
|
+
// in flight under that name; wait, steer it, or stop it);
|
|
841
|
+
// - the record is a leftover from a previous process → the queue is gone and it
|
|
842
|
+
// can never start, so `resume`/`continue` must get the "re-dispatch it"
|
|
843
|
+
// message instead of a `NAME_TAKEN` that implies something is running.
|
|
844
|
+
// That second branch is what keeps I-O3's promise and lets a caller relaunch
|
|
845
|
+
// the same name after a restart.
|
|
846
|
+
const queuedHere = this.children.get(name)?.state === "queued";
|
|
847
|
+
if (queuedHere || this.isQueuedHere(name)) {
|
|
848
|
+
if (queuedHere && !this.isQueuedHere(name)) {
|
|
849
|
+
throw new SubagentError(launchQueueLostMessage(name), ErrorCodes.BUDGET_EXCEEDED);
|
|
850
|
+
}
|
|
851
|
+
throw new SubagentError(`child is already alive: ${name}`, ErrorCodes.NAME_TAKEN);
|
|
852
|
+
}
|
|
656
853
|
// Only an EXPLICIT inprocess can reach this: the arbitration above already
|
|
657
854
|
// routed every omitted-mode launch that needs a worktree to rpc, so I1
|
|
658
855
|
// (explicit inprocess + worktree stays a loud throw) holds. This replaces
|
|
@@ -665,6 +862,38 @@ export class Orchestrator {
|
|
|
665
862
|
ErrorCodes.INVALID_PARAMS,
|
|
666
863
|
);
|
|
667
864
|
}
|
|
865
|
+
// ── Launch admission queue (issue #266, contract §4.3 frozen insertion point) ──
|
|
866
|
+
// Here and not later on purpose: the name is allocated (D4), the backend is
|
|
867
|
+
// arbitrated, `assertWithinBudgets`/`isAlive` already passed, the illegal
|
|
868
|
+
// `inprocess + worktree` combination is rejected, and **nothing has happened
|
|
869
|
+
// yet** (no worktree, no run dir, no session file, no ledger row). Enqueuing
|
|
870
|
+
// after `reserveNode` would self-lock the fanout gate on a `starting` row
|
|
871
|
+
// that belongs to a child nobody started (R3/F10).
|
|
872
|
+
const probeContext: AdmissionProbeContext = {
|
|
873
|
+
inProcess,
|
|
874
|
+
agent,
|
|
875
|
+
...(nodeId ? { nodeId } : {}),
|
|
876
|
+
ledgerPrevious,
|
|
877
|
+
utility: Boolean(input.utilityDispatch && input.agent.utility),
|
|
878
|
+
};
|
|
879
|
+
if (!internal.admissionAlreadyGranted) {
|
|
880
|
+
const verdict = this.probeAdmission(name, probeContext);
|
|
881
|
+
if (verdict.admitted === false && verdict.queueable === true) {
|
|
882
|
+
// D1: only the four queueable gates land here. Everything else falls
|
|
883
|
+
// through to master's own gate below, so its message stays byte-identical
|
|
884
|
+
// (D1-b: never re-wrap a `BUDGET_EXCEEDED` as a queue decision).
|
|
885
|
+
return this.enqueueLaunch(input, name, verdict, probeContext, mode);
|
|
886
|
+
}
|
|
887
|
+
if (verdict.admitted === true) {
|
|
888
|
+
// D2-b: reserve the slot in THIS sync block. The child does not reach the
|
|
889
|
+
// supervisor's registry until the first `await` inside `spawnChild`, so a
|
|
890
|
+
// same-tick fan-out would otherwise see 0 live children 12 times (R1).
|
|
891
|
+
this.admitting.set(name, {
|
|
892
|
+
parentKey: this.admissionParentKey(name, probeContext),
|
|
893
|
+
dimension: "fanout",
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
}
|
|
668
897
|
let worktreePath = previous?.worktreePath;
|
|
669
898
|
let worktreeBranch = previous?.worktreeBranch;
|
|
670
899
|
let createdWorktree = false;
|
|
@@ -688,7 +917,7 @@ export class Orchestrator {
|
|
|
688
917
|
timeoutMs: this.worktreeBootstrapTimeoutMs,
|
|
689
918
|
});
|
|
690
919
|
}
|
|
691
|
-
const effectiveModel = models
|
|
920
|
+
const effectiveModel = models;
|
|
692
921
|
if (this.legion && nodeId && !previous && this.legion.runStore) {
|
|
693
922
|
const roleDepth = input.agent.maxSubagentDepth === undefined
|
|
694
923
|
? Number.POSITIVE_INFINITY
|
|
@@ -712,9 +941,9 @@ export class Orchestrator {
|
|
|
712
941
|
preCreateSessionFile(sessionFile);
|
|
713
942
|
if (this.legion && nodeId) {
|
|
714
943
|
const reservationRunId = createdRunId ?? (previous ? ledgerPrevious?.runId : undefined) ?? this.legion.runId;
|
|
715
|
-
reservedNode = this.reserveNode({ nodeId, name, agent, task: input.task, sessionFile, runId: reservationRunId, team: input.team, worktreePath, model: models
|
|
944
|
+
reservedNode = this.reserveNode({ nodeId, name, agent, task: input.task, sessionFile, runId: reservationRunId, team: input.team, worktreePath, model: models[0], utility: Boolean(input.utilityDispatch && input.agent.utility) });
|
|
716
945
|
reserved = true;
|
|
717
|
-
parentReservationCreated = await this.persistParentReservation({ nodeId, name, agent: input.agent, sessionFile, ...(models
|
|
946
|
+
parentReservationCreated = await this.persistParentReservation({ nodeId, name, agent: input.agent, sessionFile, ...(models[0] ? { model: models[0] } : {}), ...(worktreePath ? { worktreePath } : {}) });
|
|
718
947
|
}
|
|
719
948
|
const childEnv = this.lineageEnv(name, input.agent, input.team, nodeId);
|
|
720
949
|
if (input.stripLegionIdentity) {
|
|
@@ -780,6 +1009,9 @@ export class Orchestrator {
|
|
|
780
1009
|
}
|
|
781
1010
|
}
|
|
782
1011
|
if (!result) throw lastError ?? new SubagentError("RPC child start failed", ErrorCodes.START_FAILED);
|
|
1012
|
+
// The child is in the supervisor's registry now, so the advisory in-flight
|
|
1013
|
+
// reservation is superseded by the real counts (D2-b).
|
|
1014
|
+
this.admitting.delete(name);
|
|
783
1015
|
if (previous) {
|
|
784
1016
|
previous.state = "working";
|
|
785
1017
|
previous.spawnedAt = new Date(this.now()).toISOString();
|
|
@@ -815,8 +1047,12 @@ export class Orchestrator {
|
|
|
815
1047
|
child,
|
|
816
1048
|
// #209: callers render this instead of assuming the RPC backend.
|
|
817
1049
|
spawnMode: mode,
|
|
1050
|
+
// Only an `rpc` child owns a process; `inprocess` reports no pid, so no
|
|
1051
|
+
// value is invented. Never persisted — a pid is host-scoped.
|
|
1052
|
+
...(result.pid ? { pid: result.pid } : {}),
|
|
818
1053
|
};
|
|
819
1054
|
} catch (error) {
|
|
1055
|
+
this.admitting.delete(name);
|
|
820
1056
|
if (spawned) await this.supervisor.retire(name).catch(() => {});
|
|
821
1057
|
if (reserved && nodeId) this.rollbackReservation(nodeId, reservedNode);
|
|
822
1058
|
if (parentReservationCreated && this.legion?.runStore) {
|
|
@@ -835,19 +1071,777 @@ export class Orchestrator {
|
|
|
835
1071
|
model?: string;
|
|
836
1072
|
origin: ModelOrigin;
|
|
837
1073
|
thinking?: string | false;
|
|
838
|
-
}):
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
const
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
1074
|
+
}): Array<string | undefined> { const requested = modelCandidates(input.model ?? input.agent.model, input.agent.fallbackModels);
|
|
1075
|
+
// Every candidate now belongs to the one supported kind, so the model the
|
|
1076
|
+
// caller asked for is the model the child starts with; an unset model is
|
|
1077
|
+
// the only case that leaves the choice to the child's own configuration.
|
|
1078
|
+
const candidates = requested.filter((model): model is string => Boolean(model));
|
|
1079
|
+
return candidates.length ? candidates : [undefined];
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
// ── Launch admission queue (issue #266) ─────────────────────────────────
|
|
1083
|
+
|
|
1084
|
+
/**
|
|
1085
|
+
* `maxChildrenPerNode` groups children by the name before the last dot — the
|
|
1086
|
+
* exact key `InProcessSupervisor` uses (`parentKeyOf`), so the probe and the
|
|
1087
|
+
* authoritative gate count the same siblings.
|
|
1088
|
+
*/
|
|
1089
|
+
private admissionParentKey(name: string, ctx: AdmissionProbeContext): string {
|
|
1090
|
+
const probe = ctx.inProcess ? inProcessProbeOf(this.supervisor) : undefined;
|
|
1091
|
+
if (probe) return probe.admissionCounts(name).parentKey;
|
|
1092
|
+
const idx = name.lastIndexOf(".");
|
|
1093
|
+
return idx === -1 ? "" : name.slice(0, idx);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* D2-b: in-flight reservations attributable to one parent key.
|
|
1098
|
+
*
|
|
1099
|
+
* A reservation must only cover the window in which the backend cannot see the
|
|
1100
|
+
* child yet. That window is narrow and backend-specific: the in-process
|
|
1101
|
+
* supervisor registers a child **synchronously** inside `spawnChild`, and the
|
|
1102
|
+
* legion path inserts its `starting` row synchronously inside `reserveNode`. Once
|
|
1103
|
+
* either has happened the real counts already include the launch, so counting the
|
|
1104
|
+
* reservation too would double it and halve the effective cap (measured: a
|
|
1105
|
+
* 9-launch same-tick fan-out with `maxChildrenPerNode: 8` admitted only 4).
|
|
1106
|
+
*/
|
|
1107
|
+
private inflightFor(parentKey: string, ctx?: AdmissionProbeContext): { fanout: number; active: number } {
|
|
1108
|
+
let fanout = 0;
|
|
1109
|
+
let active = 0;
|
|
1110
|
+
for (const [name, entry] of this.admitting) {
|
|
1111
|
+
if (this.launchAlreadyVisible(name, ctx)) continue;
|
|
1112
|
+
active += 1;
|
|
1113
|
+
if (entry.parentKey === parentKey) fanout += 1;
|
|
846
1114
|
}
|
|
847
|
-
|
|
848
|
-
|
|
1115
|
+
return { fanout, active };
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
/** Has the backend registered this in-flight launch yet? */
|
|
1119
|
+
private launchAlreadyVisible(name: string, ctx?: AdmissionProbeContext): boolean {
|
|
1120
|
+
const record = this.children.get(name);
|
|
1121
|
+
if (record && record.state !== "queued") return true;
|
|
1122
|
+
if (this.supervisor.isAlive(name)) return true;
|
|
1123
|
+
const legion = this.legion;
|
|
1124
|
+
if (!ctx?.nodeId || !legion) return false;
|
|
1125
|
+
// `reserveNode` inserts the row before the awaited `persistParentReservation`,
|
|
1126
|
+
// so an existing row means the ledger counts this launch already.
|
|
1127
|
+
return getNode(legion.db, ctx.nodeId) !== null;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
/**
|
|
1131
|
+
* The advisory admission probe (contract §4.3, I-O1).
|
|
1132
|
+
*
|
|
1133
|
+
* Synchronous on purpose: the fast path may not gain a single `await` (D2-b /
|
|
1134
|
+
* A21). `admissionCounts` and `checkLaunchBudget` are both plain reads.
|
|
1135
|
+
*
|
|
1136
|
+
* Nothing here replaces an authoritative gate: an `admitted: true` verdict only
|
|
1137
|
+
* skips the *queue*, never `assertWithinLimits` / `withLaunchBudget` (I-O1). A
|
|
1138
|
+
* launch that still loses the real gate is re-queued instead of failing (D2-c).
|
|
1139
|
+
*/
|
|
1140
|
+
private probeAdmission(name: string, ctx: AdmissionProbeContext): AdmissionVerdict {
|
|
1141
|
+
if (ctx.inProcess) {
|
|
1142
|
+
const probe = inProcessProbeOf(this.supervisor);
|
|
1143
|
+
// No readable counts (a test double) → nothing to probe.
|
|
1144
|
+
if (!probe) return { admitted: true };
|
|
1145
|
+
const counts = probe.admissionCounts(name);
|
|
1146
|
+
return evaluateInProcessAdmission(counts, this.inflightFor(counts.parentKey, ctx));
|
|
1147
|
+
}
|
|
1148
|
+
// rpc/legion: the ledger gate. Read-only (`checkLaunchBudget`), never
|
|
1149
|
+
// `withLaunchBudget` — a probe must not insert a `starting` row (R3/F10) and
|
|
1150
|
+
// must not emit `budget_refused`, which belongs to `reserveNode` (D1-b).
|
|
1151
|
+
const legion = this.legion;
|
|
1152
|
+
if (!legion || !ctx.nodeId) return { admitted: true };
|
|
1153
|
+
const parent = getNode(legion.db, legion.nodeId);
|
|
1154
|
+
// `reserveNode` owns `parent_not_found`; do not pre-empt it.
|
|
1155
|
+
if (!parent) return { admitted: true };
|
|
1156
|
+
const maxChildren = legion.settings?.maxChildrenPerNode ?? 8;
|
|
1157
|
+
const depthCeiling = ctx.utility
|
|
1158
|
+
? parent.depth + 1
|
|
1159
|
+
: Math.min(legion.settings?.maxDepth ?? this.maxDepthValue, MAX_NESTED_PATH_ENTRIES);
|
|
1160
|
+
const maxActiveNodes = legion.settings?.maxActiveNodes ?? 30;
|
|
1161
|
+
const decision = checkLaunchBudget(
|
|
1162
|
+
legion.db,
|
|
1163
|
+
{ parentId: legion.nodeId, depth: parent.depth + 1, nodeId: ctx.nodeId },
|
|
1164
|
+
{ limits: { maxDepth: depthCeiling, maxChildrenPerNode: maxChildren, maxActiveNodes } },
|
|
1165
|
+
);
|
|
1166
|
+
if (decision.allowed) return { admitted: true };
|
|
1167
|
+
// D2-b: the ledger snapshot cannot see the rows this tick has not inserted
|
|
1168
|
+
// yet, so the in-flight reservations are added before judging.
|
|
1169
|
+
const inflight = this.inflightFor("", ctx);
|
|
1170
|
+
if (decision.reason === "max_children_per_node") {
|
|
1171
|
+
const children = decision.children + inflight.fanout;
|
|
1172
|
+
if (maxChildren === 0) {
|
|
1173
|
+
// D11: `0` means "no children at all" — waiting cannot satisfy it.
|
|
1174
|
+
return {
|
|
1175
|
+
admitted: false,
|
|
1176
|
+
queueable: false,
|
|
1177
|
+
gate: "ledger.maxChildrenPerNode.disabled",
|
|
1178
|
+
message: `legion launch refused (max_children_per_node): parent already has ${children} live children; maxChildrenPerNode is 0`,
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
if (children >= maxChildren) {
|
|
1182
|
+
return {
|
|
1183
|
+
admitted: false,
|
|
1184
|
+
queueable: true,
|
|
1185
|
+
gate: "ledger.maxChildrenPerNode",
|
|
1186
|
+
dimension: "fanout",
|
|
1187
|
+
message: `legion launch refused (max_children_per_node): parent already has ${children} live children; maxChildrenPerNode is ${maxChildren}`,
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
return { admitted: true };
|
|
1191
|
+
}
|
|
1192
|
+
if (decision.reason === "max_active_nodes") {
|
|
1193
|
+
const activeNodes = decision.activeNodes + inflight.active;
|
|
1194
|
+
if (activeNodes >= maxActiveNodes) {
|
|
1195
|
+
return {
|
|
1196
|
+
admitted: false,
|
|
1197
|
+
queueable: true,
|
|
1198
|
+
gate: "ledger.maxActiveNodes",
|
|
1199
|
+
dimension: "active",
|
|
1200
|
+
message: `legion launch refused (max_active_nodes): tree already has ${activeNodes} active nodes; maxActiveNodes is ${maxActiveNodes}`,
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
return { admitted: true };
|
|
1204
|
+
}
|
|
1205
|
+
// `max_depth` / `parent_not_found`: permanent, never queued (D1).
|
|
1206
|
+
return { admitted: true };
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
/**
|
|
1210
|
+
* The process-level queue for this run directory (§13.6 / I-O6).
|
|
1211
|
+
*
|
|
1212
|
+
* `acquireAdmissionQueue` is idempotent per (pid, runDir), so the five
|
|
1213
|
+
* `new Orchestrator` sites in `index.ts` share one queue — that is what lets
|
|
1214
|
+
* `subagent({action:"stop"})` dequeue an item it never enqueued.
|
|
1215
|
+
*/
|
|
1216
|
+
private admissionQueue(): AdmissionQueue<LaunchQueuePayload> {
|
|
1217
|
+
this.admissionQueueRef ??= acquireAdmissionQueue<LaunchQueuePayload>(this.runDir, {
|
|
1218
|
+
...this.launchQueueBounds,
|
|
1219
|
+
now: this.now,
|
|
1220
|
+
});
|
|
1221
|
+
return this.admissionQueueRef;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/**
|
|
1225
|
+
* The queue **if one already exists**, without creating it.
|
|
1226
|
+
*
|
|
1227
|
+
* The read-only and control paths must not acquire: acquisition installs an
|
|
1228
|
+
* empty queue with default bounds, and "first acquisition wins" would then
|
|
1229
|
+
* shadow the real owner's `launchQueue` settings.
|
|
1230
|
+
*/
|
|
1231
|
+
private existingAdmissionQueue(): AdmissionQueue<LaunchQueuePayload> | undefined {
|
|
1232
|
+
// §6.1/§13.6: a control-action instance (stop/status/collect) must reach the
|
|
1233
|
+
// queue a SIBLING instance created — otherwise `subagent stop <queued>` is a
|
|
1234
|
+
// silent no-op and `status` reports it as "queue lost". `peek` is read-only on
|
|
1235
|
+
// purpose: acquiring here would install an empty queue with default bounds and
|
|
1236
|
+
// shadow the real owner's settings ("first acquisition wins").
|
|
1237
|
+
return this.admissionQueueRef ?? peekAdmissionQueue<LaunchQueuePayload>(this.runDir);
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/** D9: one user-visible line; a UI hook must never break admission. */
|
|
1241
|
+
private notice(line: string): void {
|
|
1242
|
+
const fn = this.onLaunchQueueNotice;
|
|
1243
|
+
if (!fn) return;
|
|
1244
|
+
try {
|
|
1245
|
+
fn(line);
|
|
1246
|
+
} catch {
|
|
1247
|
+
/* the notice is cosmetic */
|
|
849
1248
|
}
|
|
850
|
-
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* D3/D3-a: enqueue a launch that only hit a *queueable* gate.
|
|
1253
|
+
*
|
|
1254
|
+
* The three "must not create" rules hold here (asserted by A3/A15): no ledger
|
|
1255
|
+
* `nodes` row, no child run directory / `run.json`, no `<name>.jsonl` file.
|
|
1256
|
+
* `sessionFile` is a placeholder string and `nodeId` is the deterministic id
|
|
1257
|
+
* the later admission will use — computed, never inserted.
|
|
1258
|
+
*/
|
|
1259
|
+
private enqueueLaunch(
|
|
1260
|
+
input: LaunchInput,
|
|
1261
|
+
name: string,
|
|
1262
|
+
verdict: Extract<AdmissionVerdict, { queueable: true }>,
|
|
1263
|
+
ctx: AdmissionProbeContext,
|
|
1264
|
+
mode: SpawnMode,
|
|
1265
|
+
): Promise<Handle> {
|
|
1266
|
+
const queue = this.admissionQueue();
|
|
1267
|
+
const parentKey = this.admissionParentKey(name, ctx);
|
|
1268
|
+
let settle: (outcome: AdmitOutcome) => void = () => {};
|
|
1269
|
+
const settled = new Promise<AdmitOutcome>((resolve) => {
|
|
1270
|
+
settle = resolve;
|
|
1271
|
+
});
|
|
1272
|
+
const parentDepth = inProcessDepth();
|
|
1273
|
+
const parentContext = currentChildContext();
|
|
1274
|
+
const payload: LaunchQueuePayload = {
|
|
1275
|
+
name,
|
|
1276
|
+
parentKey,
|
|
1277
|
+
inProcess: ctx.inProcess,
|
|
1278
|
+
parentDepth,
|
|
1279
|
+
...(parentContext ? { parentContext } : {}),
|
|
1280
|
+
run: () => this.launch({ ...input, name }, { admissionAlreadyGranted: true }),
|
|
1281
|
+
settled,
|
|
1282
|
+
};
|
|
1283
|
+
const outcome = queue.enqueue(payload, { name, parentKey });
|
|
1284
|
+
if (!outcome.ok) {
|
|
1285
|
+
// D6: overflow refuses immediately — nothing enqueued, nothing recorded,
|
|
1286
|
+
// and any name reservation released.
|
|
1287
|
+
this.children.delete(name);
|
|
1288
|
+
throw new SubagentError(outcome.message, ErrorCodes.BUDGET_EXCEEDED);
|
|
1289
|
+
}
|
|
1290
|
+
// D3-a: one `queued` record in the parent run.json (cross-process `status`
|
|
1291
|
+
// must show it, D9), and nothing else is created.
|
|
1292
|
+
const child: ChildRecord = {
|
|
1293
|
+
name,
|
|
1294
|
+
...(ctx.nodeId ? { nodeId: ctx.nodeId } : {}),
|
|
1295
|
+
sessionFile: this.sessionFileFor(name),
|
|
1296
|
+
ownerToken: ownerToken(),
|
|
1297
|
+
state: "queued",
|
|
1298
|
+
spawnedAt: new Date(this.now()).toISOString(),
|
|
1299
|
+
agent: input.agent.name,
|
|
1300
|
+
kind: input.agent.kind,
|
|
1301
|
+
spawnMode: mode,
|
|
1302
|
+
queuePosition: outcome.position,
|
|
1303
|
+
};
|
|
1304
|
+
this.children.set(name, child);
|
|
1305
|
+
this.onChildUpdate(child);
|
|
1306
|
+
this.launchGuards.set(name, {
|
|
1307
|
+
settle,
|
|
1308
|
+
settled,
|
|
1309
|
+
probe: () => this.probeAdmission(name, ctx),
|
|
1310
|
+
dimension: verdict.dimension,
|
|
1311
|
+
enqueuedAt: this.now(),
|
|
1312
|
+
});
|
|
1313
|
+
this.watchSlotReleases(true);
|
|
1314
|
+
this.notice(`▶ ${name} (${input.agent.name}) supervisor=${mode} queued(${outcome.position})`);
|
|
1315
|
+
// D2-d-1: this enqueue may itself have a free slot behind it.
|
|
1316
|
+
this.schedulePump();
|
|
1317
|
+
// D8: `async` semantics are unchanged — a queued handle returns at once.
|
|
1318
|
+
return Promise.resolve({
|
|
1319
|
+
name,
|
|
1320
|
+
sessionFile: child.sessionFile,
|
|
1321
|
+
runId: path.basename(this.runDir),
|
|
1322
|
+
agent: input.agent.name,
|
|
1323
|
+
kind: input.agent.kind,
|
|
1324
|
+
child,
|
|
1325
|
+
spawnMode: mode,
|
|
1326
|
+
queued: true,
|
|
1327
|
+
queuePosition: outcome.position,
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* D2-d: wake the pump. `pumping` makes overlapping wakes harmless.
|
|
1333
|
+
*
|
|
1334
|
+
* Guarded on owning at least one launch: the queue is process-level and shared
|
|
1335
|
+
* by every Orchestrator on this run dir (`index.ts` has five construction
|
|
1336
|
+
* sites), but only the instance that enqueued an item holds its `LaunchQueuePayload`
|
|
1337
|
+
* — the closure that actually performs the launch. An instance with no guards is
|
|
1338
|
+
* a control-action instance (`stop`/`status`/`collect`); letting it pump would
|
|
1339
|
+
* admit a sibling's items through a probe that cannot judge them, which is how
|
|
1340
|
+
* retired unrelated children used to spawn somebody else's queued work and
|
|
1341
|
+
* reproduce the very `maxChildrenPerNode reached` line #266 removes.
|
|
1342
|
+
*/
|
|
1343
|
+
private schedulePump(): void {
|
|
1344
|
+
if (this.launchGuards.size === 0) return;
|
|
1345
|
+
void this.pumpAdmissions();
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
/**
|
|
1349
|
+
* D2-d-2: subscribe to in-process slot releases only while this instance owns a
|
|
1350
|
+
* non-empty queue, and drop the subscription as soon as it empties.
|
|
1351
|
+
*
|
|
1352
|
+
* The event source is a process-level broadcast (`routing-supervisor.ts`): the
|
|
1353
|
+
* Orchestrator need not own the supervisor, and `index.ts` hands the same
|
|
1354
|
+
* `RoutingSupervisor` to five different Orchestrator instances.
|
|
1355
|
+
*/
|
|
1356
|
+
private watchSlotReleases(wanted: boolean): void {
|
|
1357
|
+
if (wanted && !this.unsubscribeSlotReleases) {
|
|
1358
|
+
this.unsubscribeSlotReleases = subscribeInProcessSlotReleases(() => this.schedulePump());
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
if (!wanted && this.unsubscribeSlotReleases) {
|
|
1362
|
+
this.unsubscribeSlotReleases();
|
|
1363
|
+
this.unsubscribeSlotReleases = undefined;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
/**
|
|
1368
|
+
* D2-d-4: exactly one unref'd fallback timer while the queue is non-empty.
|
|
1369
|
+
*
|
|
1370
|
+
* A ledger slot can also be freed by *another* process (a sibling session, or
|
|
1371
|
+
* hygiene) and that release sends no event here (§11 R2), so event edges alone
|
|
1372
|
+
* would stall the queue. The deadline is capped by `LAUNCH_QUEUE_RECHECK_MS`
|
|
1373
|
+
* inside `nextDeadlineAt()` and the timer is cleared once the queue empties.
|
|
1374
|
+
*
|
|
1375
|
+
* The callback is fully guarded: `pumpAdmissions` never rejects, and a throwing
|
|
1376
|
+
* timer body would surface as an unhandledRejection long after the instance was
|
|
1377
|
+
* abandoned (see `pumpAdmissions`).
|
|
1378
|
+
*/
|
|
1379
|
+
private armQueueWatchdog(): void {
|
|
1380
|
+
if (this.queueWatchdog) {
|
|
1381
|
+
clearTimeout(this.queueWatchdog);
|
|
1382
|
+
this.queueWatchdog = undefined;
|
|
1383
|
+
}
|
|
1384
|
+
const queue = this.admissionQueueRef;
|
|
1385
|
+
if (!queue || queue.size === 0) return;
|
|
1386
|
+
let deadline: number | undefined;
|
|
1387
|
+
try {
|
|
1388
|
+
deadline = queue.nextDeadlineAt();
|
|
1389
|
+
} catch {
|
|
1390
|
+
// A queue we cannot even read is a queue we must stop watching.
|
|
1391
|
+
this.abandonLaunchQueue();
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
if (deadline === undefined) return;
|
|
1395
|
+
const timer = setTimeout(() => {
|
|
1396
|
+
this.queueWatchdog = undefined;
|
|
1397
|
+
try {
|
|
1398
|
+
void this.pumpAdmissions();
|
|
1399
|
+
} catch {
|
|
1400
|
+
// `pumpAdmissions` is written not to throw; this is belt-and-braces so a
|
|
1401
|
+
// timer can never become an unhandledRejection/uncaughtException.
|
|
1402
|
+
this.abandonLaunchQueue();
|
|
1403
|
+
}
|
|
1404
|
+
}, Math.max(0, deadline - this.now()));
|
|
1405
|
+
// Never keep the host process alive just to watch an empty queue.
|
|
1406
|
+
timer.unref?.();
|
|
1407
|
+
this.queueWatchdog = timer;
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
/**
|
|
1411
|
+
* Stop being a queue owner.
|
|
1412
|
+
*
|
|
1413
|
+
* Used when this instance can no longer serve the queue: its clock/db can no
|
|
1414
|
+
* longer be probed (a disposed Orchestrator, e.g. one created per control action
|
|
1415
|
+
* and then dropped). The registry entry is deliberately left in place — another
|
|
1416
|
+
* live Orchestrator on the same run directory may still own it — but this
|
|
1417
|
+
* instance stops pumping, stops its timer and releases its slot listeners.
|
|
1418
|
+
*/
|
|
1419
|
+
private abandonLaunchQueue(): void {
|
|
1420
|
+
if (this.queueWatchdog) {
|
|
1421
|
+
clearTimeout(this.queueWatchdog);
|
|
1422
|
+
this.queueWatchdog = undefined;
|
|
1423
|
+
}
|
|
1424
|
+
this.watchSlotReleases(false);
|
|
1425
|
+
this.admissionQueueRef = undefined;
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
/**
|
|
1429
|
+
* Admit whatever the queue allows right now (contract §4 / D2-c / D2-d).
|
|
1430
|
+
*
|
|
1431
|
+
* Mutually exclusive via `pumping`. Every iteration asks the queue for its next
|
|
1432
|
+
* decision, which is the ONE place a timeout is evaluated (R7): an item under
|
|
1433
|
+
* admission has already left the queue and can never expire underneath the
|
|
1434
|
+
* child it spawned.
|
|
1435
|
+
*
|
|
1436
|
+
* **Never rejects.** Every caller is fire-and-forget (`void this.pump…`), so a
|
|
1437
|
+
* throw here would become an `unhandledRejection` attributed to whatever the
|
|
1438
|
+
* event loop is doing — typically `database is not open` after a test disposed
|
|
1439
|
+
* its fixture. A probe that throws means this instance can no longer judge
|
|
1440
|
+
* admission at all, so the affected items are retired as *lost* (I-O3 wording;
|
|
1441
|
+
* they never started and nothing was reserved) and the instance goes inert.
|
|
1442
|
+
*/
|
|
1443
|
+
private async pumpAdmissions(): Promise<void> {
|
|
1444
|
+
if (this.pumping) return;
|
|
1445
|
+
this.pumping = true;
|
|
1446
|
+
let probeFailed = false;
|
|
1447
|
+
try {
|
|
1448
|
+
const queue = this.admissionQueue();
|
|
1449
|
+
for (;;) {
|
|
1450
|
+
const lost: string[] = [];
|
|
1451
|
+
const next = queue.next((item) => {
|
|
1452
|
+
const guard = this.launchGuards.get(item.name);
|
|
1453
|
+
if (!guard) {
|
|
1454
|
+
// The queue is process-level and shared, but only the instance that
|
|
1455
|
+
// enqueued an item holds the payload that performs its launch. Admitting
|
|
1456
|
+
// somebody else's item here would run a spawn this instance cannot judge
|
|
1457
|
+
// (and could not record) — the pre-review code fell back to
|
|
1458
|
+
// `{admitted:true}` and did exactly that.
|
|
1459
|
+
//
|
|
1460
|
+
// Skipping is safe and is NOT a lost item: `next()` moves on to the
|
|
1461
|
+
// next candidate (D5-2), the owner's own pump admits it, and if no owner
|
|
1462
|
+
// is left the queue's deadline expires it (D6) through the watchdog.
|
|
1463
|
+
return { admitted: false, queueable: false, gate: "other", message: "" };
|
|
1464
|
+
}
|
|
1465
|
+
try {
|
|
1466
|
+
return guard.probe();
|
|
1467
|
+
} catch {
|
|
1468
|
+
// I-O3: nothing was reserved for this item, so “lost” is the truthful
|
|
1469
|
+
// outcome — never a rejection into an abandoned async frame.
|
|
1470
|
+
lost.push(item.name);
|
|
1471
|
+
return { admitted: false, queueable: false, gate: "other", message: "" };
|
|
1472
|
+
}
|
|
1473
|
+
});
|
|
1474
|
+
if (lost.length > 0) {
|
|
1475
|
+
probeFailed = true;
|
|
1476
|
+
for (const name of lost) this.abandonQueued(name, "flushed", launchQueueLostMessage(name));
|
|
1477
|
+
}
|
|
1478
|
+
if (next.kind === "none") break;
|
|
1479
|
+
if (next.kind === "expired") {
|
|
1480
|
+
// D6: the wait ran out — retire the placeholder, tell the user, and
|
|
1481
|
+
// leave the ledger untouched (nothing was ever reserved).
|
|
1482
|
+
this.abandonQueued(next.item.name, "timeout", next.message);
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
try {
|
|
1486
|
+
await this.admitQueued(next.item, next.waitedMs);
|
|
1487
|
+
} catch (error) {
|
|
1488
|
+
// A genuine launch failure: the enqueue-time `launch()` promise has
|
|
1489
|
+
// already been returned to the caller, so surface it once and keep the
|
|
1490
|
+
// pump alive for the remaining items.
|
|
1491
|
+
this.onAdmissionFailure(next.item.name, error);
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
} catch {
|
|
1495
|
+
// The queue itself became unreadable/closed — stop owning it rather than
|
|
1496
|
+
// retrying forever from a timer.
|
|
1497
|
+
probeFailed = true;
|
|
1498
|
+
} finally {
|
|
1499
|
+
this.pumping = false;
|
|
1500
|
+
if (probeFailed || (this.admissionQueueRef?.size ?? 0) === 0) {
|
|
1501
|
+
// Nothing left to admit, or this instance can no longer judge admission:
|
|
1502
|
+
// leave no active timer and no slot-release listener behind.
|
|
1503
|
+
this.abandonLaunchQueue();
|
|
1504
|
+
} else {
|
|
1505
|
+
this.armQueueWatchdog();
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
/** Report one admission failure without tearing down the pump. */
|
|
1511
|
+
private onAdmissionFailure(name: string, error: unknown): void {
|
|
1512
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1513
|
+
this.notice(`launch queue: '${name}' failed to start: ${detail}`);
|
|
1514
|
+
this.launchGuards.delete(name);
|
|
1515
|
+
const child = this.children.get(name);
|
|
1516
|
+
if (child && child.state === "queued") {
|
|
1517
|
+
child.state = "exited";
|
|
1518
|
+
this.onChildUpdate(child);
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
/**
|
|
1523
|
+
* Take one item out of the queue and run it.
|
|
1524
|
+
*
|
|
1525
|
+
* D2-c: the probe is advisory (I-O1), so the replay can still lose the real
|
|
1526
|
+
* gate. When that refusal is attributable to a *queueable* gate, the item goes
|
|
1527
|
+
* back into the queue (keeping its original deadline) instead of failing the
|
|
1528
|
+
* launch. Anything else — a hard gate, `NAME_TAKEN`, `START_FAILED`, a genuine
|
|
1529
|
+
* spawn failure — is rethrown to the caller unchanged.
|
|
1530
|
+
*/
|
|
1531
|
+
private async admitQueued(item: QueuedLaunch<LaunchQueuePayload>, waitedMs: number): Promise<void> {
|
|
1532
|
+
const payload = item.payload;
|
|
1533
|
+
const guard = this.launchGuards.get(payload.name);
|
|
1534
|
+
this.launchGuards.delete(payload.name);
|
|
1535
|
+
// The placeholder is gone: from here on the real launch path owns the record.
|
|
1536
|
+
this.children.delete(payload.name);
|
|
1537
|
+
// D2-b: reserve the slot for the duration of the real launch.
|
|
1538
|
+
this.admitting.set(payload.name, { parentKey: payload.parentKey, dimension: guard?.dimension ?? "fanout" });
|
|
1539
|
+
try {
|
|
1540
|
+
// A18: restore the ALS depth/identity captured at enqueue time. Without
|
|
1541
|
+
// `runAtAbsoluteDepth` a grandchild admitted from the root chain would be
|
|
1542
|
+
// recorded at depth 1 and `maxDepth` could never fire.
|
|
1543
|
+
const handle = await runAtAbsoluteDepth(payload.parentDepth, () =>
|
|
1544
|
+
payload.parentContext
|
|
1545
|
+
? runInChildContext(payload.parentContext, () => payload.run())
|
|
1546
|
+
: payload.run());
|
|
1547
|
+
// D8/D20: the real wait, for `async:false`'s `↳ name waited Ns` line.
|
|
1548
|
+
handle.child.queueWaitedMs = waitedMs;
|
|
1549
|
+
handle.child.queuePosition = undefined;
|
|
1550
|
+
this.onChildUpdate(handle.child);
|
|
1551
|
+
this.notice(`▶ ${payload.name} started after queued ${Math.round(waitedMs / 1000)}s`);
|
|
1552
|
+
guard?.settle({ kind: "admitted", waitedMs });
|
|
1553
|
+
} catch (error) {
|
|
1554
|
+
const again = guard?.probe() ?? ({ admitted: true } as AdmissionVerdict);
|
|
1555
|
+
const attributable = again.admitted === false && again.queueable === true;
|
|
1556
|
+
if (attributable && error instanceof SubagentError && error.code === ErrorCodes.BUDGET_EXCEEDED) {
|
|
1557
|
+
// D2-c: requeue and do NOT pump again right away — spinning here would
|
|
1558
|
+
// busy-loop while the probe and the real gate briefly disagree.
|
|
1559
|
+
this.requeue(payload, guard, waitedMs);
|
|
1560
|
+
return;
|
|
1561
|
+
}
|
|
1562
|
+
// The guard was already removed from the map, so `abandonQueued` (which
|
|
1563
|
+
// looks it up again) cannot settle it — settle the local reference here or
|
|
1564
|
+
// a concurrent `collect` waits on a promise nobody will ever resolve.
|
|
1565
|
+
guard?.settle({
|
|
1566
|
+
kind: "rejected",
|
|
1567
|
+
reason: "flushed",
|
|
1568
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1569
|
+
});
|
|
1570
|
+
this.abandonQueued(payload.name, "flushed", error instanceof Error ? error.message : String(error));
|
|
1571
|
+
throw error;
|
|
1572
|
+
} finally {
|
|
1573
|
+
this.admitting.delete(payload.name);
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
/**
|
|
1578
|
+
* D2-c: put a refused launch back into the queue.
|
|
1579
|
+
*
|
|
1580
|
+
* The frozen wording says "original `seq` (`enqueueWithSeq` or equivalent)". A
|
|
1581
|
+
* re-queue therefore has to preserve the **deadline**, not the position: W1's
|
|
1582
|
+
* `AdmissionQueue` assigns `seq` itself, so the item goes to the back, but its
|
|
1583
|
+
* `enqueuedAt + timeoutMs` is carried over so a launch that keeps losing the
|
|
1584
|
+
* real gate cannot reset its own timeout (R7).
|
|
1585
|
+
*/
|
|
1586
|
+
private requeue(payload: LaunchQueuePayload, guard: LaunchGuard | undefined, waitedMs: number): void {
|
|
1587
|
+
const queue = this.admissionQueue();
|
|
1588
|
+
const elapsed = guard ? this.now() - guard.enqueuedAt : waitedMs;
|
|
1589
|
+
if (guard && elapsed >= queue.timeoutMs) {
|
|
1590
|
+
this.abandonQueued(payload.name, "timeout", queueTimeoutMessage(payload.name, elapsed));
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
// The wait so far, so the record's `spawnedAt` stays truthful across a requeue.
|
|
1594
|
+
const carried = this.queuedPlaceholder(payload, elapsed);
|
|
1595
|
+
this.children.set(payload.name, carried);
|
|
1596
|
+
this.onChildUpdate(carried);
|
|
1597
|
+
if (guard) this.launchGuards.set(payload.name, guard);
|
|
1598
|
+
const outcome = queue.enqueue(payload, {
|
|
1599
|
+
name: payload.name,
|
|
1600
|
+
parentKey: payload.parentKey,
|
|
1601
|
+
// D2-c: carry the ORIGINAL deadline. `seq` is deliberately fresh (the item
|
|
1602
|
+
// goes to the back — D5 gives no one priority), but the wait must not
|
|
1603
|
+
// restart: a fresh stamp would reset `waitedMs` to 0 (under-reporting the
|
|
1604
|
+
// `async:false` "waited Ns" line) and hand the item a second full
|
|
1605
|
+
// `timeoutMs`, so an item that keeps losing the admission race could wait
|
|
1606
|
+
// nearly twice the configured bound before D6 fires.
|
|
1607
|
+
...(guard ? { enqueuedAt: guard.enqueuedAt } : {}),
|
|
1608
|
+
});
|
|
1609
|
+
if (!outcome.ok) {
|
|
1610
|
+
// The queue grew while this item was being admitted; overflow is still a
|
|
1611
|
+
// hard refusal (D6), never an unbounded wait.
|
|
1612
|
+
this.abandonQueued(payload.name, "stopped", outcome.message);
|
|
1613
|
+
return;
|
|
1614
|
+
}
|
|
1615
|
+
carried.queuePosition = outcome.position;
|
|
1616
|
+
this.onChildUpdate(carried);
|
|
1617
|
+
this.armQueueWatchdog();
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
/** The `queued` ChildRecord for an item that (re-)entered the queue. */
|
|
1621
|
+
private queuedPlaceholder(payload: LaunchQueuePayload, waitedMs: number): ChildRecord {
|
|
1622
|
+
const existing = this.children.get(payload.name);
|
|
1623
|
+
const base: ChildRecord = existing ?? {
|
|
1624
|
+
name: payload.name,
|
|
1625
|
+
sessionFile: this.sessionFileFor(payload.name),
|
|
1626
|
+
ownerToken: ownerToken(),
|
|
1627
|
+
state: "queued",
|
|
1628
|
+
spawnedAt: new Date(this.now()).toISOString(),
|
|
1629
|
+
spawnMode: payload.inProcess ? "inprocess" : "rpc",
|
|
1630
|
+
};
|
|
1631
|
+
return {
|
|
1632
|
+
...base,
|
|
1633
|
+
// `spawnedAt` is the record's creation stamp; backdating it by the wait
|
|
1634
|
+
// keeps it truthful across a requeue.
|
|
1635
|
+
spawnedAt: new Date(this.now() - waitedMs).toISOString(),
|
|
1636
|
+
state: "queued",
|
|
1637
|
+
retiredAt: undefined,
|
|
1638
|
+
};
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
/**
|
|
1642
|
+
* D6/D7: an item left the queue without spawning (timeout / stop / flush).
|
|
1643
|
+
* The record becomes `retired`, nothing is spawned, nothing reaches the ledger.
|
|
1644
|
+
*/
|
|
1645
|
+
private abandonQueued(
|
|
1646
|
+
name: string,
|
|
1647
|
+
reason: "timeout" | "stopped" | "flushed",
|
|
1648
|
+
message: string,
|
|
1649
|
+
): ChildRecord | undefined {
|
|
1650
|
+
const guard = this.launchGuards.get(name);
|
|
1651
|
+
this.launchGuards.delete(name);
|
|
1652
|
+
const child = this.children.get(name);
|
|
1653
|
+
if (child && child.state === "queued") {
|
|
1654
|
+
child.state = "retired";
|
|
1655
|
+
child.retiredAt = new Date(this.now()).toISOString();
|
|
1656
|
+
child.queuePosition = undefined;
|
|
1657
|
+
this.onChildUpdate(child);
|
|
1658
|
+
}
|
|
1659
|
+
// `stopped` is announced by `retire()`, which owns that user-facing line, but
|
|
1660
|
+
// the waiter MUST still be settled: a `collect` blocked on `settled` would
|
|
1661
|
+
// otherwise hang forever after its item was stopped (D7/D8).
|
|
1662
|
+
if (reason !== "stopped") this.notice(message);
|
|
1663
|
+
guard?.settle({ kind: "rejected", reason, message });
|
|
1664
|
+
return child?.state === "retired" ? child : undefined;
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
/**
|
|
1668
|
+
* D7 / A5: empty the queue without spawning anything.
|
|
1669
|
+
*
|
|
1670
|
+
* Called from the parent turn's abort path and from session shutdown. Returns
|
|
1671
|
+
* the flushed names in `seq` order so the caller can report them.
|
|
1672
|
+
*/
|
|
1673
|
+
flushLaunchQueue(reason: string): string[] {
|
|
1674
|
+
const queue = this.existingAdmissionQueue();
|
|
1675
|
+
if (this.queueWatchdog) {
|
|
1676
|
+
clearTimeout(this.queueWatchdog);
|
|
1677
|
+
this.queueWatchdog = undefined;
|
|
1678
|
+
}
|
|
1679
|
+
if (!queue) return [];
|
|
1680
|
+
// The queue is shared with every other Orchestrator on this run dir, so a
|
|
1681
|
+
// flush must only clear what THIS instance can finish clearing: the items it
|
|
1682
|
+
// holds a guard for. Clearing a sibling's item would dequeue it without
|
|
1683
|
+
// settling the owner's guard or retiring the owner's record, leaving a
|
|
1684
|
+
// permanent `queued` zombie (and a `collect` that never returns). The owner's
|
|
1685
|
+
// own abort/flush path — or the D6 deadline — retires its items.
|
|
1686
|
+
const mine = queue.entries().filter((entry) => this.launchGuards.has(entry.name)).map((entry) => entry.name);
|
|
1687
|
+
if (mine.length === 0) return [];
|
|
1688
|
+
const removed = new Set<string>();
|
|
1689
|
+
for (const name of mine) {
|
|
1690
|
+
if (queue.remove(name)) removed.add(name);
|
|
1691
|
+
}
|
|
1692
|
+
const names: string[] = [];
|
|
1693
|
+
for (const item of queue.entries()) {
|
|
1694
|
+
// `entries()` is re-read below only to keep `seq` order for `removed`.
|
|
1695
|
+
void item;
|
|
1696
|
+
}
|
|
1697
|
+
for (const name of mine) {
|
|
1698
|
+
if (!removed.has(name)) continue;
|
|
1699
|
+
const child = this.children.get(name);
|
|
1700
|
+
if (child && child.state === "queued") {
|
|
1701
|
+
child.state = "retired";
|
|
1702
|
+
child.retiredAt = new Date(this.now()).toISOString();
|
|
1703
|
+
child.queuePosition = undefined;
|
|
1704
|
+
this.onChildUpdate(child);
|
|
1705
|
+
}
|
|
1706
|
+
const guard = this.launchGuards.get(name);
|
|
1707
|
+
this.launchGuards.delete(name);
|
|
1708
|
+
guard?.settle({ kind: "rejected", reason: "flushed", message: `launch queue flushed (${reason})` });
|
|
1709
|
+
this.notice(`launch queue flushed: ${name} (${reason})`);
|
|
1710
|
+
names.push(name);
|
|
1711
|
+
}
|
|
1712
|
+
return names;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
/** D9: read-only snapshot of the queue (shared across instances via the registry). */
|
|
1716
|
+
launchQueueEntries(): readonly AdmissionQueueEntry[] {
|
|
1717
|
+
return this.existingAdmissionQueue()?.entries() ?? [];
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
/** D8: is `name` still waiting in a live queue of this process? */
|
|
1721
|
+
private isQueuedHere(name: string): boolean {
|
|
1722
|
+
const queue = this.existingAdmissionQueue();
|
|
1723
|
+
return queue !== undefined && queue.positionOf(name) !== undefined;
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
/**
|
|
1727
|
+
* I-O3: `queued` records that need a spawned child.
|
|
1728
|
+
*
|
|
1729
|
+
* A record left over from a previous process can never become live again — the
|
|
1730
|
+
* queue is in memory. Say that instead of `unknown child`, so the operator
|
|
1731
|
+
* knows to re-dispatch. `retire()` is exempt: it may clean such an orphan up.
|
|
1732
|
+
*/
|
|
1733
|
+
private queuedStateMessage(name: string): string | undefined {
|
|
1734
|
+
const child = this.children.get(name);
|
|
1735
|
+
if (!child || child.state !== "queued") return undefined;
|
|
1736
|
+
return this.isQueuedHere(name) ? queuedNotStartedMessage(name) : launchQueueLostMessage(name);
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
/**
|
|
1740
|
+
* D8/A12: block until a queued child leaves the queue.
|
|
1741
|
+
*
|
|
1742
|
+
* Prefers the enqueue-time `settled` promise (same Orchestrator instance). A
|
|
1743
|
+
* control-action Orchestrator owns no guard, so it polls the record instead —
|
|
1744
|
+
* never `parseSessionFile` / `waitSettled` on the placeholder path (D8), which
|
|
1745
|
+
* would spin on a file that does not exist and report a fake result.
|
|
1746
|
+
*/
|
|
1747
|
+
private async waitForAdmission(
|
|
1748
|
+
child: ChildRecord,
|
|
1749
|
+
timeoutMs: number,
|
|
1750
|
+
signal?: AbortSignal,
|
|
1751
|
+
): Promise<AdmitOutcome> {
|
|
1752
|
+
const direct = this.launchGuards.get(child.name)?.settled;
|
|
1753
|
+
if (direct) {
|
|
1754
|
+
// The reservation has not started yet: keep the record truthful while we
|
|
1755
|
+
// wait, so a concurrent `status` reads the position, not `launching`.
|
|
1756
|
+
//
|
|
1757
|
+
// The bare await is NOT enough: `collect(name, { signal })` is a published
|
|
1758
|
+
// contract and every other branch of `collect` honours `signal`/deadline.
|
|
1759
|
+
// Admission may take the whole `launchQueueTimeoutMs` (15 min), so an abort
|
|
1760
|
+
// must be able to end this wait — otherwise `async:false` keeps the parent
|
|
1761
|
+
// turn blocked long after the caller gave up.
|
|
1762
|
+
return await this.waitForAdmissionOutcome(child, direct, timeoutMs, signal);
|
|
1763
|
+
}
|
|
1764
|
+
const deadline = this.now() + timeoutMs;
|
|
1765
|
+
for (;;) {
|
|
1766
|
+
if (signal?.aborted) {
|
|
1767
|
+
return {
|
|
1768
|
+
kind: "rejected",
|
|
1769
|
+
reason: "stopped",
|
|
1770
|
+
message: `aborted by parent while queued: '${child.name}'`,
|
|
1771
|
+
};
|
|
1772
|
+
}
|
|
1773
|
+
if (child.state !== "queued") {
|
|
1774
|
+
return { kind: "admitted", waitedMs: child.queueWaitedMs ?? 0 };
|
|
1775
|
+
}
|
|
1776
|
+
const remaining = deadline - this.now();
|
|
1777
|
+
if (remaining <= 0) {
|
|
1778
|
+
return {
|
|
1779
|
+
kind: "rejected",
|
|
1780
|
+
reason: "timeout",
|
|
1781
|
+
message: queueTimeoutMessage(child.name, this.now() - this.enqueuedAtFor(child.name)),
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
if (await this.sleepOrAbort(Math.min(POLL_INTERVAL_MS, remaining), signal)) continue;
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
/**
|
|
1789
|
+
* Race an enqueue-time `settled` promise against abort and the local deadline.
|
|
1790
|
+
*
|
|
1791
|
+
* `settled` resolves at most once and has no cleanup, so losing the race is
|
|
1792
|
+
* harmless: a later admission still settles it, and nothing here mutates the
|
|
1793
|
+
* queue. The loser is left pending on purpose — cancelling it would have to
|
|
1794
|
+
* dequeue the item, which is `stop`'s job, not `collect`'s.
|
|
1795
|
+
*/
|
|
1796
|
+
private async waitForAdmissionOutcome(
|
|
1797
|
+
child: ChildRecord,
|
|
1798
|
+
settled: Promise<AdmitOutcome>,
|
|
1799
|
+
timeoutMs: number,
|
|
1800
|
+
signal?: AbortSignal,
|
|
1801
|
+
): Promise<AdmitOutcome> {
|
|
1802
|
+
const aborted: AdmitOutcome = {
|
|
1803
|
+
kind: "rejected",
|
|
1804
|
+
reason: "stopped",
|
|
1805
|
+
message: `aborted by parent while queued: '${child.name}'`,
|
|
1806
|
+
};
|
|
1807
|
+
if (signal?.aborted) return aborted;
|
|
1808
|
+
const deadline = this.now() + timeoutMs;
|
|
1809
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1810
|
+
let onAbort: (() => void) | undefined;
|
|
1811
|
+
try {
|
|
1812
|
+
return await new Promise<AdmitOutcome>((resolve) => {
|
|
1813
|
+
const finish = (outcome: AdmitOutcome): void => {
|
|
1814
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1815
|
+
if (onAbort && signal) signal.removeEventListener("abort", onAbort);
|
|
1816
|
+
resolve(outcome);
|
|
1817
|
+
};
|
|
1818
|
+
void settled.then(finish, () =>
|
|
1819
|
+
finish({ kind: "rejected", reason: "flushed", message: `launch queue lost: '${child.name}'` }));
|
|
1820
|
+
timer = setTimeout(() => {
|
|
1821
|
+
finish({
|
|
1822
|
+
kind: "rejected",
|
|
1823
|
+
reason: "timeout",
|
|
1824
|
+
message: queueTimeoutMessage(child.name, this.now() - this.enqueuedAtFor(child.name)),
|
|
1825
|
+
});
|
|
1826
|
+
}, Math.max(1, deadline - this.now()));
|
|
1827
|
+
timer.unref?.();
|
|
1828
|
+
if (signal) {
|
|
1829
|
+
onAbort = () => finish(aborted);
|
|
1830
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1831
|
+
}
|
|
1832
|
+
});
|
|
1833
|
+
} finally {
|
|
1834
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1835
|
+
if (onAbort && signal) signal.removeEventListener("abort", onAbort);
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
/** Enqueue timestamp of a live queue item, for the timeout wording. */
|
|
1840
|
+
private enqueuedAtFor(name: string): number {
|
|
1841
|
+
const guard = this.launchGuards.get(name);
|
|
1842
|
+
if (guard) return guard.enqueuedAt;
|
|
1843
|
+
const waited = this.existingAdmissionQueue()?.waitedMs(name);
|
|
1844
|
+
return waited === undefined ? this.now() : this.now() - waited;
|
|
851
1845
|
}
|
|
852
1846
|
|
|
853
1847
|
private assertWithinBudgets(existingChild = false): void {
|
|
@@ -929,6 +1923,7 @@ export class Orchestrator {
|
|
|
929
1923
|
|
|
930
1924
|
async steer(name: string, text: string): Promise<void> {
|
|
931
1925
|
const child = this.requireChild(name);
|
|
1926
|
+
this.refuseQueuedControl(name);
|
|
932
1927
|
await this.supervisor.steer(name, text);
|
|
933
1928
|
this.emitProcess("node_steered", child.nodeId ?? name, { name, text });
|
|
934
1929
|
this.clearBlocked(child);
|
|
@@ -936,16 +1931,28 @@ export class Orchestrator {
|
|
|
936
1931
|
|
|
937
1932
|
async followUp(name: string, text: string): Promise<void> {
|
|
938
1933
|
const child = this.requireChild(name);
|
|
1934
|
+
this.refuseQueuedControl(name);
|
|
939
1935
|
await this.supervisor.followUp(name, text);
|
|
940
1936
|
this.clearBlocked(child);
|
|
941
1937
|
}
|
|
942
1938
|
|
|
943
1939
|
async abort(name: string): Promise<void> {
|
|
944
1940
|
const child = this.requireChild(name);
|
|
1941
|
+
this.refuseQueuedControl(name);
|
|
945
1942
|
await this.supervisor.abort(name);
|
|
946
1943
|
this.emitProcess("node_aborted", child.nodeId ?? name, { name });
|
|
947
1944
|
}
|
|
948
1945
|
|
|
1946
|
+
/**
|
|
1947
|
+
* I-O3: `steer`/`followUp`/`abort` need a spawned child. A queued one has no
|
|
1948
|
+
* process and no session, so say why instead of letting the supervisor answer
|
|
1949
|
+
* `unknown child` (or, worse, address a placeholder path).
|
|
1950
|
+
*/
|
|
1951
|
+
private refuseQueuedControl(name: string): void {
|
|
1952
|
+
const message = this.queuedStateMessage(name);
|
|
1953
|
+
if (message) throw new SubagentError(message, ErrorCodes.BUDGET_EXCEEDED);
|
|
1954
|
+
}
|
|
1955
|
+
|
|
949
1956
|
private requireChild(name: string): ChildRecord {
|
|
950
1957
|
const child = this.children.get(name);
|
|
951
1958
|
if (!child) throw new SubagentError(`unknown child: ${name}`, ErrorCodes.NOT_FOUND);
|
|
@@ -980,11 +1987,32 @@ export class Orchestrator {
|
|
|
980
1987
|
}
|
|
981
1988
|
|
|
982
1989
|
async collect(name: string, opts: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise<CollectResult> {
|
|
983
|
-
|
|
1990
|
+
let child = this.requireChild(name);
|
|
1991
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULTS.turnTimeoutMs;
|
|
1992
|
+
// D8/A12: a queued child has no session yet. Block until it is admitted
|
|
1993
|
+
// (then fall through to the normal path) or until it leaves the queue
|
|
1994
|
+
// without starting (timeout / stop / flush) — never `parseSessionFile` or
|
|
1995
|
+
// `waitSettled` on the placeholder path, which would spin on a file that
|
|
1996
|
+
// does not exist and report a fake outcome.
|
|
1997
|
+
if (child.state === "queued") {
|
|
1998
|
+
// I-O3: a `queued` record whose item is not in this process's queue is an
|
|
1999
|
+
// orphan from a previous process — it can never start, so say that rather
|
|
2000
|
+
// than waiting for a timeout that would hide the real reason.
|
|
2001
|
+
if (!this.isQueuedHere(name)) throw new SubagentError(launchQueueLostMessage(name), ErrorCodes.BUDGET_EXCEEDED);
|
|
2002
|
+
const admission = await this.waitForAdmission(child, timeoutMs, opts.signal);
|
|
2003
|
+
if (admission.kind === "rejected") {
|
|
2004
|
+
throw new SubagentError(admission.message, ErrorCodes.BUDGET_EXCEEDED);
|
|
2005
|
+
}
|
|
2006
|
+
// Admission REPLACES the placeholder record (`admitQueued` deletes it and
|
|
2007
|
+
// the real launch registers a new one). Reading further through the old
|
|
2008
|
+
// object would use its placeholder `sessionFile` — which for the legion
|
|
2009
|
+
// path is a DIFFERENT file than the spawned child's run-dir transcript, so
|
|
2010
|
+
// the outcome would be derived from a file nobody ever wrote.
|
|
2011
|
+
child = this.requireChild(name);
|
|
2012
|
+
}
|
|
984
2013
|
const abortedSnapshot = (): CollectResult => this.interruptedCollect(child, parseSessionFile(child.sessionFile));
|
|
985
2014
|
if (opts.signal?.aborted) return abortedSnapshot();
|
|
986
2015
|
if (this.pendingDialogs(name).length) return this.blockedResult(child);
|
|
987
|
-
const timeoutMs = opts.timeoutMs ?? DEFAULTS.turnTimeoutMs;
|
|
988
2016
|
let deadline = this.now() + timeoutMs;
|
|
989
2017
|
const initial = parseSessionFile(child.sessionFile);
|
|
990
2018
|
// A connection blip writes stopReason "error" before the client retries.
|
|
@@ -1004,10 +2032,19 @@ export class Orchestrator {
|
|
|
1004
2032
|
}
|
|
1005
2033
|
if (wait === "blocked") return this.blockedResult(child);
|
|
1006
2034
|
let parsed = parseSessionFile(child.sessionFile);
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
2035
|
+
// #232: no settle-quiet wait here.
|
|
2036
|
+
//
|
|
2037
|
+
// `wait === "settled"` already means the supervisor observed the turn end
|
|
2038
|
+
// (`waitSettled` resolves on the settle event, not on a stat poll), and
|
|
2039
|
+
// callers only reach `collect()` after `cachedCollect()` found the child
|
|
2040
|
+
// finished. Re-polling `statSync(sessionFile).size` for `settleQuietMs`
|
|
2041
|
+
// (2500ms) after that confirmed terminal state produced no information: an
|
|
2042
|
+
// append-only transcript cannot grow once the turn is complete. It cost
|
|
2043
|
+
// ~2.0s of dead time per successful collect and scaled with fan-out width.
|
|
2044
|
+
//
|
|
2045
|
+
// The partial-write hazard the wait was guarding against is covered by the
|
|
2046
|
+
// re-parse below: `parseSessionFile` leaves a trailing partial line out of
|
|
2047
|
+
// its committed cache (#36), so a mid-write tail is never read as a verdict.
|
|
1011
2048
|
if (opts.signal?.aborted) return abortedSnapshot();
|
|
1012
2049
|
parsed = parseSessionFile(child.sessionFile);
|
|
1013
2050
|
// #227 D5: a child whose process is gone cannot produce another message.
|
|
@@ -1065,22 +2102,63 @@ export class Orchestrator {
|
|
|
1065
2102
|
*
|
|
1066
2103
|
* Returns once the session moves past that blip, the process exits, a UI
|
|
1067
2104
|
* approval arrives, the parent aborts, or the cumulative deadline passes.
|
|
2105
|
+
*
|
|
2106
|
+
* #232: the wait is a poll over an append-only file, so it watches the one
|
|
2107
|
+
* cheap signal that changes first — the artifact size — and only pays for a
|
|
2108
|
+
* `parseSessionFile` when that size moved. Unchanged artifacts reuse the
|
|
2109
|
+
* previous parse. The poll interval also backs off geometrically
|
|
2110
|
+
* (`POLL_INTERVAL_MS` → `TRANSIENT_POLL_CAP_MS`), so the default 900s deadline
|
|
2111
|
+
* costs 229 stats (the seed stat plus 228 sleeps, one stat each) and a single
|
|
2112
|
+
* parse instead of 1800 full parses. The backoff never shortens the wait:
|
|
2113
|
+
* sleeps are clamped to the remaining budget and every iteration sleeps
|
|
2114
|
+
* strictly > 0ms, so the loop cannot busy-spin.
|
|
2115
|
+
*
|
|
2116
|
+
* Accepted tradeoff, stated plainly because it is a real regression in
|
|
2117
|
+
* observation latency: **this loop is the only detector for both the blip
|
|
2118
|
+
* clearing and the child exiting while it holds** (per D2 the orchestrator
|
|
2119
|
+
* does not subscribe to `onEvent`, and `waitSettled` has already returned
|
|
2120
|
+
* `timeout` by the time we get here). So during the hold, an exit or a settle
|
|
2121
|
+
* appended to the session is observed within one capped interval —
|
|
2122
|
+
* up to `TRANSIENT_POLL_CAP_MS` (4s) — where the pre-#232 flat 500ms cadence
|
|
2123
|
+
* observed it within 500ms. The same applies to a UI approval raising
|
|
2124
|
+
* `pendingDialogs`. Nothing is dropped and no return value changes
|
|
2125
|
+
* (`TRANSIENT_CONNECTION_REASON`, `"done"`); only the latency with which the
|
|
2126
|
+
* loop notices grows, and it is bounded by the cap rather than by the
|
|
2127
|
+
* deadline. This is the documented reason the cap is 4s and not larger.
|
|
2128
|
+
*
|
|
2129
|
+
* The seed below stats *before* parsing on purpose: an append landing between
|
|
2130
|
+
* the two calls then makes the next `size !== parsedSize` comparison true
|
|
2131
|
+
* (over-parse, safe). Parsing first would fold that append into `parsed` while
|
|
2132
|
+
* `parsedSize` also saw it, so the loop would consider the file unchanged and
|
|
2133
|
+
* never notice a recovery until the deadline expired.
|
|
1068
2134
|
*/
|
|
1069
2135
|
private async waitOutTransientConnection(
|
|
1070
2136
|
child: ChildRecord,
|
|
1071
2137
|
deadline: number,
|
|
1072
2138
|
signal?: AbortSignal,
|
|
1073
2139
|
): Promise<"aborted" | "blocked" | "done"> {
|
|
1074
|
-
|
|
1075
|
-
|
|
2140
|
+
let parsedSize = artifactSize(child.sessionFile);
|
|
2141
|
+
let parsed = parseSessionFile(child.sessionFile);
|
|
2142
|
+
let interval = POLL_INTERVAL_MS;
|
|
2143
|
+
for (;;) {
|
|
1076
2144
|
if (signal?.aborted) return "aborted";
|
|
1077
2145
|
if (this.pendingDialogs(child.name).length) return "blocked";
|
|
1078
2146
|
if (!this.supervisor.isAlive(child.name)) return "done";
|
|
1079
|
-
if (!isTransientConnectionFailure(deriveOutcome(
|
|
1080
|
-
|
|
1081
|
-
if (
|
|
2147
|
+
if (!isTransientConnectionFailure(deriveOutcome(parsed))) return "done";
|
|
2148
|
+
const remaining = deadline - this.now();
|
|
2149
|
+
if (remaining <= 0) return "done";
|
|
2150
|
+
const sleptFrom = this.now();
|
|
2151
|
+
if (await this.sleepOrAbort(Math.min(interval, remaining), signal)) return "aborted";
|
|
2152
|
+
// A sleep dep that never advances the clock would make this loop a busy
|
|
2153
|
+
// spin; stop holding instead of hammering the filesystem.
|
|
2154
|
+
if (this.now() <= sleptFrom) return "done";
|
|
2155
|
+
const size = artifactSize(child.sessionFile);
|
|
2156
|
+
if (size !== parsedSize) {
|
|
2157
|
+
parsed = parseSessionFile(child.sessionFile);
|
|
2158
|
+
parsedSize = size;
|
|
2159
|
+
}
|
|
2160
|
+
interval = Math.min(interval * 2, TRANSIENT_POLL_CAP_MS);
|
|
1082
2161
|
}
|
|
1083
|
-
return "done";
|
|
1084
2162
|
}
|
|
1085
2163
|
|
|
1086
2164
|
private interruptedCollect(child: ChildRecord, parsed: ReturnType<typeof parseSessionFile>): CollectResult {
|
|
@@ -1224,11 +2302,24 @@ export class Orchestrator {
|
|
|
1224
2302
|
} catch { return undefined; }
|
|
1225
2303
|
}
|
|
1226
2304
|
|
|
2305
|
+
/**
|
|
2306
|
+
* Did the session artifact grow inside one `POLL_INTERVAL_MS` window?
|
|
2307
|
+
*
|
|
2308
|
+
* #232: this used to sleep the whole window and stat twice. The window is
|
|
2309
|
+
* unchanged (a grace extension is still only granted to an artifact that is
|
|
2310
|
+
* genuinely growing), but it is now sampled at `ARTIFACT_GROWTH_POLL_MS` and
|
|
2311
|
+
* returns on the first observed growth. The sleeps stay on the injected
|
|
2312
|
+
* `now`/`sleep` deps, so `test/perf/collect-latency.test.ts` can drive them
|
|
2313
|
+
* on its virtual clock.
|
|
2314
|
+
*/
|
|
1227
2315
|
private async collectArtifactGrowing(child: ChildRecord, signal?: AbortSignal): Promise<boolean> {
|
|
1228
2316
|
const before = artifactSize(child.sessionFile);
|
|
1229
2317
|
if (before <= 0 || signal?.aborted) return false;
|
|
1230
|
-
|
|
1231
|
-
|
|
2318
|
+
for (let waited = 0; waited < POLL_INTERVAL_MS; waited += ARTIFACT_GROWTH_POLL_MS) {
|
|
2319
|
+
if (await this.sleepOrAbort(Math.min(ARTIFACT_GROWTH_POLL_MS, POLL_INTERVAL_MS - waited), signal)) return false;
|
|
2320
|
+
if (artifactSize(child.sessionFile) > before) return true;
|
|
2321
|
+
}
|
|
2322
|
+
return false;
|
|
1232
2323
|
}
|
|
1233
2324
|
|
|
1234
2325
|
private async sleepOrAbort(ms: number, signal?: AbortSignal): Promise<boolean> {
|
|
@@ -1260,20 +2351,6 @@ export class Orchestrator {
|
|
|
1260
2351
|
});
|
|
1261
2352
|
}
|
|
1262
2353
|
|
|
1263
|
-
private async waitForQuiet(sessionFile: string, deadline: number, signal?: AbortSignal): Promise<void> {
|
|
1264
|
-
let previous = -1;
|
|
1265
|
-
let quietSince = this.now();
|
|
1266
|
-
const maxPolls = Math.max(1, Math.ceil(DEFAULTS.turnTimeoutMs / POLL_INTERVAL_MS));
|
|
1267
|
-
for (let poll = 0; poll < maxPolls; poll += 1) {
|
|
1268
|
-
if (signal?.aborted) return;
|
|
1269
|
-
const size = artifactSize(sessionFile);
|
|
1270
|
-
if (size !== previous) { previous = size; quietSince = this.now(); }
|
|
1271
|
-
else if (this.now() - quietSince >= DEFAULTS.settleQuietMs) return;
|
|
1272
|
-
if (this.now() >= deadline) return;
|
|
1273
|
-
if (await this.sleepOrAbort(POLL_INTERVAL_MS, signal)) return;
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
2354
|
cachedCollect(name: string): CollectResult | undefined {
|
|
1278
2355
|
const child = this.children.get(name);
|
|
1279
2356
|
if (!child?.execution || !canUseCachedCollect(child.state)) return undefined;
|
|
@@ -1294,6 +2371,27 @@ export class Orchestrator {
|
|
|
1294
2371
|
const child = this.requireChild(name);
|
|
1295
2372
|
if (opts?.cause === "session-shutdown") this.parentShutdownRetire.add(name);
|
|
1296
2373
|
if (child.state === "retired") return child;
|
|
2374
|
+
if (child.state === "queued") {
|
|
2375
|
+
// D7: stopping a queued item is a pure dequeue — no spawn, no ledger row.
|
|
2376
|
+
// I-O3: an orphaned `queued` record from a previous process is retired the
|
|
2377
|
+
// same way; there is no queue entry left to remove.
|
|
2378
|
+
const queuedAt = this.enqueuedAtFor(name);
|
|
2379
|
+
this.existingAdmissionQueue()?.remove(name);
|
|
2380
|
+
this.abandonQueued(
|
|
2381
|
+
name,
|
|
2382
|
+
"stopped",
|
|
2383
|
+
`launch queue stopped ('${name}') after waiting ${Math.round((this.now() - queuedAt) / 1000)}s`,
|
|
2384
|
+
);
|
|
2385
|
+
child.state = "retired";
|
|
2386
|
+
child.retiredAt ??= new Date(this.now()).toISOString();
|
|
2387
|
+
child.queuePosition = undefined;
|
|
2388
|
+
this.onChildUpdate(child);
|
|
2389
|
+
this.emitProcess("node_retired", child.nodeId ?? name, { name });
|
|
2390
|
+
this.notice(`launch queue stopped: ${name}`);
|
|
2391
|
+
// D2-d-3: a dequeue can also free the *queue slot* the next item needed.
|
|
2392
|
+
this.schedulePump();
|
|
2393
|
+
return child;
|
|
2394
|
+
}
|
|
1297
2395
|
const sessionExists = fs.existsSync(child.sessionFile);
|
|
1298
2396
|
const parsed = sessionExists ? parseSessionFile(child.sessionFile) : emptyParsedSession();
|
|
1299
2397
|
if (!child.execution && sessionExists) child.execution = deriveOutcome(parsed);
|
|
@@ -1322,6 +2420,11 @@ export class Orchestrator {
|
|
|
1322
2420
|
child.retiredAt = new Date(this.now()).toISOString();
|
|
1323
2421
|
this.emitProcess("node_retired", child.nodeId ?? name, { name });
|
|
1324
2422
|
this.onChildUpdate(child);
|
|
2423
|
+
// D2-d-3: retiring a child frees a slot. The in-process backend also emits
|
|
2424
|
+
// `onSlotReleased`, but that hook is wired by the supervisor's owner and the
|
|
2425
|
+
// rpc path emits nothing at all — so the orchestrator wakes its own pump on
|
|
2426
|
+
// the edge it just observed. `pumping` makes the duplicate wake harmless.
|
|
2427
|
+
this.schedulePump();
|
|
1325
2428
|
return child;
|
|
1326
2429
|
}
|
|
1327
2430
|
|