pi-subagents 0.47.1 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +2 -0
- package/docs/agents.md +1 -0
- package/docs/configuration.md +74 -7
- package/docs/missions.md +4 -2
- package/docs/observability.md +27 -3
- package/docs/tool-reference.md +1 -1
- package/package.json +1 -1
- package/src/agents/agents.ts +36 -15
- package/src/api/preflight.ts +1 -1
- package/src/extension/config.ts +26 -0
- package/src/extension/doctor.ts +40 -0
- package/src/extension/index.ts +39 -8
- package/src/extension/public-execution.ts +35 -4
- package/src/extension/rpc.ts +3 -9
- package/src/extension/schemas.ts +10 -9
- package/src/extension/tool-description.ts +10 -8
- package/src/inspectors/herdr/actions.ts +11 -2
- package/src/inspectors/herdr/inspector-runner.ts +16 -3
- package/src/intercom/intercom-bridge.ts +7 -2
- package/src/missions/lifecycle.ts +4 -7
- package/src/missions/store.ts +12 -7
- package/src/missions/workflow-state.ts +6 -2
- package/src/runs/background/active-async-capacity.ts +431 -0
- package/src/runs/background/active-run-index.ts +9 -5
- package/src/runs/background/async-execution.ts +141 -65
- package/src/runs/background/async-job-tracker.ts +4 -0
- package/src/runs/background/async-resume.ts +18 -2
- package/src/runs/background/async-status.ts +11 -5
- package/src/runs/background/chain-append.ts +33 -15
- package/src/runs/background/fleet-view.ts +18 -5
- package/src/runs/background/owned-process-tree.ts +104 -0
- package/src/runs/background/process-terminal.ts +17 -3
- package/src/runs/background/resume-guidance.ts +27 -7
- package/src/runs/background/retained-children.ts +14 -6
- package/src/runs/background/run-status.ts +101 -5
- package/src/runs/background/stale-run-reconciler.ts +3 -3
- package/src/runs/background/subagent-runner.ts +60 -32
- package/src/runs/foreground/chain-execution.ts +37 -2
- package/src/runs/foreground/execution.ts +90 -19
- package/src/runs/foreground/foreground-control.ts +12 -0
- package/src/runs/foreground/prompt-audit.ts +172 -0
- package/src/runs/foreground/subagent-executor.ts +829 -203
- package/src/runs/shared/acceptance.ts +13 -4
- package/src/runs/shared/completion-guard.ts +17 -1
- package/src/runs/shared/llm-intent-arbiter.ts +302 -0
- package/src/runs/shared/parallel-utils.ts +2 -0
- package/src/runs/shared/pi-args.ts +44 -1
- package/src/runs/shared/run-fanout-budget.ts +280 -0
- package/src/runs/shared/single-output.ts +4 -2
- package/src/runs/shared/task-intent.ts +19 -3
- package/src/runs/shared/worktree.ts +17 -5
- package/src/shared/agent-stream-options.ts +5 -0
- package/src/shared/artifacts.ts +2 -6
- package/src/shared/display-text.ts +50 -0
- package/src/shared/node-executable.ts +21 -0
- package/src/shared/types.ts +115 -4
- package/src/shared/utils.ts +3 -1
- package/src/tui/fleet-status.ts +7 -5
- package/src/tui/fleet-transcript.ts +1 -48
- package/src/tui/fleet.ts +228 -13
- package/src/tui/render.ts +86 -39
- package/src/watchdog/permission-arbiter.ts +2 -1
- package/src/watchdog/review.ts +4 -3
- package/src/workflows/chat-progress.ts +8 -2
- package/src/workflows/scripted-workflow.ts +66 -10
package/src/missions/store.ts
CHANGED
|
@@ -2,7 +2,6 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import * as os from "node:os";
|
|
4
4
|
import * as path from "node:path";
|
|
5
|
-
import { getProjectSubagentsDir } from "../shared/artifacts.ts";
|
|
6
5
|
import { writePrivateAtomicJson } from "../shared/atomic-json.ts";
|
|
7
6
|
import { getAgentDir } from "../shared/utils.ts";
|
|
8
7
|
import {
|
|
@@ -259,6 +258,11 @@ function expandConfiguredPath(value: string, projectRoot: string): string {
|
|
|
259
258
|
return path.isAbsolute(expanded) ? path.normalize(expanded) : path.resolve(projectRoot, expanded);
|
|
260
259
|
}
|
|
261
260
|
|
|
261
|
+
function projectMissionDirectory(agentDir: string, projectRoot: string): string {
|
|
262
|
+
const projectKey = createHash("sha256").update(projectRoot).digest("hex");
|
|
263
|
+
return path.join(agentDir, "missions", "projects", projectKey);
|
|
264
|
+
}
|
|
265
|
+
|
|
262
266
|
export function validateMissionStoreConfig(value: unknown, label = "config.missions"): MissionStoreConfig | undefined {
|
|
263
267
|
if (value === undefined) return undefined;
|
|
264
268
|
const input = asObject(value, label);
|
|
@@ -289,12 +293,13 @@ export function resolveMissionStoreLocation(input: {
|
|
|
289
293
|
agentDir?: string;
|
|
290
294
|
}): MissionStoreLocation {
|
|
291
295
|
const projectRoot = path.resolve(input.projectRoot);
|
|
296
|
+
const agentDir = input.agentDir ?? getAgentDir();
|
|
292
297
|
const missionDir = input.config?.directory
|
|
293
298
|
? expandConfiguredPath(input.config.directory, projectRoot)
|
|
294
|
-
:
|
|
299
|
+
: projectMissionDirectory(agentDir, projectRoot);
|
|
295
300
|
const globalIndexDir = input.config?.globalIndexDir
|
|
296
301
|
? expandConfiguredPath(input.config.globalIndexDir, projectRoot)
|
|
297
|
-
: path.join(
|
|
302
|
+
: path.join(agentDir, "missions", "index");
|
|
298
303
|
return {
|
|
299
304
|
projectRoot,
|
|
300
305
|
missionDir,
|
|
@@ -396,11 +401,11 @@ export class MissionNotFoundError extends Error {
|
|
|
396
401
|
readonly missionId: string;
|
|
397
402
|
readonly missionDir: string;
|
|
398
403
|
|
|
399
|
-
constructor(missionId: string,
|
|
400
|
-
super(`Mission '${missionId}' was not found in ${missionDir}
|
|
404
|
+
constructor(missionId: string, location: MissionStoreLocation) {
|
|
405
|
+
super(`Mission '${missionId}' was not found in mission directory '${location.missionDir}' for project root '${location.projectRoot}'. If it was created in another worktree, run the request from that worktree.`);
|
|
401
406
|
this.name = "MissionNotFoundError";
|
|
402
407
|
this.missionId = missionId;
|
|
403
|
-
this.missionDir = missionDir;
|
|
408
|
+
this.missionDir = location.missionDir;
|
|
404
409
|
}
|
|
405
410
|
}
|
|
406
411
|
|
|
@@ -410,7 +415,7 @@ export function readMission(location: MissionStoreLocation, missionId: string):
|
|
|
410
415
|
try {
|
|
411
416
|
raw = fs.readFileSync(filePath, "utf-8");
|
|
412
417
|
} catch (error) {
|
|
413
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") throw new MissionNotFoundError(missionId, location
|
|
418
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") throw new MissionNotFoundError(missionId, location);
|
|
414
419
|
throw error;
|
|
415
420
|
}
|
|
416
421
|
try {
|
|
@@ -95,8 +95,12 @@ function stateLockIsStale(lockPath: string, now = Date.now()): boolean {
|
|
|
95
95
|
const owner = readStateLockOwner(lockPath);
|
|
96
96
|
if (owner) {
|
|
97
97
|
if (!isProcessAlive(owner.pid)) return true;
|
|
98
|
-
|
|
99
|
-
|
|
98
|
+
if (owner.processKey) {
|
|
99
|
+
const currentProcessKey = owner.pid === process.pid ? CURRENT_PROCESS_KEY : processStartKey(owner.pid);
|
|
100
|
+
if (currentProcessKey) return owner.processKey !== currentProcessKey;
|
|
101
|
+
if (owner.pid === process.pid) return true;
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
100
104
|
}
|
|
101
105
|
try {
|
|
102
106
|
return now - fs.statSync(lockPath).mtimeMs > STATE_LOCK_STALE_MS;
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { writePrivateAtomicJson } from "../../shared/atomic-json.ts";
|
|
5
|
+
import { TEMP_ROOT_DIR, type ActiveAsyncCapacitySnapshot, type AsyncStatus } from "../../shared/types.ts";
|
|
6
|
+
import { readStatus } from "../../shared/utils.ts";
|
|
7
|
+
import { readProcessTerminal } from "./process-terminal.ts";
|
|
8
|
+
|
|
9
|
+
export const ACTIVE_ASYNC_CAPACITY_DIR = path.join(TEMP_ROOT_DIR, "session-active-async-capacity");
|
|
10
|
+
|
|
11
|
+
export interface ActiveAsyncCapacityOwnerV1 {
|
|
12
|
+
version: 1;
|
|
13
|
+
reservationToken: string;
|
|
14
|
+
ownerSessionId: string;
|
|
15
|
+
ownerSessionKey: string;
|
|
16
|
+
slot: number;
|
|
17
|
+
runId: string;
|
|
18
|
+
sourceRunId?: string;
|
|
19
|
+
generation: number;
|
|
20
|
+
kind: "runner" | "workflow";
|
|
21
|
+
asyncDir: string;
|
|
22
|
+
reservedAt: number;
|
|
23
|
+
runnerProcessInstanceId?: string;
|
|
24
|
+
runnerStartedAt?: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface ActiveAsyncCapacityHandle {
|
|
28
|
+
readonly owner: ActiveAsyncCapacityOwnerV1;
|
|
29
|
+
markStarted(runnerProcessInstanceId: string): void;
|
|
30
|
+
markWorkflowStarted(): void;
|
|
31
|
+
rollback(): boolean;
|
|
32
|
+
reconcile(liveWorkflowRunIds?: ReadonlySet<string>): ActiveAsyncCapacitySnapshot;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface CapacityOptions {
|
|
36
|
+
rootDir?: string;
|
|
37
|
+
now?: () => number;
|
|
38
|
+
token?: () => string;
|
|
39
|
+
afterSlotRename?: (releasedDir: string) => void;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type ActiveAsyncCapacityReleaseVerdict =
|
|
43
|
+
| { state: "releasable"; reason: string }
|
|
44
|
+
| { state: "retained"; reason: string }
|
|
45
|
+
| { state: "not-owned"; reason: string };
|
|
46
|
+
|
|
47
|
+
export interface ActiveAsyncCapacityInspection {
|
|
48
|
+
owner?: ActiveAsyncCapacityOwnerV1;
|
|
49
|
+
relation: "current" | "source" | "none";
|
|
50
|
+
slotDir?: string;
|
|
51
|
+
release: ActiveAsyncCapacityReleaseVerdict;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export class ActiveAsyncCapacityError extends Error {
|
|
55
|
+
readonly snapshot: ActiveAsyncCapacitySnapshot;
|
|
56
|
+
|
|
57
|
+
constructor(snapshot: ActiveAsyncCapacitySnapshot) {
|
|
58
|
+
super(`Active async run capacity exhausted: ${snapshot.used}/${snapshot.limit} used.`);
|
|
59
|
+
this.name = "ActiveAsyncCapacityError";
|
|
60
|
+
this.snapshot = snapshot;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function resolveMaxActiveAsyncRunsPerSession(value: unknown): number | undefined {
|
|
65
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) return undefined;
|
|
66
|
+
return value === 0 ? undefined : value;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function activeAsyncCapacitySessionKey(sessionId: string): string {
|
|
70
|
+
return createHash("sha256").update(sessionId).digest("hex");
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function sessionDir(sessionId: string, rootDir: string): string {
|
|
74
|
+
return path.join(rootDir, activeAsyncCapacitySessionKey(sessionId));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function slotDir(poolDir: string, slot: number): string {
|
|
78
|
+
return path.join(poolDir, `slot-${slot}`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function parseOwner(value: unknown): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
82
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
83
|
+
const owner = value as Partial<ActiveAsyncCapacityOwnerV1>;
|
|
84
|
+
if (owner.version !== 1
|
|
85
|
+
|| typeof owner.reservationToken !== "string" || !owner.reservationToken
|
|
86
|
+
|| typeof owner.ownerSessionId !== "string" || !owner.ownerSessionId
|
|
87
|
+
|| typeof owner.ownerSessionKey !== "string" || !owner.ownerSessionKey
|
|
88
|
+
|| typeof owner.slot !== "number" || !Number.isInteger(owner.slot) || owner.slot < 0
|
|
89
|
+
|| typeof owner.runId !== "string" || !owner.runId
|
|
90
|
+
|| typeof owner.generation !== "number" || !Number.isInteger(owner.generation) || owner.generation < 0
|
|
91
|
+
|| (owner.kind !== "runner" && owner.kind !== "workflow")
|
|
92
|
+
|| typeof owner.asyncDir !== "string" || !owner.asyncDir
|
|
93
|
+
|| typeof owner.reservedAt !== "number" || !Number.isFinite(owner.reservedAt)) return undefined;
|
|
94
|
+
if (owner.sourceRunId !== undefined && typeof owner.sourceRunId !== "string") return undefined;
|
|
95
|
+
if (owner.runnerProcessInstanceId !== undefined && (typeof owner.runnerProcessInstanceId !== "string" || !owner.runnerProcessInstanceId)) return undefined;
|
|
96
|
+
if (owner.runnerStartedAt !== undefined && (typeof owner.runnerStartedAt !== "number" || !Number.isFinite(owner.runnerStartedAt))) return undefined;
|
|
97
|
+
return owner as ActiveAsyncCapacityOwnerV1;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function readOwner(dir: string): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
101
|
+
try {
|
|
102
|
+
return parseOwner(JSON.parse(fs.readFileSync(path.join(dir, "owner.json"), "utf-8")));
|
|
103
|
+
} catch {
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function occupiedSlots(poolDir: string): string[] {
|
|
109
|
+
try {
|
|
110
|
+
return fs.readdirSync(poolDir, { withFileTypes: true })
|
|
111
|
+
.filter((entry) => entry.isDirectory() && /^slot-\d+$/.test(entry.name))
|
|
112
|
+
.map((entry) => path.join(poolDir, entry.name));
|
|
113
|
+
} catch (error) {
|
|
114
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function snapshotFor(sessionId: string, limit: number | undefined, rootDir: string): ActiveAsyncCapacitySnapshot {
|
|
120
|
+
return { used: occupiedSlots(sessionDir(sessionId, rootDir)).length, limit: limit ?? 0 };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function matchingOwner(dir: string, expected: ActiveAsyncCapacityOwnerV1): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
124
|
+
const owner = readOwner(dir);
|
|
125
|
+
return owner?.reservationToken === expected.reservationToken
|
|
126
|
+
&& owner.runId === expected.runId
|
|
127
|
+
&& owner.generation === expected.generation
|
|
128
|
+
? owner
|
|
129
|
+
: undefined;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function withSlotClaim<T>(dir: string, operation: () => T): { acquired: true; value: T } | { acquired: false } {
|
|
133
|
+
const claimPath = path.join(dir, "capacity.claim");
|
|
134
|
+
const claimToken = randomUUID();
|
|
135
|
+
let claim: number | undefined;
|
|
136
|
+
try {
|
|
137
|
+
claim = fs.openSync(claimPath, "wx", 0o600);
|
|
138
|
+
fs.writeFileSync(claim, claimToken, "utf-8");
|
|
139
|
+
fs.closeSync(claim);
|
|
140
|
+
claim = undefined;
|
|
141
|
+
} catch (error) {
|
|
142
|
+
if (claim !== undefined) fs.closeSync(claim);
|
|
143
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST" || (error as NodeJS.ErrnoException).code === "ENOENT") return { acquired: false };
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
return { acquired: true, value: operation() };
|
|
148
|
+
} finally {
|
|
149
|
+
try {
|
|
150
|
+
if (fs.readFileSync(claimPath, "utf-8") === claimToken) fs.rmSync(claimPath, { force: true });
|
|
151
|
+
} catch (error) {
|
|
152
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function removeOwnedSlot(dir: string, expected: ActiveAsyncCapacityOwnerV1, options: CapacityOptions, requireUnstarted = false): boolean {
|
|
158
|
+
if (requireUnstarted && (expected.runnerProcessInstanceId || expected.runnerStartedAt)) return false;
|
|
159
|
+
const claimed = withSlotClaim(dir, () => {
|
|
160
|
+
const current = matchingOwner(dir, expected);
|
|
161
|
+
if (!current || (requireUnstarted && (current.runnerProcessInstanceId || current.runnerStartedAt))) return false;
|
|
162
|
+
const releasedDir = path.join(path.dirname(dir), `.${path.basename(dir)}.released-${randomUUID()}`);
|
|
163
|
+
fs.renameSync(dir, releasedDir);
|
|
164
|
+
options.afterSlotRename?.(releasedDir);
|
|
165
|
+
fs.rmSync(releasedDir, { recursive: true, force: true });
|
|
166
|
+
return true;
|
|
167
|
+
});
|
|
168
|
+
return claimed.acquired && claimed.value;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function terminalState(state: AsyncStatus["state"]): boolean {
|
|
172
|
+
return state !== "queued" && state !== "running" && state !== "paused";
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function runnerReleaseVerdict(owner: ActiveAsyncCapacityOwnerV1, status: AsyncStatus | null): ActiveAsyncCapacityReleaseVerdict {
|
|
176
|
+
if (!status) return { state: "retained", reason: "status file is missing or unreadable" };
|
|
177
|
+
if (!owner.runnerProcessInstanceId) return { state: "retained", reason: "runner process identity has not been recorded" };
|
|
178
|
+
if (status.sessionId !== owner.ownerSessionId) return { state: "retained", reason: `status session ${status.sessionId ?? "unknown"} does not match owner session ${owner.ownerSessionId}` };
|
|
179
|
+
if (status.runId !== owner.runId) return { state: "retained", reason: `status run ${status.runId} does not match owner run ${owner.runId}` };
|
|
180
|
+
if (!terminalState(status.state)) return { state: "retained", reason: `run is still ${status.state}` };
|
|
181
|
+
if (status.processTerminal?.state === "not-started"
|
|
182
|
+
&& status.processTerminal.runId === owner.runId
|
|
183
|
+
&& status.processTerminal.runnerProcessInstanceId === owner.runnerProcessInstanceId
|
|
184
|
+
&& typeof status.error === "string"
|
|
185
|
+
&& status.error) return { state: "releasable", reason: "run failed before child startup completed" };
|
|
186
|
+
const proof = readProcessTerminal(owner.asyncDir, {
|
|
187
|
+
runId: owner.runId,
|
|
188
|
+
runnerProcessInstanceId: owner.runnerProcessInstanceId,
|
|
189
|
+
});
|
|
190
|
+
return proof?.state === "observed"
|
|
191
|
+
&& proof.runId === owner.runId
|
|
192
|
+
&& proof.runnerProcessInstanceId === owner.runnerProcessInstanceId
|
|
193
|
+
? { state: "releasable", reason: "matching observed process-terminal proof is present" }
|
|
194
|
+
: { state: "retained", reason: `process-terminal proof is ${proof?.state ?? "missing"}` };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function workflowReleaseVerdict(owner: ActiveAsyncCapacityOwnerV1, status: AsyncStatus | null, liveWorkflowRunIds: ReadonlySet<string>): ActiveAsyncCapacityReleaseVerdict {
|
|
198
|
+
if (!status) return { state: "retained", reason: "status file is missing or unreadable" };
|
|
199
|
+
if (status.sessionId !== owner.ownerSessionId) return { state: "retained", reason: `status session ${status.sessionId ?? "unknown"} does not match owner session ${owner.ownerSessionId}` };
|
|
200
|
+
if (status.runId !== owner.runId) return { state: "retained", reason: `status run ${status.runId} does not match owner run ${owner.runId}` };
|
|
201
|
+
if (status.mode !== "workflow") return { state: "retained", reason: `status mode is ${status.mode}, not workflow` };
|
|
202
|
+
if (!terminalState(status.state)) return { state: "retained", reason: `workflow is still ${status.state}` };
|
|
203
|
+
if (liveWorkflowRunIds.has(owner.runId)) return { state: "retained", reason: "workflow controller is still live" };
|
|
204
|
+
for (const step of status.steps ?? []) {
|
|
205
|
+
const label = step.workflowKey ?? step.agent;
|
|
206
|
+
if (step.status === "pending" || step.status === "running" || step.status === "paused") return { state: "retained", reason: `workflow child ${label} is still ${step.status}` };
|
|
207
|
+
if (typeof step.async !== "boolean") return { state: "retained", reason: `workflow child ${label} is missing async classification` };
|
|
208
|
+
if (!step.async) continue;
|
|
209
|
+
if (!step.runId) return { state: "retained", reason: `async workflow child ${label} is missing run id` };
|
|
210
|
+
const childDir = path.join(path.dirname(owner.asyncDir), step.runId);
|
|
211
|
+
if (!fs.existsSync(childDir)) return { state: "retained", reason: `async workflow child ${label} directory is missing` };
|
|
212
|
+
const childStatus = readStatus(childDir);
|
|
213
|
+
if (!childStatus) return { state: "retained", reason: `async workflow child ${label} status is missing or unreadable` };
|
|
214
|
+
if (!terminalState(childStatus.state)) return { state: "retained", reason: `async workflow child ${label} is still ${childStatus.state}` };
|
|
215
|
+
if (!childStatus.processTerminal?.runnerProcessInstanceId) return { state: "retained", reason: `async workflow child ${label} has no runner process identity` };
|
|
216
|
+
const proof = readProcessTerminal(childDir, {
|
|
217
|
+
runId: step.runId,
|
|
218
|
+
runnerProcessInstanceId: childStatus.processTerminal.runnerProcessInstanceId,
|
|
219
|
+
});
|
|
220
|
+
if (proof?.state !== "observed" || proof.runId !== step.runId) return { state: "retained", reason: `async workflow child ${label} process-terminal proof is ${proof?.state ?? "missing"}` };
|
|
221
|
+
}
|
|
222
|
+
return { state: "releasable", reason: "workflow is terminal, controller is gone, and async children have observed proof" };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function ownerReleaseVerdict(owner: ActiveAsyncCapacityOwnerV1, liveWorkflowRunIds: ReadonlySet<string>): ActiveAsyncCapacityReleaseVerdict {
|
|
226
|
+
const status = readStatus(owner.asyncDir);
|
|
227
|
+
return owner.kind === "runner"
|
|
228
|
+
? runnerReleaseVerdict(owner, status)
|
|
229
|
+
: workflowReleaseVerdict(owner, status, liveWorkflowRunIds);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function capacitySessionDirs(rootDir: string, sessionId?: string): string[] {
|
|
233
|
+
if (sessionId) return [sessionDir(sessionId, rootDir)];
|
|
234
|
+
try {
|
|
235
|
+
return fs.readdirSync(rootDir, { withFileTypes: true })
|
|
236
|
+
.filter((entry) => entry.isDirectory())
|
|
237
|
+
.map((entry) => path.join(rootDir, entry.name));
|
|
238
|
+
} catch (error) {
|
|
239
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function inspectActiveAsyncCapacityOwner(
|
|
245
|
+
input: { runId: string; sessionId?: string; asyncDir?: string },
|
|
246
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
247
|
+
): ActiveAsyncCapacityInspection {
|
|
248
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
249
|
+
const liveWorkflowRunIds = options.liveWorkflowRunIds ?? new Set<string>();
|
|
250
|
+
for (const poolDir of capacitySessionDirs(rootDir, input.sessionId)) {
|
|
251
|
+
for (const dir of occupiedSlots(poolDir)) {
|
|
252
|
+
const owner = readOwner(dir);
|
|
253
|
+
if (!owner) continue;
|
|
254
|
+
const sameRun = owner.runId === input.runId || (input.asyncDir !== undefined && path.resolve(owner.asyncDir) === path.resolve(input.asyncDir));
|
|
255
|
+
const sourceRun = owner.sourceRunId === input.runId;
|
|
256
|
+
if (!sameRun && !sourceRun) continue;
|
|
257
|
+
if (sourceRun && !sameRun) {
|
|
258
|
+
return {
|
|
259
|
+
owner,
|
|
260
|
+
relation: "source",
|
|
261
|
+
slotDir: dir,
|
|
262
|
+
release: { state: "not-owned", reason: `slot was transferred to ${owner.runId}` },
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
return { owner, relation: "current", slotDir: dir, release: ownerReleaseVerdict(owner, liveWorkflowRunIds) };
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return { relation: "none", release: { state: "not-owned", reason: "no active-capacity slot records this run" } };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function reconcileActiveAsyncCapacity(
|
|
272
|
+
sessionId: string,
|
|
273
|
+
limit: number | undefined,
|
|
274
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
275
|
+
): ActiveAsyncCapacitySnapshot {
|
|
276
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
277
|
+
const poolDir = sessionDir(sessionId, rootDir);
|
|
278
|
+
const liveWorkflowRunIds = options.liveWorkflowRunIds ?? new Set<string>();
|
|
279
|
+
for (const dir of occupiedSlots(poolDir)) {
|
|
280
|
+
const owner = readOwner(dir);
|
|
281
|
+
if (!owner
|
|
282
|
+
|| owner.ownerSessionId !== sessionId
|
|
283
|
+
|| owner.ownerSessionKey !== activeAsyncCapacitySessionKey(sessionId)
|
|
284
|
+
|| path.basename(dir) !== `slot-${owner.slot}`
|
|
285
|
+
|| ownerReleaseVerdict(owner, liveWorkflowRunIds).state !== "releasable") continue;
|
|
286
|
+
removeOwnedSlot(dir, owner, options);
|
|
287
|
+
}
|
|
288
|
+
return snapshotFor(sessionId, limit, rootDir);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
export function getActiveAsyncCapacitySnapshot(
|
|
292
|
+
sessionId: string,
|
|
293
|
+
limit: number | undefined,
|
|
294
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
295
|
+
): ActiveAsyncCapacitySnapshot {
|
|
296
|
+
return reconcileActiveAsyncCapacity(sessionId, limit, options);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function createSlot(poolDir: string, owner: ActiveAsyncCapacityOwnerV1): boolean {
|
|
300
|
+
const destination = slotDir(poolDir, owner.slot);
|
|
301
|
+
fs.mkdirSync(poolDir, { recursive: true, mode: 0o700 });
|
|
302
|
+
try {
|
|
303
|
+
fs.mkdirSync(destination, { mode: 0o700 });
|
|
304
|
+
} catch (error) {
|
|
305
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST") return false;
|
|
306
|
+
throw error;
|
|
307
|
+
}
|
|
308
|
+
// If owner persistence fails, the corrupt occupied directory remains and
|
|
309
|
+
// fails closed instead of becoming available to another admission.
|
|
310
|
+
fs.writeFileSync(path.join(destination, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, { encoding: "utf-8", mode: 0o600 });
|
|
311
|
+
return true;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function handleFor(owner: ActiveAsyncCapacityOwnerV1, limit: number, options: CapacityOptions, rollbackOwner?: ActiveAsyncCapacityOwnerV1): ActiveAsyncCapacityHandle {
|
|
315
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
316
|
+
const dir = slotDir(sessionDir(owner.ownerSessionId, rootDir), owner.slot);
|
|
317
|
+
return {
|
|
318
|
+
owner,
|
|
319
|
+
markStarted(runnerProcessInstanceId) {
|
|
320
|
+
const claimed = withSlotClaim(dir, () => {
|
|
321
|
+
const current = matchingOwner(dir, owner);
|
|
322
|
+
if (!current) return false;
|
|
323
|
+
const next = { ...current, runnerProcessInstanceId, runnerStartedAt: options.now?.() ?? Date.now() };
|
|
324
|
+
// Mark memory first. If persistence fails after the process starts, caller
|
|
325
|
+
// cleanup must retain the occupied slot instead of rolling it back.
|
|
326
|
+
Object.assign(owner, next);
|
|
327
|
+
try {
|
|
328
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
329
|
+
} catch (error) {
|
|
330
|
+
console.error(`Failed to bind active async capacity to runner '${runnerProcessInstanceId}'; capacity will remain occupied:`, error);
|
|
331
|
+
}
|
|
332
|
+
return true;
|
|
333
|
+
});
|
|
334
|
+
if (!claimed.acquired || !claimed.value) throw new Error(`Active async capacity ownership changed for run '${owner.runId}'.`);
|
|
335
|
+
},
|
|
336
|
+
markWorkflowStarted() {
|
|
337
|
+
const claimed = withSlotClaim(dir, () => {
|
|
338
|
+
const current = matchingOwner(dir, owner);
|
|
339
|
+
if (!current || current.kind !== "workflow") return false;
|
|
340
|
+
const next = { ...current, runnerStartedAt: options.now?.() ?? Date.now() };
|
|
341
|
+
Object.assign(owner, next);
|
|
342
|
+
try {
|
|
343
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
344
|
+
} catch (error) {
|
|
345
|
+
console.error(`Failed to mark async workflow '${owner.runId}' started; capacity will remain occupied:`, error);
|
|
346
|
+
}
|
|
347
|
+
return true;
|
|
348
|
+
});
|
|
349
|
+
if (!claimed.acquired || !claimed.value) throw new Error(`Active async capacity ownership changed for workflow '${owner.runId}'.`);
|
|
350
|
+
},
|
|
351
|
+
rollback() {
|
|
352
|
+
if (!rollbackOwner) return removeOwnedSlot(dir, owner, options, true);
|
|
353
|
+
if (owner.runnerProcessInstanceId || owner.runnerStartedAt) return false;
|
|
354
|
+
const claimed = withSlotClaim(dir, () => {
|
|
355
|
+
const current = matchingOwner(dir, owner);
|
|
356
|
+
if (!current || current.runnerProcessInstanceId || current.runnerStartedAt) return false;
|
|
357
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), rollbackOwner);
|
|
358
|
+
Object.assign(owner, rollbackOwner);
|
|
359
|
+
return true;
|
|
360
|
+
});
|
|
361
|
+
return claimed.acquired && claimed.value;
|
|
362
|
+
},
|
|
363
|
+
reconcile(liveWorkflowRunIds) {
|
|
364
|
+
return reconcileActiveAsyncCapacity(owner.ownerSessionId, limit, { ...options, rootDir, liveWorkflowRunIds });
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
export function acquireActiveAsyncCapacity(
|
|
370
|
+
input: { sessionId: string; limit: number | undefined; runId: string; kind: "runner" | "workflow"; asyncDir: string },
|
|
371
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
372
|
+
): ActiveAsyncCapacityHandle | undefined {
|
|
373
|
+
if (input.limit === undefined) return undefined;
|
|
374
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
375
|
+
const reconciled = reconcileActiveAsyncCapacity(input.sessionId, input.limit, options);
|
|
376
|
+
if (reconciled.used >= input.limit) throw new ActiveAsyncCapacityError(reconciled);
|
|
377
|
+
const poolDir = sessionDir(input.sessionId, rootDir);
|
|
378
|
+
const token = options.token?.() ?? randomUUID();
|
|
379
|
+
for (let slot = 0; slot < input.limit; slot++) {
|
|
380
|
+
const owner: ActiveAsyncCapacityOwnerV1 = {
|
|
381
|
+
version: 1,
|
|
382
|
+
reservationToken: token,
|
|
383
|
+
ownerSessionId: input.sessionId,
|
|
384
|
+
ownerSessionKey: activeAsyncCapacitySessionKey(input.sessionId),
|
|
385
|
+
slot,
|
|
386
|
+
runId: input.runId,
|
|
387
|
+
generation: 0,
|
|
388
|
+
kind: input.kind,
|
|
389
|
+
asyncDir: input.asyncDir,
|
|
390
|
+
reservedAt: options.now?.() ?? Date.now(),
|
|
391
|
+
};
|
|
392
|
+
if (createSlot(poolDir, owner)) return handleFor(owner, input.limit, { ...options, rootDir });
|
|
393
|
+
}
|
|
394
|
+
throw new ActiveAsyncCapacityError(snapshotFor(input.sessionId, input.limit, rootDir));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
export function transferActiveAsyncCapacity(
|
|
398
|
+
input: { sessionId: string; limit: number | undefined; sourceRunId: string; runId: string; asyncDir: string },
|
|
399
|
+
options: CapacityOptions = {},
|
|
400
|
+
): ActiveAsyncCapacityHandle | undefined {
|
|
401
|
+
const limit = input.limit ?? 0;
|
|
402
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
403
|
+
const poolDir = sessionDir(input.sessionId, rootDir);
|
|
404
|
+
for (const dir of occupiedSlots(poolDir)) {
|
|
405
|
+
const source = readOwner(dir);
|
|
406
|
+
if (!source || source.ownerSessionId !== input.sessionId || source.runId !== input.sourceRunId) continue;
|
|
407
|
+
const claimed = withSlotClaim(dir, () => {
|
|
408
|
+
const current = matchingOwner(dir, source);
|
|
409
|
+
const status = readStatus(source.asyncDir);
|
|
410
|
+
if (!current || !status || status.runId !== input.sourceRunId || status.state === "queued" || status.state === "running") {
|
|
411
|
+
throw new Error(`Active async capacity source '${input.sourceRunId}' is not transferable.`);
|
|
412
|
+
}
|
|
413
|
+
const next: ActiveAsyncCapacityOwnerV1 = {
|
|
414
|
+
...current,
|
|
415
|
+
runId: input.runId,
|
|
416
|
+
sourceRunId: input.sourceRunId,
|
|
417
|
+
generation: current.generation + 1,
|
|
418
|
+
kind: "runner",
|
|
419
|
+
asyncDir: input.asyncDir,
|
|
420
|
+
reservedAt: options.now?.() ?? Date.now(),
|
|
421
|
+
};
|
|
422
|
+
delete next.runnerProcessInstanceId;
|
|
423
|
+
delete next.runnerStartedAt;
|
|
424
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
425
|
+
return handleFor(next, limit, { ...options, rootDir }, current);
|
|
426
|
+
});
|
|
427
|
+
if (!claimed.acquired) throw new Error(`Active async capacity transfer is already in progress for run '${input.sourceRunId}'.`);
|
|
428
|
+
return claimed.value;
|
|
429
|
+
}
|
|
430
|
+
return acquireActiveAsyncCapacity({ sessionId: input.sessionId, limit: input.limit, runId: input.runId, kind: "runner", asyncDir: input.asyncDir }, options);
|
|
431
|
+
}
|
|
@@ -16,6 +16,14 @@ export function isActiveAsyncState(state: AsyncStatus["state"]): boolean {
|
|
|
16
16
|
return state === "queued" || state === "running";
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
export function releaseActiveRunIndex(asyncDir: string): void {
|
|
20
|
+
try {
|
|
21
|
+
fs.rmSync(markerPath(asyncDir));
|
|
22
|
+
} catch (error) {
|
|
23
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
19
27
|
export function updateActiveRunIndex(asyncDir: string, state: AsyncStatus["state"]): void {
|
|
20
28
|
const marker = markerPath(asyncDir);
|
|
21
29
|
if (isActiveAsyncState(state)) {
|
|
@@ -23,11 +31,7 @@ export function updateActiveRunIndex(asyncDir: string, state: AsyncStatus["state
|
|
|
23
31
|
fs.writeFileSync(marker, "", { flag: "a" });
|
|
24
32
|
return;
|
|
25
33
|
}
|
|
26
|
-
|
|
27
|
-
fs.rmSync(marker);
|
|
28
|
-
} catch (error) {
|
|
29
|
-
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
30
|
-
}
|
|
34
|
+
releaseActiveRunIndex(asyncDir);
|
|
31
35
|
}
|
|
32
36
|
|
|
33
37
|
export function readActiveRunIndex(asyncDirRoot: string): string[] | undefined {
|