pi-subagents 0.47.0 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +2 -0
- package/agents/reviewer.md +3 -4
- package/docs/configuration.md +46 -2
- package/docs/observability.md +1 -1
- package/docs/tool-reference.md +1 -1
- package/package.json +1 -1
- package/src/agents/agents.ts +9 -3
- package/src/extension/config.ts +16 -0
- package/src/extension/doctor.ts +40 -0
- package/src/extension/index.ts +30 -11
- package/src/extension/public-execution.ts +5 -0
- package/src/extension/rpc.ts +7 -9
- package/src/extension/schemas.ts +4 -4
- package/src/intercom/intercom-bridge.ts +4 -1
- package/src/intercom/native-supervisor-channel.ts +102 -4
- package/src/missions/lifecycle.ts +4 -7
- package/src/missions/store.ts +4 -4
- package/src/missions/workflow-state.ts +6 -2
- package/src/runs/background/active-async-capacity.ts +374 -0
- package/src/runs/background/active-run-index.ts +46 -0
- package/src/runs/background/async-execution.ts +117 -29
- package/src/runs/background/async-job-tracker.ts +279 -134
- package/src/runs/background/async-resume.ts +7 -1
- package/src/runs/background/async-status.ts +33 -4
- package/src/runs/background/chain-append.ts +33 -15
- package/src/runs/background/control-channel.ts +55 -17
- package/src/runs/background/owned-process-tree.ts +104 -0
- package/src/runs/background/process-terminal.ts +17 -3
- package/src/runs/background/result-watcher.ts +87 -6
- package/src/runs/background/run-status.ts +23 -1
- package/src/runs/background/scheduled-runs.ts +23 -0
- package/src/runs/background/stale-run-reconciler.ts +10 -3
- package/src/runs/background/subagent-runner.ts +75 -33
- package/src/runs/foreground/async-dismiss-action.ts +85 -0
- package/src/runs/foreground/async-steering-action.ts +6 -7
- package/src/runs/foreground/chain-execution.ts +37 -2
- package/src/runs/foreground/execution.ts +90 -19
- package/src/runs/foreground/foreground-control.ts +12 -0
- package/src/runs/foreground/prompt-audit.ts +171 -0
- package/src/runs/foreground/subagent-executor.ts +648 -195
- package/src/runs/shared/acceptance.ts +13 -4
- package/src/runs/shared/llm-intent-arbiter.ts +286 -0
- package/src/runs/shared/parallel-utils.ts +2 -0
- package/src/runs/shared/pi-args.ts +44 -1
- package/src/runs/shared/run-fanout-budget.ts +280 -0
- package/src/runs/shared/single-output.ts +4 -2
- package/src/runs/shared/subagent-prompt-runtime.ts +24 -5
- package/src/runs/shared/task-intent.ts +19 -3
- package/src/runs/shared/worktree.ts +17 -5
- package/src/shared/artifacts.ts +1 -1
- package/src/shared/file-coalescer.ts +9 -0
- package/src/shared/types.ts +97 -1
- package/src/shared/utils.ts +3 -1
- package/src/tui/fleet-status.ts +7 -5
- package/src/tui/fleet.ts +225 -12
- package/src/workflows/scripted-workflow.ts +26 -6
|
@@ -22,6 +22,7 @@ export const NATIVE_SUPERVISOR_TOOL_NAME = "subagent_supervisor";
|
|
|
22
22
|
const MAX_MESSAGE_BYTES = 64 * 1024;
|
|
23
23
|
const DEFAULT_ASK_TIMEOUT_MS = 10 * 60 * 1000;
|
|
24
24
|
const CHANNEL_POLL_MS = Math.min(POLL_INTERVAL_MS, 500);
|
|
25
|
+
const CHANNEL_SAFETY_POLL_MS = 5000;
|
|
25
26
|
const STALE_EMPTY_CHANNEL_AGE_MS = 60 * 1000;
|
|
26
27
|
const STALE_EMPTY_CHANNEL_CLEANUP_INTERVAL_MS = 60 * 1000;
|
|
27
28
|
|
|
@@ -69,6 +70,13 @@ interface IntercomParams {
|
|
|
69
70
|
replyTo?: string;
|
|
70
71
|
}
|
|
71
72
|
|
|
73
|
+
type SupervisorWatch = (filename: fs.PathLike, listener: fs.WatchListener<string>) => fs.FSWatcher;
|
|
74
|
+
|
|
75
|
+
interface NativeSupervisorChannelDeps {
|
|
76
|
+
platform?: NodeJS.Platform;
|
|
77
|
+
watch?: SupervisorWatch;
|
|
78
|
+
}
|
|
79
|
+
|
|
72
80
|
const ContactSupervisorParamsSchema = Type.Object({
|
|
73
81
|
reason: Type.String({ enum: ["need_decision", "interview_request", "progress_update"] }),
|
|
74
82
|
message: Type.Optional(Type.String()),
|
|
@@ -626,10 +634,16 @@ function buildParentIntercomTool(pending: Map<string, PendingSupervisorRequest>,
|
|
|
626
634
|
};
|
|
627
635
|
}
|
|
628
636
|
|
|
629
|
-
export function createNativeSupervisorChannel(pi: ExtensionAPI, state: SubagentState): { start: () => void; dispose: () => void; pending: Map<string, PendingSupervisorRequest> } {
|
|
637
|
+
export function createNativeSupervisorChannel(pi: ExtensionAPI, state: SubagentState, deps: NativeSupervisorChannelDeps = {}): { start: () => void; dispose: () => void; pending: Map<string, PendingSupervisorRequest> } {
|
|
638
|
+
const watch = deps.watch ?? fs.watch;
|
|
630
639
|
const pending = new Map<string, PendingSupervisorRequest>();
|
|
631
640
|
const seenFiles = new Set<string>();
|
|
641
|
+
const requestWatchers = new Map<string, fs.FSWatcher>();
|
|
642
|
+
let rootWatcher: fs.FSWatcher | undefined;
|
|
632
643
|
let poller: ReturnType<typeof setInterval> | undefined;
|
|
644
|
+
let safetyPoller: ReturnType<typeof setInterval> | undefined;
|
|
645
|
+
let deferredWatcherRefresh: ReturnType<typeof setImmediate> | undefined;
|
|
646
|
+
let started = false;
|
|
633
647
|
let lastStaleCleanupAt = 0;
|
|
634
648
|
|
|
635
649
|
const registerParentTools = (): void => {
|
|
@@ -696,17 +710,101 @@ export function createNativeSupervisorChannel(pi: ExtensionAPI, state: SubagentS
|
|
|
696
710
|
}
|
|
697
711
|
};
|
|
698
712
|
|
|
713
|
+
const startPolling = (): void => {
|
|
714
|
+
if (poller) return;
|
|
715
|
+
poller = setInterval(poll, CHANNEL_POLL_MS);
|
|
716
|
+
poller.unref?.();
|
|
717
|
+
};
|
|
718
|
+
const startSafetyPolling = (): void => {
|
|
719
|
+
if (safetyPoller) return;
|
|
720
|
+
safetyPoller = setInterval(() => {
|
|
721
|
+
watchExistingRequestDirs();
|
|
722
|
+
poll();
|
|
723
|
+
}, CHANNEL_SAFETY_POLL_MS);
|
|
724
|
+
safetyPoller.unref?.();
|
|
725
|
+
};
|
|
726
|
+
const watchRequestDir = (requestsDir: string): void => {
|
|
727
|
+
if (requestWatchers.has(requestsDir)) return;
|
|
728
|
+
try {
|
|
729
|
+
const watcher = watch(requestsDir, () => poll());
|
|
730
|
+
watcher.on("error", () => {
|
|
731
|
+
try { watcher.close(); } catch {}
|
|
732
|
+
requestWatchers.delete(requestsDir);
|
|
733
|
+
startPolling();
|
|
734
|
+
});
|
|
735
|
+
watcher.unref?.();
|
|
736
|
+
requestWatchers.set(requestsDir, watcher);
|
|
737
|
+
} catch (error) {
|
|
738
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") startPolling();
|
|
739
|
+
}
|
|
740
|
+
};
|
|
741
|
+
const watchExistingRequestDirs = (): void => {
|
|
742
|
+
let channelEntries: fs.Dirent[];
|
|
743
|
+
try {
|
|
744
|
+
channelEntries = fs.readdirSync(SUPERVISOR_CHANNEL_ROOT, { withFileTypes: true });
|
|
745
|
+
} catch (error) {
|
|
746
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
747
|
+
startPolling();
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
for (const entry of channelEntries) {
|
|
751
|
+
if (entry.isDirectory()) watchRequestDir(path.join(SUPERVISOR_CHANNEL_ROOT, entry.name, REQUESTS_DIR));
|
|
752
|
+
}
|
|
753
|
+
};
|
|
754
|
+
const scheduleWatcherRefresh = (): void => {
|
|
755
|
+
if (deferredWatcherRefresh) return;
|
|
756
|
+
deferredWatcherRefresh = setImmediate(() => {
|
|
757
|
+
deferredWatcherRefresh = undefined;
|
|
758
|
+
if (!started) return;
|
|
759
|
+
watchExistingRequestDirs();
|
|
760
|
+
poll();
|
|
761
|
+
});
|
|
762
|
+
deferredWatcherRefresh.unref?.();
|
|
763
|
+
};
|
|
764
|
+
|
|
699
765
|
return {
|
|
700
766
|
start: () => {
|
|
701
|
-
if (
|
|
767
|
+
if (started) return;
|
|
768
|
+
started = true;
|
|
702
769
|
registerParentTools();
|
|
703
770
|
poll();
|
|
704
|
-
|
|
705
|
-
|
|
771
|
+
try {
|
|
772
|
+
fs.mkdirSync(SUPERVISOR_CHANNEL_ROOT, { recursive: true });
|
|
773
|
+
if ((deps.platform ?? process.platform) === "win32") {
|
|
774
|
+
startPolling();
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
watchExistingRequestDirs();
|
|
778
|
+
rootWatcher = watch(SUPERVISOR_CHANNEL_ROOT, () => {
|
|
779
|
+
watchExistingRequestDirs();
|
|
780
|
+
poll();
|
|
781
|
+
scheduleWatcherRefresh();
|
|
782
|
+
});
|
|
783
|
+
rootWatcher.on("error", startPolling);
|
|
784
|
+
startSafetyPolling();
|
|
785
|
+
scheduleWatcherRefresh();
|
|
786
|
+
} catch {
|
|
787
|
+
startPolling();
|
|
788
|
+
}
|
|
706
789
|
},
|
|
707
790
|
dispose: () => {
|
|
791
|
+
started = false;
|
|
792
|
+
try {
|
|
793
|
+
rootWatcher?.close();
|
|
794
|
+
} catch {
|
|
795
|
+
// Best effort during shutdown.
|
|
796
|
+
}
|
|
797
|
+
rootWatcher = undefined;
|
|
798
|
+
for (const watcher of requestWatchers.values()) {
|
|
799
|
+
try { watcher.close(); } catch {}
|
|
800
|
+
}
|
|
801
|
+
requestWatchers.clear();
|
|
708
802
|
if (poller) clearInterval(poller);
|
|
709
803
|
poller = undefined;
|
|
804
|
+
if (safetyPoller) clearInterval(safetyPoller);
|
|
805
|
+
safetyPoller = undefined;
|
|
806
|
+
if (deferredWatcherRefresh) clearImmediate(deferredWatcherRefresh);
|
|
807
|
+
deferredWatcherRefresh = undefined;
|
|
710
808
|
pending.clear();
|
|
711
809
|
seenFiles.clear();
|
|
712
810
|
},
|
|
@@ -2,6 +2,7 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { AgentToolResult } from "@earendil-works/pi-agent-core";
|
|
4
4
|
import { writePrivateAtomicJson } from "../shared/atomic-json.ts";
|
|
5
|
+
import { PROMPT_REDACTED } from "../shared/utils.ts";
|
|
5
6
|
import type { Details, SubagentRunMode } from "../shared/types.ts";
|
|
6
7
|
import { validateMissionLaunch } from "./actions.ts";
|
|
7
8
|
import type { MissionArtifact, MissionRecord, MissionRunLink, MissionRunMode, MissionStatus, MissionStoreConfig, MissionStoreLocation } from "./types.ts";
|
|
@@ -47,11 +48,6 @@ function workflowObjective(params: MissionLaunchParams): string | undefined {
|
|
|
47
48
|
return undefined;
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
function conciseTitle(objective: string): string {
|
|
51
|
-
const firstLine = objective.split(/\r?\n/, 1)[0]?.trim() || objective.trim();
|
|
52
|
-
return firstLine.length > 100 ? `${firstLine.slice(0, 97)}...` : firstLine;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
51
|
export function prepareMissionLaunch(input: {
|
|
56
52
|
params: MissionLaunchParams;
|
|
57
53
|
projectRoot: string;
|
|
@@ -73,10 +69,11 @@ export function prepareMissionLaunch(input: {
|
|
|
73
69
|
return { missionId, location, autoCreated: false, announceInContent: true };
|
|
74
70
|
}
|
|
75
71
|
const mission = input.params.mission !== undefined ? validateMissionLaunch(input.params.mission) : undefined;
|
|
76
|
-
const
|
|
72
|
+
const promptDerivedObjective = mission?.objective ?? (mission ? mission.title : objective ? PROMPT_REDACTED : undefined);
|
|
73
|
+
const title = mission?.title || PROMPT_REDACTED;
|
|
77
74
|
const record = createMission(location, {
|
|
78
75
|
title,
|
|
79
|
-
objective:
|
|
76
|
+
objective: promptDerivedObjective || title,
|
|
80
77
|
...(mission?.goal === true ? { goal: true as const } : {}),
|
|
81
78
|
...(mission?.budget ? { budget: mission.budget } : {}),
|
|
82
79
|
status: "active",
|
package/src/missions/store.ts
CHANGED
|
@@ -396,11 +396,11 @@ export class MissionNotFoundError extends Error {
|
|
|
396
396
|
readonly missionId: string;
|
|
397
397
|
readonly missionDir: string;
|
|
398
398
|
|
|
399
|
-
constructor(missionId: string,
|
|
400
|
-
super(`Mission '${missionId}' was not found in ${missionDir}
|
|
399
|
+
constructor(missionId: string, location: MissionStoreLocation) {
|
|
400
|
+
super(`Mission '${missionId}' was not found in mission directory '${location.missionDir}' for project root '${location.projectRoot}'. If it was created in another worktree, run the request from that worktree.`);
|
|
401
401
|
this.name = "MissionNotFoundError";
|
|
402
402
|
this.missionId = missionId;
|
|
403
|
-
this.missionDir = missionDir;
|
|
403
|
+
this.missionDir = location.missionDir;
|
|
404
404
|
}
|
|
405
405
|
}
|
|
406
406
|
|
|
@@ -410,7 +410,7 @@ export function readMission(location: MissionStoreLocation, missionId: string):
|
|
|
410
410
|
try {
|
|
411
411
|
raw = fs.readFileSync(filePath, "utf-8");
|
|
412
412
|
} catch (error) {
|
|
413
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") throw new MissionNotFoundError(missionId, location
|
|
413
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") throw new MissionNotFoundError(missionId, location);
|
|
414
414
|
throw error;
|
|
415
415
|
}
|
|
416
416
|
try {
|
|
@@ -95,8 +95,12 @@ function stateLockIsStale(lockPath: string, now = Date.now()): boolean {
|
|
|
95
95
|
const owner = readStateLockOwner(lockPath);
|
|
96
96
|
if (owner) {
|
|
97
97
|
if (!isProcessAlive(owner.pid)) return true;
|
|
98
|
-
|
|
99
|
-
|
|
98
|
+
if (owner.processKey) {
|
|
99
|
+
const currentProcessKey = owner.pid === process.pid ? CURRENT_PROCESS_KEY : processStartKey(owner.pid);
|
|
100
|
+
if (currentProcessKey) return owner.processKey !== currentProcessKey;
|
|
101
|
+
if (owner.pid === process.pid) return true;
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
100
104
|
}
|
|
101
105
|
try {
|
|
102
106
|
return now - fs.statSync(lockPath).mtimeMs > STATE_LOCK_STALE_MS;
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { writePrivateAtomicJson } from "../../shared/atomic-json.ts";
|
|
5
|
+
import { TEMP_ROOT_DIR, type ActiveAsyncCapacitySnapshot, type AsyncStatus } from "../../shared/types.ts";
|
|
6
|
+
import { readStatus } from "../../shared/utils.ts";
|
|
7
|
+
import { readProcessTerminal } from "./process-terminal.ts";
|
|
8
|
+
|
|
9
|
+
export const ACTIVE_ASYNC_CAPACITY_DIR = path.join(TEMP_ROOT_DIR, "session-active-async-capacity");
|
|
10
|
+
|
|
11
|
+
export interface ActiveAsyncCapacityOwnerV1 {
|
|
12
|
+
version: 1;
|
|
13
|
+
reservationToken: string;
|
|
14
|
+
ownerSessionId: string;
|
|
15
|
+
ownerSessionKey: string;
|
|
16
|
+
slot: number;
|
|
17
|
+
runId: string;
|
|
18
|
+
sourceRunId?: string;
|
|
19
|
+
generation: number;
|
|
20
|
+
kind: "runner" | "workflow";
|
|
21
|
+
asyncDir: string;
|
|
22
|
+
reservedAt: number;
|
|
23
|
+
runnerProcessInstanceId?: string;
|
|
24
|
+
runnerStartedAt?: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface ActiveAsyncCapacityHandle {
|
|
28
|
+
readonly owner: ActiveAsyncCapacityOwnerV1;
|
|
29
|
+
markStarted(runnerProcessInstanceId: string): void;
|
|
30
|
+
markWorkflowStarted(): void;
|
|
31
|
+
rollback(): boolean;
|
|
32
|
+
reconcile(liveWorkflowRunIds?: ReadonlySet<string>): ActiveAsyncCapacitySnapshot;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface CapacityOptions {
|
|
36
|
+
rootDir?: string;
|
|
37
|
+
now?: () => number;
|
|
38
|
+
token?: () => string;
|
|
39
|
+
afterSlotRename?: (releasedDir: string) => void;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export class ActiveAsyncCapacityError extends Error {
|
|
43
|
+
readonly snapshot: ActiveAsyncCapacitySnapshot;
|
|
44
|
+
|
|
45
|
+
constructor(snapshot: ActiveAsyncCapacitySnapshot) {
|
|
46
|
+
super(`Active async run capacity exhausted: ${snapshot.used}/${snapshot.limit} used.`);
|
|
47
|
+
this.name = "ActiveAsyncCapacityError";
|
|
48
|
+
this.snapshot = snapshot;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function resolveMaxActiveAsyncRunsPerSession(value: unknown): number | undefined {
|
|
53
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) return undefined;
|
|
54
|
+
return value === 0 ? undefined : value;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function activeAsyncCapacitySessionKey(sessionId: string): string {
|
|
58
|
+
return createHash("sha256").update(sessionId).digest("hex");
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function sessionDir(sessionId: string, rootDir: string): string {
|
|
62
|
+
return path.join(rootDir, activeAsyncCapacitySessionKey(sessionId));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function slotDir(poolDir: string, slot: number): string {
|
|
66
|
+
return path.join(poolDir, `slot-${slot}`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function parseOwner(value: unknown): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
70
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
71
|
+
const owner = value as Partial<ActiveAsyncCapacityOwnerV1>;
|
|
72
|
+
if (owner.version !== 1
|
|
73
|
+
|| typeof owner.reservationToken !== "string" || !owner.reservationToken
|
|
74
|
+
|| typeof owner.ownerSessionId !== "string" || !owner.ownerSessionId
|
|
75
|
+
|| typeof owner.ownerSessionKey !== "string" || !owner.ownerSessionKey
|
|
76
|
+
|| typeof owner.slot !== "number" || !Number.isInteger(owner.slot) || owner.slot < 0
|
|
77
|
+
|| typeof owner.runId !== "string" || !owner.runId
|
|
78
|
+
|| typeof owner.generation !== "number" || !Number.isInteger(owner.generation) || owner.generation < 0
|
|
79
|
+
|| (owner.kind !== "runner" && owner.kind !== "workflow")
|
|
80
|
+
|| typeof owner.asyncDir !== "string" || !owner.asyncDir
|
|
81
|
+
|| typeof owner.reservedAt !== "number" || !Number.isFinite(owner.reservedAt)) return undefined;
|
|
82
|
+
if (owner.sourceRunId !== undefined && typeof owner.sourceRunId !== "string") return undefined;
|
|
83
|
+
if (owner.runnerProcessInstanceId !== undefined && (typeof owner.runnerProcessInstanceId !== "string" || !owner.runnerProcessInstanceId)) return undefined;
|
|
84
|
+
if (owner.runnerStartedAt !== undefined && (typeof owner.runnerStartedAt !== "number" || !Number.isFinite(owner.runnerStartedAt))) return undefined;
|
|
85
|
+
return owner as ActiveAsyncCapacityOwnerV1;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function readOwner(dir: string): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
89
|
+
try {
|
|
90
|
+
return parseOwner(JSON.parse(fs.readFileSync(path.join(dir, "owner.json"), "utf-8")));
|
|
91
|
+
} catch {
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function occupiedSlots(poolDir: string): string[] {
|
|
97
|
+
try {
|
|
98
|
+
return fs.readdirSync(poolDir, { withFileTypes: true })
|
|
99
|
+
.filter((entry) => entry.isDirectory() && /^slot-\d+$/.test(entry.name))
|
|
100
|
+
.map((entry) => path.join(poolDir, entry.name));
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function snapshotFor(sessionId: string, limit: number | undefined, rootDir: string): ActiveAsyncCapacitySnapshot {
|
|
108
|
+
return { used: occupiedSlots(sessionDir(sessionId, rootDir)).length, limit: limit ?? 0 };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function matchingOwner(dir: string, expected: ActiveAsyncCapacityOwnerV1): ActiveAsyncCapacityOwnerV1 | undefined {
|
|
112
|
+
const owner = readOwner(dir);
|
|
113
|
+
return owner?.reservationToken === expected.reservationToken
|
|
114
|
+
&& owner.runId === expected.runId
|
|
115
|
+
&& owner.generation === expected.generation
|
|
116
|
+
? owner
|
|
117
|
+
: undefined;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function withSlotClaim<T>(dir: string, operation: () => T): { acquired: true; value: T } | { acquired: false } {
|
|
121
|
+
const claimPath = path.join(dir, "capacity.claim");
|
|
122
|
+
const claimToken = randomUUID();
|
|
123
|
+
let claim: number | undefined;
|
|
124
|
+
try {
|
|
125
|
+
claim = fs.openSync(claimPath, "wx", 0o600);
|
|
126
|
+
fs.writeFileSync(claim, claimToken, "utf-8");
|
|
127
|
+
fs.closeSync(claim);
|
|
128
|
+
claim = undefined;
|
|
129
|
+
} catch (error) {
|
|
130
|
+
if (claim !== undefined) fs.closeSync(claim);
|
|
131
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST" || (error as NodeJS.ErrnoException).code === "ENOENT") return { acquired: false };
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
return { acquired: true, value: operation() };
|
|
136
|
+
} finally {
|
|
137
|
+
try {
|
|
138
|
+
if (fs.readFileSync(claimPath, "utf-8") === claimToken) fs.rmSync(claimPath, { force: true });
|
|
139
|
+
} catch (error) {
|
|
140
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function removeOwnedSlot(dir: string, expected: ActiveAsyncCapacityOwnerV1, options: CapacityOptions, requireUnstarted = false): boolean {
|
|
146
|
+
if (requireUnstarted && (expected.runnerProcessInstanceId || expected.runnerStartedAt)) return false;
|
|
147
|
+
const claimed = withSlotClaim(dir, () => {
|
|
148
|
+
const current = matchingOwner(dir, expected);
|
|
149
|
+
if (!current || (requireUnstarted && (current.runnerProcessInstanceId || current.runnerStartedAt))) return false;
|
|
150
|
+
const releasedDir = path.join(path.dirname(dir), `.${path.basename(dir)}.released-${randomUUID()}`);
|
|
151
|
+
fs.renameSync(dir, releasedDir);
|
|
152
|
+
options.afterSlotRename?.(releasedDir);
|
|
153
|
+
fs.rmSync(releasedDir, { recursive: true, force: true });
|
|
154
|
+
return true;
|
|
155
|
+
});
|
|
156
|
+
return claimed.acquired && claimed.value;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function terminalState(state: AsyncStatus["state"]): boolean {
|
|
160
|
+
return state !== "queued" && state !== "running" && state !== "paused";
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function runnerCanRelease(owner: ActiveAsyncCapacityOwnerV1, status: AsyncStatus): boolean {
|
|
164
|
+
if (!owner.runnerProcessInstanceId
|
|
165
|
+
|| status.sessionId !== owner.ownerSessionId
|
|
166
|
+
|| status.runId !== owner.runId
|
|
167
|
+
|| !terminalState(status.state)) return false;
|
|
168
|
+
if (status.processTerminal?.state === "not-started"
|
|
169
|
+
&& status.processTerminal.runId === owner.runId
|
|
170
|
+
&& status.processTerminal.runnerProcessInstanceId === owner.runnerProcessInstanceId
|
|
171
|
+
&& typeof status.error === "string"
|
|
172
|
+
&& status.error) return true;
|
|
173
|
+
const proof = readProcessTerminal(owner.asyncDir, {
|
|
174
|
+
runId: owner.runId,
|
|
175
|
+
runnerProcessInstanceId: owner.runnerProcessInstanceId,
|
|
176
|
+
});
|
|
177
|
+
return proof?.state === "observed"
|
|
178
|
+
&& proof.runId === owner.runId
|
|
179
|
+
&& proof.runnerProcessInstanceId === owner.runnerProcessInstanceId;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function workflowCanRelease(owner: ActiveAsyncCapacityOwnerV1, status: AsyncStatus, liveWorkflowRunIds: ReadonlySet<string>): boolean {
|
|
183
|
+
if (status.sessionId !== owner.ownerSessionId
|
|
184
|
+
|| status.runId !== owner.runId
|
|
185
|
+
|| status.mode !== "workflow"
|
|
186
|
+
|| !terminalState(status.state)
|
|
187
|
+
|| liveWorkflowRunIds.has(owner.runId)) return false;
|
|
188
|
+
for (const step of status.steps ?? []) {
|
|
189
|
+
if (step.status === "pending" || step.status === "running" || step.status === "paused") return false;
|
|
190
|
+
if (typeof step.async !== "boolean") return false;
|
|
191
|
+
if (!step.async) continue;
|
|
192
|
+
if (!step.runId) return false;
|
|
193
|
+
const childDir = path.join(path.dirname(owner.asyncDir), step.runId);
|
|
194
|
+
if (!fs.existsSync(childDir)) return false;
|
|
195
|
+
const childStatus = readStatus(childDir);
|
|
196
|
+
if (!childStatus || !terminalState(childStatus.state) || !childStatus.processTerminal?.runnerProcessInstanceId) return false;
|
|
197
|
+
const proof = readProcessTerminal(childDir, {
|
|
198
|
+
runId: step.runId,
|
|
199
|
+
runnerProcessInstanceId: childStatus.processTerminal.runnerProcessInstanceId,
|
|
200
|
+
});
|
|
201
|
+
if (proof?.state !== "observed" || proof.runId !== step.runId) return false;
|
|
202
|
+
}
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function ownerCanRelease(owner: ActiveAsyncCapacityOwnerV1, liveWorkflowRunIds: ReadonlySet<string>): boolean {
|
|
207
|
+
const status = readStatus(owner.asyncDir);
|
|
208
|
+
if (!status) return false;
|
|
209
|
+
return owner.kind === "runner"
|
|
210
|
+
? runnerCanRelease(owner, status)
|
|
211
|
+
: workflowCanRelease(owner, status, liveWorkflowRunIds);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export function reconcileActiveAsyncCapacity(
|
|
215
|
+
sessionId: string,
|
|
216
|
+
limit: number | undefined,
|
|
217
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
218
|
+
): ActiveAsyncCapacitySnapshot {
|
|
219
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
220
|
+
const poolDir = sessionDir(sessionId, rootDir);
|
|
221
|
+
const liveWorkflowRunIds = options.liveWorkflowRunIds ?? new Set<string>();
|
|
222
|
+
for (const dir of occupiedSlots(poolDir)) {
|
|
223
|
+
const owner = readOwner(dir);
|
|
224
|
+
if (!owner
|
|
225
|
+
|| owner.ownerSessionId !== sessionId
|
|
226
|
+
|| owner.ownerSessionKey !== activeAsyncCapacitySessionKey(sessionId)
|
|
227
|
+
|| path.basename(dir) !== `slot-${owner.slot}`
|
|
228
|
+
|| !ownerCanRelease(owner, liveWorkflowRunIds)) continue;
|
|
229
|
+
removeOwnedSlot(dir, owner, options);
|
|
230
|
+
}
|
|
231
|
+
return snapshotFor(sessionId, limit, rootDir);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export function getActiveAsyncCapacitySnapshot(
|
|
235
|
+
sessionId: string,
|
|
236
|
+
limit: number | undefined,
|
|
237
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
238
|
+
): ActiveAsyncCapacitySnapshot {
|
|
239
|
+
return reconcileActiveAsyncCapacity(sessionId, limit, options);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function createSlot(poolDir: string, owner: ActiveAsyncCapacityOwnerV1): boolean {
|
|
243
|
+
const destination = slotDir(poolDir, owner.slot);
|
|
244
|
+
fs.mkdirSync(poolDir, { recursive: true, mode: 0o700 });
|
|
245
|
+
try {
|
|
246
|
+
fs.mkdirSync(destination, { mode: 0o700 });
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST") return false;
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
// If owner persistence fails, the corrupt occupied directory remains and
|
|
252
|
+
// fails closed instead of becoming available to another admission.
|
|
253
|
+
fs.writeFileSync(path.join(destination, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, { encoding: "utf-8", mode: 0o600 });
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function handleFor(owner: ActiveAsyncCapacityOwnerV1, limit: number, options: CapacityOptions, rollbackOwner?: ActiveAsyncCapacityOwnerV1): ActiveAsyncCapacityHandle {
|
|
258
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
259
|
+
const dir = slotDir(sessionDir(owner.ownerSessionId, rootDir), owner.slot);
|
|
260
|
+
return {
|
|
261
|
+
owner,
|
|
262
|
+
markStarted(runnerProcessInstanceId) {
|
|
263
|
+
const claimed = withSlotClaim(dir, () => {
|
|
264
|
+
const current = matchingOwner(dir, owner);
|
|
265
|
+
if (!current) return false;
|
|
266
|
+
const next = { ...current, runnerProcessInstanceId, runnerStartedAt: options.now?.() ?? Date.now() };
|
|
267
|
+
// Mark memory first. If persistence fails after the process starts, caller
|
|
268
|
+
// cleanup must retain the occupied slot instead of rolling it back.
|
|
269
|
+
Object.assign(owner, next);
|
|
270
|
+
try {
|
|
271
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
272
|
+
} catch (error) {
|
|
273
|
+
console.error(`Failed to bind active async capacity to runner '${runnerProcessInstanceId}'; capacity will remain occupied:`, error);
|
|
274
|
+
}
|
|
275
|
+
return true;
|
|
276
|
+
});
|
|
277
|
+
if (!claimed.acquired || !claimed.value) throw new Error(`Active async capacity ownership changed for run '${owner.runId}'.`);
|
|
278
|
+
},
|
|
279
|
+
markWorkflowStarted() {
|
|
280
|
+
const claimed = withSlotClaim(dir, () => {
|
|
281
|
+
const current = matchingOwner(dir, owner);
|
|
282
|
+
if (!current || current.kind !== "workflow") return false;
|
|
283
|
+
const next = { ...current, runnerStartedAt: options.now?.() ?? Date.now() };
|
|
284
|
+
Object.assign(owner, next);
|
|
285
|
+
try {
|
|
286
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
287
|
+
} catch (error) {
|
|
288
|
+
console.error(`Failed to mark async workflow '${owner.runId}' started; capacity will remain occupied:`, error);
|
|
289
|
+
}
|
|
290
|
+
return true;
|
|
291
|
+
});
|
|
292
|
+
if (!claimed.acquired || !claimed.value) throw new Error(`Active async capacity ownership changed for workflow '${owner.runId}'.`);
|
|
293
|
+
},
|
|
294
|
+
rollback() {
|
|
295
|
+
if (!rollbackOwner) return removeOwnedSlot(dir, owner, options, true);
|
|
296
|
+
if (owner.runnerProcessInstanceId || owner.runnerStartedAt) return false;
|
|
297
|
+
const claimed = withSlotClaim(dir, () => {
|
|
298
|
+
const current = matchingOwner(dir, owner);
|
|
299
|
+
if (!current || current.runnerProcessInstanceId || current.runnerStartedAt) return false;
|
|
300
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), rollbackOwner);
|
|
301
|
+
Object.assign(owner, rollbackOwner);
|
|
302
|
+
return true;
|
|
303
|
+
});
|
|
304
|
+
return claimed.acquired && claimed.value;
|
|
305
|
+
},
|
|
306
|
+
reconcile(liveWorkflowRunIds) {
|
|
307
|
+
return reconcileActiveAsyncCapacity(owner.ownerSessionId, limit, { ...options, rootDir, liveWorkflowRunIds });
|
|
308
|
+
},
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function acquireActiveAsyncCapacity(
|
|
313
|
+
input: { sessionId: string; limit: number | undefined; runId: string; kind: "runner" | "workflow"; asyncDir: string },
|
|
314
|
+
options: CapacityOptions & { liveWorkflowRunIds?: ReadonlySet<string> } = {},
|
|
315
|
+
): ActiveAsyncCapacityHandle | undefined {
|
|
316
|
+
if (input.limit === undefined) return undefined;
|
|
317
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
318
|
+
const reconciled = reconcileActiveAsyncCapacity(input.sessionId, input.limit, options);
|
|
319
|
+
if (reconciled.used >= input.limit) throw new ActiveAsyncCapacityError(reconciled);
|
|
320
|
+
const poolDir = sessionDir(input.sessionId, rootDir);
|
|
321
|
+
const token = options.token?.() ?? randomUUID();
|
|
322
|
+
for (let slot = 0; slot < input.limit; slot++) {
|
|
323
|
+
const owner: ActiveAsyncCapacityOwnerV1 = {
|
|
324
|
+
version: 1,
|
|
325
|
+
reservationToken: token,
|
|
326
|
+
ownerSessionId: input.sessionId,
|
|
327
|
+
ownerSessionKey: activeAsyncCapacitySessionKey(input.sessionId),
|
|
328
|
+
slot,
|
|
329
|
+
runId: input.runId,
|
|
330
|
+
generation: 0,
|
|
331
|
+
kind: input.kind,
|
|
332
|
+
asyncDir: input.asyncDir,
|
|
333
|
+
reservedAt: options.now?.() ?? Date.now(),
|
|
334
|
+
};
|
|
335
|
+
if (createSlot(poolDir, owner)) return handleFor(owner, input.limit, { ...options, rootDir });
|
|
336
|
+
}
|
|
337
|
+
throw new ActiveAsyncCapacityError(snapshotFor(input.sessionId, input.limit, rootDir));
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
export function transferActiveAsyncCapacity(
|
|
341
|
+
input: { sessionId: string; limit: number | undefined; sourceRunId: string; runId: string; asyncDir: string },
|
|
342
|
+
options: CapacityOptions = {},
|
|
343
|
+
): ActiveAsyncCapacityHandle | undefined {
|
|
344
|
+
const limit = input.limit ?? 0;
|
|
345
|
+
const rootDir = options.rootDir ?? ACTIVE_ASYNC_CAPACITY_DIR;
|
|
346
|
+
const poolDir = sessionDir(input.sessionId, rootDir);
|
|
347
|
+
for (const dir of occupiedSlots(poolDir)) {
|
|
348
|
+
const source = readOwner(dir);
|
|
349
|
+
if (!source || source.ownerSessionId !== input.sessionId || source.runId !== input.sourceRunId) continue;
|
|
350
|
+
const claimed = withSlotClaim(dir, () => {
|
|
351
|
+
const current = matchingOwner(dir, source);
|
|
352
|
+
const status = readStatus(source.asyncDir);
|
|
353
|
+
if (!current || !status || status.runId !== input.sourceRunId || status.state === "queued" || status.state === "running") {
|
|
354
|
+
throw new Error(`Active async capacity source '${input.sourceRunId}' is not transferable.`);
|
|
355
|
+
}
|
|
356
|
+
const next: ActiveAsyncCapacityOwnerV1 = {
|
|
357
|
+
...current,
|
|
358
|
+
runId: input.runId,
|
|
359
|
+
sourceRunId: input.sourceRunId,
|
|
360
|
+
generation: current.generation + 1,
|
|
361
|
+
kind: "runner",
|
|
362
|
+
asyncDir: input.asyncDir,
|
|
363
|
+
reservedAt: options.now?.() ?? Date.now(),
|
|
364
|
+
};
|
|
365
|
+
delete next.runnerProcessInstanceId;
|
|
366
|
+
delete next.runnerStartedAt;
|
|
367
|
+
writePrivateAtomicJson(path.join(dir, "owner.json"), next);
|
|
368
|
+
return handleFor(next, limit, { ...options, rootDir }, current);
|
|
369
|
+
});
|
|
370
|
+
if (!claimed.acquired) throw new Error(`Active async capacity transfer is already in progress for run '${input.sourceRunId}'.`);
|
|
371
|
+
return claimed.value;
|
|
372
|
+
}
|
|
373
|
+
return acquireActiveAsyncCapacity({ sessionId: input.sessionId, limit: input.limit, runId: input.runId, kind: "runner", asyncDir: input.asyncDir }, options);
|
|
374
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import type { AsyncStatus } from "../../shared/types.ts";
|
|
4
|
+
|
|
5
|
+
export const ACTIVE_RUN_INDEX_DIR = ".active-runs";
|
|
6
|
+
|
|
7
|
+
function indexDir(asyncDirRoot: string): string {
|
|
8
|
+
return path.join(asyncDirRoot, ACTIVE_RUN_INDEX_DIR);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function markerPath(asyncDir: string): string {
|
|
12
|
+
return path.join(indexDir(path.dirname(asyncDir)), path.basename(asyncDir));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function isActiveAsyncState(state: AsyncStatus["state"]): boolean {
|
|
16
|
+
return state === "queued" || state === "running";
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function releaseActiveRunIndex(asyncDir: string): void {
|
|
20
|
+
try {
|
|
21
|
+
fs.rmSync(markerPath(asyncDir));
|
|
22
|
+
} catch (error) {
|
|
23
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function updateActiveRunIndex(asyncDir: string, state: AsyncStatus["state"]): void {
|
|
28
|
+
const marker = markerPath(asyncDir);
|
|
29
|
+
if (isActiveAsyncState(state)) {
|
|
30
|
+
fs.mkdirSync(path.dirname(marker), { recursive: true });
|
|
31
|
+
fs.writeFileSync(marker, "", { flag: "a" });
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
releaseActiveRunIndex(asyncDir);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function readActiveRunIndex(asyncDirRoot: string): string[] | undefined {
|
|
38
|
+
try {
|
|
39
|
+
return fs.readdirSync(indexDir(asyncDirRoot), { withFileTypes: true })
|
|
40
|
+
.filter((entry) => entry.isFile())
|
|
41
|
+
.map((entry) => entry.name);
|
|
42
|
+
} catch (error) {
|
|
43
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|