triflux 10.2.0 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +236 -156
- package/hub/bridge.mjs +638 -290
- package/hub/codex-compat.mjs +1 -1
- package/hub/fullcycle.mjs +1 -1
- package/hub/intent.mjs +1 -0
- package/hub/lib/mcp-response-cache.mjs +205 -0
- package/hub/pipe.mjs +228 -119
- package/hub/reflexion.mjs +87 -13
- package/hub/research.mjs +1 -0
- package/hub/server.mjs +997 -611
- package/hub/team/ansi.mjs +1 -1
- package/hub/team/conductor-registry.mjs +121 -0
- package/hub/team/conductor.mjs +256 -125
- package/hub/team/execution-mode.mjs +105 -0
- package/hub/team/headless.mjs +686 -252
- package/hub/team/lead-control.mjs +91 -4
- package/hub/team/mcp-selector.mjs +145 -0
- package/hub/team/session-sync.mjs +153 -6
- package/hub/team/swarm-hypervisor.mjs +208 -86
- package/hub/team/tui-lite.mjs +18 -2
- package/hub/token-mode.mjs +1 -0
- package/hub/tools.mjs +474 -252
- package/package.json +5 -5
- package/scripts/codex-gateway-preflight.mjs +133 -0
- package/scripts/codex-mcp-gateway-sync.mjs +199 -0
- package/skills/star-prompt/SKILL.md +169 -69
- package/skills/tfx-setup/SKILL.md +124 -0
- package/skills/tfx-swarm/SKILL.md +124 -72
package/hub/team/conductor.mjs
CHANGED
|
@@ -8,43 +8,55 @@
|
|
|
8
8
|
// 3. Auto-restart (maxRestarts=3)
|
|
9
9
|
// 4. JSONL event log (블랙박스 리코더)
|
|
10
10
|
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import { broker } from
|
|
11
|
+
import { execFile, spawn } from "node:child_process";
|
|
12
|
+
import { EventEmitter } from "node:events";
|
|
13
|
+
import {
|
|
14
|
+
copyFileSync,
|
|
15
|
+
createWriteStream,
|
|
16
|
+
mkdirSync,
|
|
17
|
+
readFileSync,
|
|
18
|
+
} from "node:fs";
|
|
19
|
+
import { homedir } from "node:os";
|
|
20
|
+
import { dirname, join } from "node:path";
|
|
21
|
+
import { createRegistry } from "../../mesh/mesh-registry.mjs";
|
|
22
|
+
import { broker } from "../account-broker.mjs";
|
|
23
|
+
import { killProcess } from "../platform.mjs";
|
|
24
|
+
import { createConductorMeshBridge } from "./conductor-mesh-bridge.mjs";
|
|
25
|
+
import { getConductorRegistry } from "./conductor-registry.mjs";
|
|
26
|
+
import { createEventLog } from "./event-log.mjs";
|
|
27
|
+
import { createHealthProbe } from "./health-probe.mjs";
|
|
28
|
+
import { buildLauncher } from "./launcher-template.mjs";
|
|
29
|
+
import { createRemoteProbe } from "./remote-probe.mjs";
|
|
23
30
|
|
|
24
31
|
/** 세션 상태 */
|
|
25
32
|
export const STATES = Object.freeze({
|
|
26
|
-
INIT:
|
|
27
|
-
STARTING:
|
|
28
|
-
HEALTHY:
|
|
29
|
-
STALLED:
|
|
30
|
-
INPUT_WAIT:
|
|
31
|
-
FAILED:
|
|
32
|
-
RESTARTING:
|
|
33
|
-
DEAD:
|
|
34
|
-
COMPLETED:
|
|
33
|
+
INIT: "init",
|
|
34
|
+
STARTING: "starting",
|
|
35
|
+
HEALTHY: "healthy",
|
|
36
|
+
STALLED: "stalled",
|
|
37
|
+
INPUT_WAIT: "input_wait",
|
|
38
|
+
FAILED: "failed",
|
|
39
|
+
RESTARTING: "restarting",
|
|
40
|
+
DEAD: "dead",
|
|
41
|
+
COMPLETED: "completed",
|
|
35
42
|
});
|
|
36
43
|
|
|
37
44
|
/** 유효한 상태 전이 테이블 */
|
|
38
45
|
const TRANSITIONS = Object.freeze({
|
|
39
|
-
[STATES.INIT]:
|
|
40
|
-
[STATES.STARTING]:
|
|
41
|
-
[STATES.HEALTHY]:
|
|
42
|
-
|
|
46
|
+
[STATES.INIT]: [STATES.STARTING],
|
|
47
|
+
[STATES.STARTING]: [STATES.HEALTHY, STATES.FAILED],
|
|
48
|
+
[STATES.HEALTHY]: [
|
|
49
|
+
STATES.STALLED,
|
|
50
|
+
STATES.INPUT_WAIT,
|
|
51
|
+
STATES.FAILED,
|
|
52
|
+
STATES.COMPLETED,
|
|
53
|
+
],
|
|
54
|
+
[STATES.STALLED]: [STATES.HEALTHY, STATES.FAILED],
|
|
43
55
|
[STATES.INPUT_WAIT]: [STATES.HEALTHY, STATES.FAILED],
|
|
44
|
-
[STATES.FAILED]:
|
|
56
|
+
[STATES.FAILED]: [STATES.RESTARTING, STATES.DEAD],
|
|
45
57
|
[STATES.RESTARTING]: [STATES.STARTING],
|
|
46
|
-
[STATES.DEAD]:
|
|
47
|
-
[STATES.COMPLETED]:
|
|
58
|
+
[STATES.DEAD]: [],
|
|
59
|
+
[STATES.COMPLETED]: [],
|
|
48
60
|
});
|
|
49
61
|
|
|
50
62
|
const TERMINAL_STATES = new Set([STATES.DEAD, STATES.COMPLETED]);
|
|
@@ -68,15 +80,16 @@ export function createConductor(opts = {}) {
|
|
|
68
80
|
probeOpts = {},
|
|
69
81
|
} = opts;
|
|
70
82
|
|
|
71
|
-
if (!logsDir) throw new Error(
|
|
83
|
+
if (!logsDir) throw new Error("logsDir is required");
|
|
72
84
|
mkdirSync(logsDir, { recursive: true });
|
|
73
85
|
|
|
74
86
|
const emitter = new EventEmitter();
|
|
75
87
|
const sessions = new Map();
|
|
76
88
|
let shuttingDown = false;
|
|
89
|
+
let publicApi = null;
|
|
77
90
|
|
|
78
91
|
// 공유 event log (모든 세션 이벤트를 하나의 JSONL에)
|
|
79
|
-
const eventLog = createEventLog(join(logsDir,
|
|
92
|
+
const eventLog = createEventLog(join(logsDir, "conductor-events.jsonl"));
|
|
80
93
|
|
|
81
94
|
/**
|
|
82
95
|
* 세션 상태 전이.
|
|
@@ -84,10 +97,10 @@ export function createConductor(opts = {}) {
|
|
|
84
97
|
* @param {string} nextState
|
|
85
98
|
* @param {string} [reason]
|
|
86
99
|
*/
|
|
87
|
-
function transition(session, nextState, reason =
|
|
100
|
+
function transition(session, nextState, reason = "") {
|
|
88
101
|
const valid = TRANSITIONS[session.state] || [];
|
|
89
102
|
if (!valid.includes(nextState)) {
|
|
90
|
-
eventLog.append(
|
|
103
|
+
eventLog.append("invalid_transition", {
|
|
91
104
|
session: session.id,
|
|
92
105
|
from: session.state,
|
|
93
106
|
to: nextState,
|
|
@@ -99,7 +112,7 @@ export function createConductor(opts = {}) {
|
|
|
99
112
|
const prev = session.state;
|
|
100
113
|
session.state = nextState;
|
|
101
114
|
|
|
102
|
-
eventLog.append(
|
|
115
|
+
eventLog.append("stateChange", {
|
|
103
116
|
session: session.id,
|
|
104
117
|
from: prev,
|
|
105
118
|
to: nextState,
|
|
@@ -107,11 +120,17 @@ export function createConductor(opts = {}) {
|
|
|
107
120
|
restarts: session.restarts,
|
|
108
121
|
});
|
|
109
122
|
|
|
110
|
-
emitter.emit(
|
|
123
|
+
emitter.emit("stateChange", {
|
|
124
|
+
sessionId: session.id,
|
|
125
|
+
from: prev,
|
|
126
|
+
to: nextState,
|
|
127
|
+
reason,
|
|
128
|
+
});
|
|
111
129
|
|
|
112
130
|
// Terminal state cleanup
|
|
113
131
|
if (TERMINAL_STATES.has(nextState)) {
|
|
114
132
|
session.probe?.stop();
|
|
133
|
+
getConductorRegistry()?.unregister?.(session.id, publicApi);
|
|
115
134
|
}
|
|
116
135
|
|
|
117
136
|
return true;
|
|
@@ -123,7 +142,12 @@ export function createConductor(opts = {}) {
|
|
|
123
142
|
*/
|
|
124
143
|
function forceKill(pid) {
|
|
125
144
|
if (!pid || pid <= 0) return;
|
|
126
|
-
killProcess(pid, {
|
|
145
|
+
killProcess(pid, {
|
|
146
|
+
signal: "SIGKILL",
|
|
147
|
+
tree: true,
|
|
148
|
+
force: true,
|
|
149
|
+
timeout: 5000,
|
|
150
|
+
});
|
|
127
151
|
}
|
|
128
152
|
|
|
129
153
|
/**
|
|
@@ -137,19 +161,34 @@ export function createConductor(opts = {}) {
|
|
|
137
161
|
let sshIp = host;
|
|
138
162
|
// hosts.json에서 ssh_user/IP 해결
|
|
139
163
|
try {
|
|
140
|
-
const hostsPath = join(
|
|
141
|
-
|
|
164
|
+
const hostsPath = join(
|
|
165
|
+
opts.repoRoot || process.cwd(),
|
|
166
|
+
"references",
|
|
167
|
+
"hosts.json",
|
|
168
|
+
);
|
|
169
|
+
const hosts = JSON.parse(readFileSync(hostsPath, "utf8"));
|
|
142
170
|
const hostCfg = hosts.hosts?.[host];
|
|
143
171
|
if (hostCfg) {
|
|
144
172
|
sshUser = sshUser || hostCfg.ssh_user;
|
|
145
173
|
sshIp = hostCfg.tailscale?.ip || host;
|
|
146
174
|
}
|
|
147
|
-
} catch {
|
|
175
|
+
} catch {
|
|
176
|
+
/* hosts.json 없으면 fallback */
|
|
177
|
+
}
|
|
148
178
|
if (!sshUser) return;
|
|
149
179
|
const execFn = opts.deps?.execFile || execFile;
|
|
150
|
-
execFn(
|
|
151
|
-
|
|
152
|
-
|
|
180
|
+
execFn(
|
|
181
|
+
"ssh",
|
|
182
|
+
[`${sshUser}@${sshIp}`, "psmux", "kill-session", "-t", session.id],
|
|
183
|
+
{ timeout: 10_000 },
|
|
184
|
+
() => {},
|
|
185
|
+
);
|
|
186
|
+
eventLog.append("remote_kill", {
|
|
187
|
+
session: session.id,
|
|
188
|
+
host,
|
|
189
|
+
sshUser,
|
|
190
|
+
sshIp,
|
|
191
|
+
});
|
|
153
192
|
}
|
|
154
193
|
|
|
155
194
|
/**
|
|
@@ -172,7 +211,11 @@ export function createConductor(opts = {}) {
|
|
|
172
211
|
if (!pid) return;
|
|
173
212
|
|
|
174
213
|
// SIGTERM 먼저
|
|
175
|
-
try {
|
|
214
|
+
try {
|
|
215
|
+
child.kill("SIGTERM");
|
|
216
|
+
} catch {
|
|
217
|
+
/* already dead */
|
|
218
|
+
}
|
|
176
219
|
|
|
177
220
|
// Grace period 대기
|
|
178
221
|
await new Promise((resolve) => {
|
|
@@ -181,7 +224,7 @@ export function createConductor(opts = {}) {
|
|
|
181
224
|
resolve();
|
|
182
225
|
}, graceMs);
|
|
183
226
|
timer.unref?.();
|
|
184
|
-
child.once(
|
|
227
|
+
child.once("exit", () => {
|
|
185
228
|
clearTimeout(timer);
|
|
186
229
|
resolve();
|
|
187
230
|
});
|
|
@@ -193,26 +236,27 @@ export function createConductor(opts = {}) {
|
|
|
193
236
|
*/
|
|
194
237
|
function handleProbeResult(session, result) {
|
|
195
238
|
if (TERMINAL_STATES.has(session.state)) return;
|
|
196
|
-
if (session.state === STATES.INIT || session.state === STATES.RESTARTING)
|
|
239
|
+
if (session.state === STATES.INIT || session.state === STATES.RESTARTING)
|
|
240
|
+
return;
|
|
197
241
|
|
|
198
|
-
eventLog.append(
|
|
242
|
+
eventLog.append("health", {
|
|
199
243
|
session: session.id,
|
|
200
244
|
...result,
|
|
201
245
|
});
|
|
202
246
|
|
|
203
247
|
// L0 실패 — 로컬: exit handler에서 처리. 원격: probe가 유일한 감지 수단.
|
|
204
|
-
if (result.l0 ===
|
|
248
|
+
if (result.l0 === "fail") {
|
|
205
249
|
if (session.config.remote) {
|
|
206
|
-
handleFailure(session,
|
|
250
|
+
handleFailure(session, "remote_L0_fail");
|
|
207
251
|
}
|
|
208
252
|
return;
|
|
209
253
|
}
|
|
210
254
|
|
|
211
255
|
// L3 completed (원격 완료 토큰 감지)
|
|
212
|
-
if (result.l3 ===
|
|
213
|
-
transition(session, STATES.COMPLETED,
|
|
214
|
-
emitter.emit(
|
|
215
|
-
if (typeof session.config.onCompleted ===
|
|
256
|
+
if (result.l3 === "completed" && session.config.remote) {
|
|
257
|
+
transition(session, STATES.COMPLETED, "remote_completion_token");
|
|
258
|
+
emitter.emit("completed", { sessionId: session.id });
|
|
259
|
+
if (typeof session.config.onCompleted === "function") {
|
|
216
260
|
session.config.onCompleted({ sessionId: session.id });
|
|
217
261
|
}
|
|
218
262
|
maybeAutoShutdown();
|
|
@@ -220,9 +264,13 @@ export function createConductor(opts = {}) {
|
|
|
220
264
|
}
|
|
221
265
|
|
|
222
266
|
// L1 INPUT_WAIT 감지
|
|
223
|
-
if (result.l1 ===
|
|
224
|
-
transition(
|
|
225
|
-
|
|
267
|
+
if (result.l1 === "input_wait" && session.state === STATES.HEALTHY) {
|
|
268
|
+
transition(
|
|
269
|
+
session,
|
|
270
|
+
STATES.INPUT_WAIT,
|
|
271
|
+
`input_wait:${result.inputWaitPattern}`,
|
|
272
|
+
);
|
|
273
|
+
emitter.emit("inputWait", {
|
|
226
274
|
sessionId: session.id,
|
|
227
275
|
pattern: result.inputWaitPattern,
|
|
228
276
|
});
|
|
@@ -230,32 +278,36 @@ export function createConductor(opts = {}) {
|
|
|
230
278
|
}
|
|
231
279
|
|
|
232
280
|
// INPUT_WAIT → output 재개 시 HEALTHY 복귀
|
|
233
|
-
if (session.state === STATES.INPUT_WAIT && result.l1 ===
|
|
234
|
-
transition(session, STATES.HEALTHY,
|
|
281
|
+
if (session.state === STATES.INPUT_WAIT && result.l1 === "ok") {
|
|
282
|
+
transition(session, STATES.HEALTHY, "output_resumed");
|
|
235
283
|
return;
|
|
236
284
|
}
|
|
237
285
|
|
|
238
286
|
// L1 stall
|
|
239
|
-
if (result.l1 ===
|
|
240
|
-
transition(session, STATES.STALLED,
|
|
287
|
+
if (result.l1 === "stall" && session.state === STATES.HEALTHY) {
|
|
288
|
+
transition(session, STATES.STALLED, "L1_stall");
|
|
241
289
|
return;
|
|
242
290
|
}
|
|
243
291
|
|
|
244
292
|
// STALLED → output 재개 시 HEALTHY 복귀
|
|
245
|
-
if (session.state === STATES.STALLED && result.l1 ===
|
|
246
|
-
transition(session, STATES.HEALTHY,
|
|
293
|
+
if (session.state === STATES.STALLED && result.l1 === "ok") {
|
|
294
|
+
transition(session, STATES.HEALTHY, "output_resumed");
|
|
247
295
|
return;
|
|
248
296
|
}
|
|
249
297
|
|
|
250
298
|
// L3 timeout (아직 STARTING 상태)
|
|
251
|
-
if (result.l3 ===
|
|
252
|
-
handleFailure(session,
|
|
299
|
+
if (result.l3 === "timeout" && session.state === STATES.STARTING) {
|
|
300
|
+
handleFailure(session, "L3_timeout");
|
|
253
301
|
return;
|
|
254
302
|
}
|
|
255
303
|
|
|
256
304
|
// STARTING → L0 ok + L3 ok → HEALTHY
|
|
257
|
-
if (
|
|
258
|
-
|
|
305
|
+
if (
|
|
306
|
+
session.state === STATES.STARTING &&
|
|
307
|
+
result.l0 === "ok" &&
|
|
308
|
+
result.l3 === "ok"
|
|
309
|
+
) {
|
|
310
|
+
transition(session, STATES.HEALTHY, "probe_healthy");
|
|
259
311
|
return;
|
|
260
312
|
}
|
|
261
313
|
|
|
@@ -271,17 +323,24 @@ export function createConductor(opts = {}) {
|
|
|
271
323
|
transition(session, STATES.FAILED, reason);
|
|
272
324
|
|
|
273
325
|
if (session.restarts < maxRestarts) {
|
|
274
|
-
transition(
|
|
326
|
+
transition(
|
|
327
|
+
session,
|
|
328
|
+
STATES.RESTARTING,
|
|
329
|
+
`restart_${session.restarts + 1}/${maxRestarts}`,
|
|
330
|
+
);
|
|
275
331
|
session.restarts += 1;
|
|
276
332
|
void respawnSession(session);
|
|
277
333
|
} else {
|
|
278
334
|
transition(session, STATES.DEAD, `maxRestarts(${maxRestarts})_exceeded`);
|
|
279
|
-
emitter.emit(
|
|
335
|
+
emitter.emit("dead", { sessionId: session.id, reason });
|
|
280
336
|
|
|
281
337
|
// broker release on final death
|
|
282
338
|
if (broker && session.config.accountId) {
|
|
283
|
-
broker.release(session.config.accountId, {
|
|
284
|
-
|
|
339
|
+
broker.release(session.config.accountId, {
|
|
340
|
+
ok: false,
|
|
341
|
+
failureMode: session.lastFailureMode,
|
|
342
|
+
});
|
|
343
|
+
if (session.lastFailureMode === "rate_limited") {
|
|
285
344
|
broker.markRateLimited(session.config.accountId, 5 * 60 * 1000);
|
|
286
345
|
}
|
|
287
346
|
}
|
|
@@ -295,29 +354,36 @@ export function createConductor(opts = {}) {
|
|
|
295
354
|
// 기존 child 정리
|
|
296
355
|
await cleanupChild(session);
|
|
297
356
|
|
|
298
|
-
transition(
|
|
357
|
+
transition(
|
|
358
|
+
session,
|
|
359
|
+
STATES.STARTING,
|
|
360
|
+
session.restarts > 0 ? "respawn" : "initial",
|
|
361
|
+
);
|
|
299
362
|
|
|
300
363
|
const launcher = session.launcher;
|
|
301
364
|
const outPath = join(logsDir, `${session.id}.out.log`);
|
|
302
365
|
const errPath = join(logsDir, `${session.id}.err.log`);
|
|
303
366
|
mkdirSync(logsDir, { recursive: true });
|
|
304
367
|
|
|
305
|
-
const outWs = createWriteStream(outPath, { flags:
|
|
306
|
-
const errWs = createWriteStream(errPath, { flags:
|
|
368
|
+
const outWs = createWriteStream(outPath, { flags: "a" });
|
|
369
|
+
const errWs = createWriteStream(errPath, { flags: "a" });
|
|
307
370
|
|
|
308
371
|
let outputBytes = 0;
|
|
309
|
-
let recentOutput =
|
|
372
|
+
let recentOutput = "";
|
|
310
373
|
|
|
311
374
|
let child;
|
|
312
375
|
try {
|
|
313
376
|
child = spawn(launcher.command, {
|
|
314
377
|
shell: true,
|
|
315
378
|
env: { ...process.env, ...launcher.env, ...(session.config.env || {}) },
|
|
316
|
-
stdio: [
|
|
379
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
317
380
|
windowsHide: true,
|
|
318
381
|
});
|
|
319
382
|
} catch (err) {
|
|
320
|
-
eventLog.append(
|
|
383
|
+
eventLog.append("spawn_error", {
|
|
384
|
+
session: session.id,
|
|
385
|
+
error: err.message,
|
|
386
|
+
});
|
|
321
387
|
handleFailure(session, `spawn_error:${err.message}`);
|
|
322
388
|
return;
|
|
323
389
|
}
|
|
@@ -326,7 +392,7 @@ export function createConductor(opts = {}) {
|
|
|
326
392
|
session.outPath = outPath;
|
|
327
393
|
session.errPath = errPath;
|
|
328
394
|
|
|
329
|
-
eventLog.append(
|
|
395
|
+
eventLog.append("spawn", {
|
|
330
396
|
session: session.id,
|
|
331
397
|
agent: session.config.agent,
|
|
332
398
|
pid: child.pid,
|
|
@@ -345,15 +411,29 @@ export function createConductor(opts = {}) {
|
|
|
345
411
|
}
|
|
346
412
|
};
|
|
347
413
|
|
|
348
|
-
child.stdout?.on(
|
|
349
|
-
|
|
414
|
+
child.stdout?.on("data", (buf) => {
|
|
415
|
+
outWs.write(buf);
|
|
416
|
+
trackOutput(buf);
|
|
417
|
+
});
|
|
418
|
+
child.stderr?.on("data", (buf) => {
|
|
419
|
+
errWs.write(buf);
|
|
420
|
+
trackOutput(buf);
|
|
421
|
+
});
|
|
350
422
|
|
|
351
|
-
child.on(
|
|
423
|
+
child.on("exit", (code, signal) => {
|
|
352
424
|
session.alive = false;
|
|
353
|
-
try {
|
|
354
|
-
|
|
425
|
+
try {
|
|
426
|
+
outWs.end();
|
|
427
|
+
} catch {
|
|
428
|
+
/* ignore */
|
|
429
|
+
}
|
|
430
|
+
try {
|
|
431
|
+
errWs.end();
|
|
432
|
+
} catch {
|
|
433
|
+
/* ignore */
|
|
434
|
+
}
|
|
355
435
|
|
|
356
|
-
eventLog.append(
|
|
436
|
+
eventLog.append("exit", {
|
|
357
437
|
session: session.id,
|
|
358
438
|
code,
|
|
359
439
|
signal,
|
|
@@ -363,9 +443,9 @@ export function createConductor(opts = {}) {
|
|
|
363
443
|
if (TERMINAL_STATES.has(session.state)) return;
|
|
364
444
|
|
|
365
445
|
if (code === 0 && !signal) {
|
|
366
|
-
transition(session, STATES.COMPLETED,
|
|
367
|
-
emitter.emit(
|
|
368
|
-
if (typeof session.config.onCompleted ===
|
|
446
|
+
transition(session, STATES.COMPLETED, "exit_0");
|
|
447
|
+
emitter.emit("completed", { sessionId: session.id });
|
|
448
|
+
if (typeof session.config.onCompleted === "function") {
|
|
369
449
|
session.config.onCompleted({ sessionId: session.id });
|
|
370
450
|
}
|
|
371
451
|
if (broker && session.config.accountId) {
|
|
@@ -373,8 +453,12 @@ export function createConductor(opts = {}) {
|
|
|
373
453
|
}
|
|
374
454
|
} else {
|
|
375
455
|
// detect rate_limited from recent output before handleFailure
|
|
376
|
-
if (
|
|
377
|
-
|
|
456
|
+
if (
|
|
457
|
+
/(rate.?limit|quota|throttl|too.many.requests|429|usage.limit)/iu.test(
|
|
458
|
+
recentOutput,
|
|
459
|
+
)
|
|
460
|
+
) {
|
|
461
|
+
session.lastFailureMode = "rate_limited";
|
|
378
462
|
}
|
|
379
463
|
handleFailure(session, `exit_code:${code},signal:${signal}`);
|
|
380
464
|
}
|
|
@@ -382,9 +466,12 @@ export function createConductor(opts = {}) {
|
|
|
382
466
|
maybeAutoShutdown();
|
|
383
467
|
});
|
|
384
468
|
|
|
385
|
-
child.on(
|
|
469
|
+
child.on("error", (err) => {
|
|
386
470
|
session.alive = false;
|
|
387
|
-
eventLog.append(
|
|
471
|
+
eventLog.append("child_error", {
|
|
472
|
+
session: session.id,
|
|
473
|
+
error: err.message,
|
|
474
|
+
});
|
|
388
475
|
if (!TERMINAL_STATES.has(session.state)) {
|
|
389
476
|
handleFailure(session, `child_error:${err.message}`);
|
|
390
477
|
}
|
|
@@ -396,8 +483,12 @@ export function createConductor(opts = {}) {
|
|
|
396
483
|
session.probe?.stop();
|
|
397
484
|
const probe = createHealthProbe(
|
|
398
485
|
{
|
|
399
|
-
get pid() {
|
|
400
|
-
|
|
486
|
+
get pid() {
|
|
487
|
+
return child.pid;
|
|
488
|
+
},
|
|
489
|
+
get alive() {
|
|
490
|
+
return session.alive;
|
|
491
|
+
},
|
|
401
492
|
getOutputBytes: () => outputBytes,
|
|
402
493
|
getRecentOutput: () => recentOutput,
|
|
403
494
|
},
|
|
@@ -415,13 +506,13 @@ export function createConductor(opts = {}) {
|
|
|
415
506
|
* 원격 세션은 remote-spawn.mjs가 이미 psmux 세션을 생성한 상태를 가정.
|
|
416
507
|
*/
|
|
417
508
|
function startRemoteSession(session) {
|
|
418
|
-
transition(session, STATES.STARTING,
|
|
509
|
+
transition(session, STATES.STARTING, "remote_initial");
|
|
419
510
|
|
|
420
511
|
const { host, paneTarget, sessionName } = session.config;
|
|
421
512
|
const resolvedPane = paneTarget || `${sessionName || session.id}:0.0`;
|
|
422
513
|
const resolvedSessionName = sessionName || session.id;
|
|
423
514
|
|
|
424
|
-
eventLog.append(
|
|
515
|
+
eventLog.append("remote_start", {
|
|
425
516
|
session: session.id,
|
|
426
517
|
host,
|
|
427
518
|
paneTarget: resolvedPane,
|
|
@@ -452,11 +543,11 @@ export function createConductor(opts = {}) {
|
|
|
452
543
|
*/
|
|
453
544
|
function maybeAutoShutdown() {
|
|
454
545
|
if (shuttingDown) return;
|
|
455
|
-
const allTerminal = [...sessions.values()].every(
|
|
456
|
-
|
|
546
|
+
const allTerminal = [...sessions.values()].every((s) =>
|
|
547
|
+
TERMINAL_STATES.has(s.state),
|
|
457
548
|
);
|
|
458
549
|
if (allTerminal && sessions.size > 0) {
|
|
459
|
-
emitter.emit(
|
|
550
|
+
emitter.emit("allCompleted");
|
|
460
551
|
}
|
|
461
552
|
}
|
|
462
553
|
|
|
@@ -479,10 +570,12 @@ export function createConductor(opts = {}) {
|
|
|
479
570
|
* @returns {string} session ID
|
|
480
571
|
*/
|
|
481
572
|
function spawnSession(config) {
|
|
482
|
-
if (shuttingDown) throw new Error(
|
|
483
|
-
if (!config.id) throw new Error(
|
|
484
|
-
if (sessions.has(config.id))
|
|
485
|
-
|
|
573
|
+
if (shuttingDown) throw new Error("Conductor is shutting down");
|
|
574
|
+
if (!config.id) throw new Error("session id is required");
|
|
575
|
+
if (sessions.has(config.id))
|
|
576
|
+
throw new Error(`Session "${config.id}" already exists`);
|
|
577
|
+
if (config.remote && !config.host)
|
|
578
|
+
throw new Error("host is required for remote sessions");
|
|
486
579
|
|
|
487
580
|
// broker lease (graceful — broker null if accounts.json absent)
|
|
488
581
|
let lease = null;
|
|
@@ -490,10 +583,10 @@ export function createConductor(opts = {}) {
|
|
|
490
583
|
lease = broker.lease({ provider: config.agent });
|
|
491
584
|
if (lease === null) {
|
|
492
585
|
const eta = broker.nextAvailableEta(config.agent);
|
|
493
|
-
eventLog.append(
|
|
586
|
+
eventLog.append("broker_no_lease", {
|
|
494
587
|
session: config.id,
|
|
495
588
|
agent: config.agent,
|
|
496
|
-
eta: eta ? new Date(eta).toISOString() :
|
|
589
|
+
eta: eta ? new Date(eta).toISOString() : "unknown",
|
|
497
590
|
});
|
|
498
591
|
// 계정이 모두 cooldown이어도 세션 생성 자체는 유지한다.
|
|
499
592
|
// 로컬 테스트/단일 계정 없는 환경에서도 상태 머신이 일관되게 동작해야 한다.
|
|
@@ -511,20 +604,29 @@ export function createConductor(opts = {}) {
|
|
|
511
604
|
: config;
|
|
512
605
|
|
|
513
606
|
// auth file copy — broker resolved absolute path, conductor does the actual copy
|
|
514
|
-
if (lease?.mode ===
|
|
515
|
-
const dests =
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
607
|
+
if (lease?.mode === "auth" && lease.authFile) {
|
|
608
|
+
const dests =
|
|
609
|
+
config.agent === "codex"
|
|
610
|
+
? [join(homedir(), ".codex", "auth.json")]
|
|
611
|
+
: [
|
|
612
|
+
join(homedir(), ".gemini", "oauth_creds.json"),
|
|
613
|
+
join(homedir(), ".gemini", "gemini-credentials.json"),
|
|
614
|
+
];
|
|
521
615
|
for (const dest of dests) {
|
|
522
616
|
try {
|
|
523
617
|
mkdirSync(dirname(dest), { recursive: true });
|
|
524
618
|
copyFileSync(lease.authFile, dest);
|
|
525
|
-
eventLog.append(
|
|
619
|
+
eventLog.append("auth_copy", {
|
|
620
|
+
session: config.id,
|
|
621
|
+
agent: config.agent,
|
|
622
|
+
dest,
|
|
623
|
+
});
|
|
526
624
|
} catch (err) {
|
|
527
|
-
eventLog.append(
|
|
625
|
+
eventLog.append("auth_copy_error", {
|
|
626
|
+
session: config.id,
|
|
627
|
+
dest,
|
|
628
|
+
error: err.message,
|
|
629
|
+
});
|
|
528
630
|
}
|
|
529
631
|
}
|
|
530
632
|
}
|
|
@@ -555,6 +657,7 @@ export function createConductor(opts = {}) {
|
|
|
555
657
|
};
|
|
556
658
|
|
|
557
659
|
sessions.set(resolvedConfig.id, session);
|
|
660
|
+
getConductorRegistry()?.register?.(resolvedConfig.id, publicApi);
|
|
558
661
|
|
|
559
662
|
if (resolvedConfig.remote) {
|
|
560
663
|
startRemoteSession(session);
|
|
@@ -569,12 +672,12 @@ export function createConductor(opts = {}) {
|
|
|
569
672
|
* @param {string} id
|
|
570
673
|
* @param {string} [reason]
|
|
571
674
|
*/
|
|
572
|
-
async function killSession(id, reason =
|
|
675
|
+
async function killSession(id, reason = "user_kill") {
|
|
573
676
|
const session = sessions.get(id);
|
|
574
677
|
if (!session) return;
|
|
575
678
|
if (TERMINAL_STATES.has(session.state)) return;
|
|
576
679
|
|
|
577
|
-
eventLog.append(
|
|
680
|
+
eventLog.append("kill", { session: id, reason });
|
|
578
681
|
await cleanupChild(session);
|
|
579
682
|
transition(session, STATES.FAILED, reason);
|
|
580
683
|
transition(session, STATES.DEAD, reason);
|
|
@@ -591,14 +694,14 @@ export function createConductor(opts = {}) {
|
|
|
591
694
|
|
|
592
695
|
// 원격 세션 — stdin 미지원 (psmux send-keys는 별도 경로)
|
|
593
696
|
if (session.config.remote) {
|
|
594
|
-
eventLog.append(
|
|
697
|
+
eventLog.append("stdin_remote_unsupported", { session: id });
|
|
595
698
|
return false;
|
|
596
699
|
}
|
|
597
700
|
|
|
598
701
|
if (!session.child) return false;
|
|
599
702
|
try {
|
|
600
703
|
session.child.stdin.write(`${text}\n`);
|
|
601
|
-
eventLog.append(
|
|
704
|
+
eventLog.append("stdin", { session: id, text: text.slice(0, 100) });
|
|
602
705
|
return true;
|
|
603
706
|
} catch {
|
|
604
707
|
return false;
|
|
@@ -628,11 +731,11 @@ export function createConductor(opts = {}) {
|
|
|
628
731
|
/**
|
|
629
732
|
* Graceful shutdown — 전체 세션 종료.
|
|
630
733
|
*/
|
|
631
|
-
async function shutdown(reason =
|
|
734
|
+
async function shutdown(reason = "shutdown") {
|
|
632
735
|
if (shuttingDown) return;
|
|
633
736
|
shuttingDown = true;
|
|
634
737
|
|
|
635
|
-
eventLog.append(
|
|
738
|
+
eventLog.append("shutdown", { reason, sessions: sessions.size });
|
|
636
739
|
|
|
637
740
|
const cleanups = [...sessions.values()]
|
|
638
741
|
.filter((s) => !TERMINAL_STATES.has(s.state))
|
|
@@ -646,26 +749,54 @@ export function createConductor(opts = {}) {
|
|
|
646
749
|
});
|
|
647
750
|
|
|
648
751
|
await Promise.allSettled(cleanups);
|
|
752
|
+
if (conductor._meshBridge) conductor._meshBridge.detach();
|
|
649
753
|
await eventLog.flush();
|
|
650
754
|
await eventLog.close();
|
|
651
|
-
emitter.emit(
|
|
755
|
+
emitter.emit("shutdown");
|
|
652
756
|
}
|
|
653
757
|
|
|
654
758
|
// Shutdown traps
|
|
655
|
-
const onSignal = () => {
|
|
656
|
-
|
|
657
|
-
|
|
759
|
+
const onSignal = () => {
|
|
760
|
+
void shutdown("signal");
|
|
761
|
+
};
|
|
762
|
+
process.on("SIGINT", onSignal);
|
|
763
|
+
process.on("SIGTERM", onSignal);
|
|
658
764
|
|
|
659
|
-
|
|
765
|
+
const conductor = {
|
|
660
766
|
spawnSession,
|
|
661
767
|
killSession,
|
|
662
768
|
sendInput,
|
|
663
769
|
getSnapshot,
|
|
770
|
+
getMeshRegistry() {
|
|
771
|
+
return this._meshRegistry || null;
|
|
772
|
+
},
|
|
664
773
|
shutdown,
|
|
665
774
|
on: emitter.on.bind(emitter),
|
|
666
775
|
off: emitter.off.bind(emitter),
|
|
667
|
-
get sessionCount() {
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
776
|
+
get sessionCount() {
|
|
777
|
+
return sessions.size;
|
|
778
|
+
},
|
|
779
|
+
get isShuttingDown() {
|
|
780
|
+
return shuttingDown;
|
|
781
|
+
},
|
|
782
|
+
get eventLogPath() {
|
|
783
|
+
return eventLog.filePath;
|
|
784
|
+
},
|
|
785
|
+
};
|
|
786
|
+
|
|
787
|
+
if (opts.enableMesh !== false) {
|
|
788
|
+
try {
|
|
789
|
+
const registry = opts.meshRegistry || createRegistry();
|
|
790
|
+
const bridge = createConductorMeshBridge(conductor, registry);
|
|
791
|
+
bridge.attach();
|
|
792
|
+
conductor._meshBridge = bridge;
|
|
793
|
+
conductor._meshRegistry = registry;
|
|
794
|
+
} catch {
|
|
795
|
+
// mesh 실패해도 conductor 정상 동작
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
const frozenApi = Object.freeze(conductor);
|
|
800
|
+
getConductorRegistry().register(frozenApi);
|
|
801
|
+
return frozenApi;
|
|
671
802
|
}
|