tono 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -3
- package/dist/cli/checks.js +504 -0
- package/dist/cli/checks.js.map +1 -0
- package/dist/cli/commands/config.js +20 -3
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/configure.js +1 -1
- package/dist/cli/commands/configure.js.map +1 -1
- package/dist/cli/commands/doctor.js +22 -0
- package/dist/cli/commands/doctor.js.map +1 -0
- package/dist/cli/commands/gateway.js +23 -10
- package/dist/cli/commands/gateway.js.map +1 -1
- package/dist/cli/commands/init.js +22 -3
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/start.js +72 -18
- package/dist/cli/commands/start.js.map +1 -1
- package/dist/cli/commands/worker.js +458 -0
- package/dist/cli/commands/worker.js.map +1 -0
- package/dist/cli/index.js +28 -9
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/launchd.js +5 -5
- package/dist/cli/launchd.js.map +1 -1
- package/dist/server/app.js +88 -13
- package/dist/server/app.js.map +1 -1
- package/dist/server/config/load.js +19 -1
- package/dist/server/config/load.js.map +1 -1
- package/dist/server/config/schema.json +19 -0
- package/dist/server/db/client.js +50 -1
- package/dist/server/db/client.js.map +1 -1
- package/dist/server/db/queries.js +126 -3
- package/dist/server/db/queries.js.map +1 -1
- package/dist/server/db/schema.sql +36 -21
- package/dist/server/events.js.map +1 -1
- package/dist/server/server.js +28 -11
- package/dist/server/server.js.map +1 -1
- package/dist/server/worker/agent.js +396 -0
- package/dist/server/worker/agent.js.map +1 -0
- package/dist/server/worker/pty-bridge.js +27 -0
- package/dist/server/worker/pty-bridge.js.map +1 -0
- package/dist/server/workers/github-poller.js +26 -8
- package/dist/server/workers/github-poller.js.map +1 -1
- package/dist/server/workers/reaper.js +44 -0
- package/dist/server/workers/reaper.js.map +1 -0
- package/dist/server/workers/registry.js +166 -0
- package/dist/server/workers/registry.js.map +1 -0
- package/dist/server/workers/scheduler.js +79 -130
- package/dist/server/workers/scheduler.js.map +1 -1
- package/dist/server/ws/pty.js +66 -54
- package/dist/server/ws/pty.js.map +1 -1
- package/dist/server/ws/workers.js +206 -0
- package/dist/server/ws/workers.js.map +1 -0
- package/dist/shared/types.js +4 -1
- package/dist/shared/types.js.map +1 -1
- package/dist/shared/worker-protocol.js +28 -0
- package/dist/shared/worker-protocol.js.map +1 -0
- package/dist/web/assets/{index-5VFn-lxF.js → index-DPB0x7SX.js} +21 -21
- package/dist/web/index.html +1 -1
- package/package.json +18 -26
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marks tasks failed when their owning worker stays disconnected past graceMs.
|
|
3
|
+
* Runs alongside the scheduler. Cheap to over-run; idempotent.
|
|
4
|
+
*/
|
|
5
|
+
export class Reaper {
|
|
6
|
+
opts;
|
|
7
|
+
timer = null;
|
|
8
|
+
log;
|
|
9
|
+
constructor(opts) {
|
|
10
|
+
this.opts = opts;
|
|
11
|
+
this.log = opts.log ?? console.log;
|
|
12
|
+
}
|
|
13
|
+
start() {
|
|
14
|
+
if (this.timer)
|
|
15
|
+
return;
|
|
16
|
+
const tickMs = this.opts.tickMs ?? Math.max(5_000, Math.floor(this.opts.graceMs / 2));
|
|
17
|
+
this.timer = setInterval(() => this.tick(), tickMs);
|
|
18
|
+
}
|
|
19
|
+
stop() {
|
|
20
|
+
if (this.timer)
|
|
21
|
+
clearInterval(this.timer);
|
|
22
|
+
this.timer = null;
|
|
23
|
+
}
|
|
24
|
+
tick() {
|
|
25
|
+
const cutoff = Date.now() - this.opts.graceMs;
|
|
26
|
+
const stale = this.opts.q.listDisconnectedSessions();
|
|
27
|
+
for (const s of stale) {
|
|
28
|
+
if (!s.disconnectedAt)
|
|
29
|
+
continue;
|
|
30
|
+
const at = Date.parse(s.disconnectedAt);
|
|
31
|
+
if (Number.isNaN(at) || at > cutoff)
|
|
32
|
+
continue;
|
|
33
|
+
this.log(`[reaper] session ${s.id} (task #${s.taskId}) past grace; failing task`);
|
|
34
|
+
this.opts.q.endSession(s.id);
|
|
35
|
+
this.opts.q.setTaskStatus(s.taskId, "failed", { exitCode: -3 });
|
|
36
|
+
this.opts.q.clearTaskAssignment(s.taskId);
|
|
37
|
+
this.opts.registry.forgetSession(s.id);
|
|
38
|
+
const updated = this.opts.q.getTask(s.taskId);
|
|
39
|
+
if (updated)
|
|
40
|
+
this.opts.bus.emit("task:updated", { task: updated });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=reaper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reaper.js","sourceRoot":"","sources":["../../../src/server/workers/reaper.ts"],"names":[],"mappings":"AAeA;;;GAGG;AACH,MAAM,OAAO,MAAM;IAIY;IAHrB,KAAK,GAA0B,IAAI,CAAC;IAC3B,GAAG,CAAwB;IAE5C,YAA6B,IAAmB;QAAnB,SAAI,GAAJ,IAAI,CAAe;QAC9C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACrC,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACtF,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAEO,IAAI;QACV,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,wBAAwB,EAAE,CAAC;QACrD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,cAAc;gBAAE,SAAS;YAChC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACxC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,MAAM;gBAAE,SAAS;YAC9C,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,MAAM,4BAA4B,CAAC,CAAC;YAClF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,OAAO;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { EventEmitter } from "node:events";
|
|
2
|
+
import { encodePtyFrame, } from "../../shared/worker-protocol.js";
|
|
3
|
+
import { RingBuffer } from "../pty/ring-buffer.js";
|
|
4
|
+
const SCROLLBACK_BYTES = 4 * 1024 * 1024;
|
|
5
|
+
/**
|
|
6
|
+
* In-memory registry of connected workers. The DB has the persistent worker
|
|
7
|
+
* row; this object holds the live WS connection and per-session fan-out.
|
|
8
|
+
*
|
|
9
|
+
* Scrollback for distributed sessions lives here on the gateway so multiple
|
|
10
|
+
* browser viewers can attach without round-tripping the worker for replay.
|
|
11
|
+
*/
|
|
12
|
+
export class WorkerRegistry extends EventEmitter {
|
|
13
|
+
conns = new Map();
|
|
14
|
+
sessions = new Map();
|
|
15
|
+
/** Sticky round-robin cursor per (agent|kind) to spread tasks evenly. */
|
|
16
|
+
rrCursor = 0;
|
|
17
|
+
register(conn) {
|
|
18
|
+
// If a worker reconnects with the same id, replace the old connection.
|
|
19
|
+
const existing = this.conns.get(conn.workerId);
|
|
20
|
+
if (existing && existing.ws !== conn.ws) {
|
|
21
|
+
try {
|
|
22
|
+
existing.ws.close(4000, "reconnected");
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
/* ignore */
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
this.conns.set(conn.workerId, conn);
|
|
29
|
+
for (const ref of conn.runningSessions.values()) {
|
|
30
|
+
this.ensureSession(ref.sessionId, conn.workerId, ref.taskId);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
unregister(workerId) {
|
|
34
|
+
this.conns.delete(workerId);
|
|
35
|
+
// Sessions remain in the map (with their buffers) so an in-flight browser
|
|
36
|
+
// viewer can still see scrollback. The reaper / scheduler decides when to
|
|
37
|
+
// drop them based on disconnect grace.
|
|
38
|
+
}
|
|
39
|
+
get(workerId) {
|
|
40
|
+
return this.conns.get(workerId);
|
|
41
|
+
}
|
|
42
|
+
list() {
|
|
43
|
+
return [...this.conns.values()];
|
|
44
|
+
}
|
|
45
|
+
/** Update capacity + running sessions from a heartbeat. */
|
|
46
|
+
updateHeartbeat(workerId, capacity, running) {
|
|
47
|
+
const conn = this.conns.get(workerId);
|
|
48
|
+
if (!conn)
|
|
49
|
+
return;
|
|
50
|
+
conn.capacity = capacity;
|
|
51
|
+
conn.lastHeartbeatAt = new Date();
|
|
52
|
+
conn.runningSessions = new Map(running.map((r) => [r.sessionId, r]));
|
|
53
|
+
for (const ref of running) {
|
|
54
|
+
this.ensureSession(ref.sessionId, workerId, ref.taskId);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Choose a worker that advertises the given agent and has free capacity for
|
|
59
|
+
* `kind`. Round-robins across eligible workers so no single box gets all
|
|
60
|
+
* the tasks when several are equally qualified.
|
|
61
|
+
*/
|
|
62
|
+
findEligible(agent, kind) {
|
|
63
|
+
const candidates = [];
|
|
64
|
+
for (const conn of this.conns.values()) {
|
|
65
|
+
if (!conn.capabilities.agents[agent])
|
|
66
|
+
continue;
|
|
67
|
+
const slot = conn.capacity.byAgent[agent];
|
|
68
|
+
if (!slot)
|
|
69
|
+
continue;
|
|
70
|
+
if (slot[kind] <= 0)
|
|
71
|
+
continue;
|
|
72
|
+
candidates.push(conn);
|
|
73
|
+
}
|
|
74
|
+
if (candidates.length === 0)
|
|
75
|
+
return null;
|
|
76
|
+
const pick = candidates[this.rrCursor % candidates.length];
|
|
77
|
+
this.rrCursor = (this.rrCursor + 1) % Math.max(1, candidates.length);
|
|
78
|
+
return pick;
|
|
79
|
+
}
|
|
80
|
+
// ---- per-session state ----
|
|
81
|
+
ensureSession(sessionId, workerId, taskId) {
|
|
82
|
+
let s = this.sessions.get(sessionId);
|
|
83
|
+
if (!s) {
|
|
84
|
+
s = {
|
|
85
|
+
workerId,
|
|
86
|
+
taskId,
|
|
87
|
+
cols: 120,
|
|
88
|
+
rows: 32,
|
|
89
|
+
buffer: new RingBuffer(SCROLLBACK_BYTES),
|
|
90
|
+
};
|
|
91
|
+
this.sessions.set(sessionId, s);
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
// Reattach: the worker reclaimed a session it already owns. Refresh ids.
|
|
95
|
+
s.workerId = workerId;
|
|
96
|
+
s.taskId = taskId;
|
|
97
|
+
}
|
|
98
|
+
return s;
|
|
99
|
+
}
|
|
100
|
+
getSession(sessionId) {
|
|
101
|
+
return this.sessions.get(sessionId);
|
|
102
|
+
}
|
|
103
|
+
forgetSession(sessionId) {
|
|
104
|
+
this.sessions.delete(sessionId);
|
|
105
|
+
}
|
|
106
|
+
/** Append PTY output bytes from the worker into the session ring + fan out. */
|
|
107
|
+
appendSessionData(sessionId, payload) {
|
|
108
|
+
let s = this.sessions.get(sessionId);
|
|
109
|
+
if (!s) {
|
|
110
|
+
// Late-arriving frame for a session we don't know about yet — start a
|
|
111
|
+
// placeholder so scrollback gets captured. The taskId is fixed up later
|
|
112
|
+
// when session.started arrives.
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
s.buffer.push(payload);
|
|
116
|
+
this.emit(`session:${sessionId}:data`, payload);
|
|
117
|
+
}
|
|
118
|
+
setSessionSize(sessionId, cols, rows) {
|
|
119
|
+
const s = this.sessions.get(sessionId);
|
|
120
|
+
if (!s)
|
|
121
|
+
return;
|
|
122
|
+
if (cols > 0)
|
|
123
|
+
s.cols = cols;
|
|
124
|
+
if (rows > 0)
|
|
125
|
+
s.rows = rows;
|
|
126
|
+
}
|
|
127
|
+
scrollbackFor(sessionId) {
|
|
128
|
+
const s = this.sessions.get(sessionId);
|
|
129
|
+
return s ? s.buffer.snapshot() : Buffer.alloc(0);
|
|
130
|
+
}
|
|
131
|
+
// ---- send helpers ----
|
|
132
|
+
sendControl(workerId, msg) {
|
|
133
|
+
const conn = this.conns.get(workerId);
|
|
134
|
+
if (!conn || conn.ws.readyState !== conn.ws.OPEN)
|
|
135
|
+
return false;
|
|
136
|
+
try {
|
|
137
|
+
conn.ws.send(JSON.stringify(msg));
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
sendInput(workerId, sessionId, data) {
|
|
145
|
+
const conn = this.conns.get(workerId);
|
|
146
|
+
if (!conn || conn.ws.readyState !== conn.ws.OPEN)
|
|
147
|
+
return false;
|
|
148
|
+
try {
|
|
149
|
+
conn.ws.send(encodePtyFrame(sessionId, data), { binary: true });
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Subscribe to PTY data frames for a session. Returns an unsubscribe fn.
|
|
158
|
+
* Listeners receive the raw payload bytes (already stripped of the frame header).
|
|
159
|
+
*/
|
|
160
|
+
subscribe(sessionId, listener) {
|
|
161
|
+
const evt = `session:${sessionId}:data`;
|
|
162
|
+
this.on(evt, listener);
|
|
163
|
+
return () => this.off(evt, listener);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../../src/server/workers/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO3C,OAAO,EACL,cAAc,GAKf,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AA0BzC;;;;;;GAMG;AACH,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC7B,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAC5C,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5D,yEAAyE;IACjE,QAAQ,GAAG,CAAC,CAAC;IAErB,QAAQ,CAAC,IAAsB;QAC7B,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,QAAQ,IAAI,QAAQ,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC;gBACH,QAAQ,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;YACzC,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,UAAU,CAAC,QAAgB;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,0EAA0E;QAC1E,0EAA0E;QAC1E,uCAAuC;IACzC,CAAC;IAED,GAAG,CAAC,QAAgB;QAClB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,2DAA2D;IAC3D,eAAe,CAAC,QAAgB,EAAE,QAAwB,EAAE,OAA4B;QACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,KAAgB,EAAE,IAAc;QAC3C,MAAM,UAAU,GAAuB,EAAE,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;gBAAE,SAAS;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC9B,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAE,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8BAA8B;IAE9B,aAAa,CAAC,SAAiB,EAAE,QAAgB,EAAE,MAAc;QAC/D,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,GAAG;gBACF,QAAQ;gBACR,MAAM;gBACN,IAAI,EAAE,GAAG;gBACT,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,IAAI,UAAU,CAAC,gBAAgB,CAAC;aACzC,CAAC;YACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,yEAAyE;YACzE,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACtB,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;QACpB,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,aAAa,CAAC,SAAiB;QAC7B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,+EAA+E;IAC/E,iBAAiB,CAAC,SAAiB,EAAE,OAAe;QAClD,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,sEAAsE;YACtE,wEAAwE;YACxE,gCAAgC;YAChC,OAAO;QACT,CAAC;QACD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,WAAW,SAAS,OAAO,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,cAAc,CAAC,SAAiB,EAAE,IAAY,EAAE,IAAY;QAC1D,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,IAAI,GAAG,CAAC;YAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;QAC5B,IAAI,IAAI,GAAG,CAAC;YAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED,aAAa,CAAC,SAAiB;QAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,yBAAyB;IAEzB,WAAW,CAAC,QAAgB,EAAE,GAAuB;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,IAAI,CAAC,EAAE,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAC/D,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,SAAS,CAAC,QAAgB,EAAE,SAAiB,EAAE,IAAY;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,IAAI,CAAC,EAAE,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAC/D,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,SAAiB,EAAE,QAAiC;QAC5D,MAAM,GAAG,GAAG,WAAW,SAAS,OAAO,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACvB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import { resolveAgentConfig } from "../agents/registry.js";
|
|
2
|
+
/**
|
|
3
|
+
* Dispatches queued tasks to connected workers.
|
|
4
|
+
*
|
|
5
|
+
* Decisions:
|
|
6
|
+
* - Concurrency is enforced PER (agent, kind) globally — sums across workers.
|
|
7
|
+
* - Worker selection is round-robin across workers that advertise the agent
|
|
8
|
+
* AND have free capacity for the kind.
|
|
9
|
+
* - The scheduler does NOT spawn agents directly; everything goes through a
|
|
10
|
+
* `task.assign` over the registry's worker WS. The session.started reply
|
|
11
|
+
* flips the task to 'running'; the gateway-side WS handler does that.
|
|
12
|
+
*/
|
|
7
13
|
export class Scheduler {
|
|
8
14
|
opts;
|
|
9
15
|
running = false;
|
|
@@ -15,6 +21,7 @@ export class Scheduler {
|
|
|
15
21
|
}
|
|
16
22
|
start() {
|
|
17
23
|
this.opts.bus.on("task:queued", () => this.wake());
|
|
24
|
+
this.opts.bus.on("worker:connected", () => this.wake());
|
|
18
25
|
this.recoverInterruptedTasks();
|
|
19
26
|
this.wake();
|
|
20
27
|
}
|
|
@@ -55,155 +62,97 @@ export class Scheduler {
|
|
|
55
62
|
const cap = agentCfg.concurrency[kind];
|
|
56
63
|
if (cap <= 0)
|
|
57
64
|
continue;
|
|
58
|
-
const
|
|
59
|
-
|
|
65
|
+
const inFlight = this.opts.q.countByStatus("assigning", "running");
|
|
66
|
+
// Cheap: per-(agent, kind) running gate. Keeps current single-machine
|
|
67
|
+
// semantics — global cap holds across whichever workers picked it up.
|
|
68
|
+
const runningForThis = countMatching(this.opts.q.listByStatus("assigning", "running"), agent, kind);
|
|
69
|
+
if (runningForThis >= cap)
|
|
60
70
|
continue;
|
|
61
71
|
const queued = this.opts.q.listQueuedFor(agent, kind);
|
|
62
|
-
const slots = cap -
|
|
72
|
+
const slots = cap - runningForThis;
|
|
63
73
|
for (const task of queued.slice(0, slots)) {
|
|
64
74
|
try {
|
|
65
|
-
|
|
75
|
+
const ok = this.dispatch(task);
|
|
76
|
+
if (!ok) {
|
|
77
|
+
// Couldn't place — no eligible worker right now. Stop pushing
|
|
78
|
+
// this partition for this tick; we'll try again on next worker
|
|
79
|
+
// connect or task:queued.
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
66
82
|
}
|
|
67
83
|
catch (err) {
|
|
68
84
|
this.log(`[scheduler] dispatch failed for task #${task.id}: ${err.message}`);
|
|
69
85
|
this.opts.q.setTaskStatus(task.id, "failed", { exitCode: -1 });
|
|
70
86
|
}
|
|
71
87
|
}
|
|
88
|
+
// Touch inFlight to silence unused warning; keep for future budget check.
|
|
89
|
+
void inFlight;
|
|
72
90
|
}
|
|
73
91
|
}
|
|
74
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* On gateway start: any task we left in 'running' or 'assigning' has lost
|
|
95
|
+
* its worker (process restart). Mark them failed so the user can retry.
|
|
96
|
+
* Workers that reconnect within the disconnect grace window will reattach
|
|
97
|
+
* via the WS handshake — those don't go through this path.
|
|
98
|
+
*/
|
|
75
99
|
recoverInterruptedTasks() {
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
this.log(`[scheduler] recovering task #${t.id} that was running at shutdown — marking failed`);
|
|
100
|
+
for (const t of this.opts.q.listByStatus("running", "assigning")) {
|
|
101
|
+
this.log(`[scheduler] recovering task #${t.id} (was ${t.status} at startup) — marking failed`);
|
|
79
102
|
this.opts.q.setTaskStatus(t.id, "failed", { exitCode: -1 });
|
|
103
|
+
this.opts.q.clearTaskAssignment(t.id);
|
|
80
104
|
}
|
|
81
105
|
}
|
|
82
|
-
|
|
106
|
+
/**
|
|
107
|
+
* Send `task.assign` to an eligible worker. Returns true when placed.
|
|
108
|
+
* Returns false when no worker is available — caller should stop and wait
|
|
109
|
+
* for a worker:connected or task:queued wakeup.
|
|
110
|
+
*/
|
|
111
|
+
dispatch(task) {
|
|
83
112
|
const cfg = this.opts.configManager.cfg;
|
|
84
113
|
const repo = cfg.repos.find((r) => r.slug === task.repoSlug);
|
|
85
114
|
if (!repo) {
|
|
86
115
|
throw new Error(`task #${task.id}: repo ${task.repoSlug} no longer in config`);
|
|
87
116
|
}
|
|
88
|
-
const
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
: await createWorktree({
|
|
97
|
-
workspacesRoot: cfg.workspaces.root,
|
|
98
|
-
slug: task.repoSlug,
|
|
99
|
-
issueNumber: task.issueNumber,
|
|
100
|
-
baseBranch: repo.baseBranch,
|
|
101
|
-
sourcePath: repo.path,
|
|
102
|
-
});
|
|
103
|
-
const adapter = getAdapter(task.agent);
|
|
104
|
-
const specArgs = {
|
|
105
|
-
agentConfig: resolveAgentConfig(cfg, task.agent),
|
|
117
|
+
const conn = this.opts.registry.findEligible(task.agent, task.kind);
|
|
118
|
+
if (!conn)
|
|
119
|
+
return false;
|
|
120
|
+
const agentConfig = resolveAgentConfig(cfg, task.agent);
|
|
121
|
+
const branch = task.kind === "implement" ? `tono/issue-${task.issueNumber}` : `tono/pr-${task.issueNumber}`;
|
|
122
|
+
const assign = {
|
|
123
|
+
type: "task.assign",
|
|
124
|
+
taskId: task.id,
|
|
106
125
|
kind: task.kind,
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
126
|
+
repoSlug: task.repoSlug,
|
|
127
|
+
repoSourcePath: repo.path ?? null,
|
|
128
|
+
baseBranch: repo.baseBranch,
|
|
129
|
+
issueNumber: task.issueNumber,
|
|
130
|
+
issueTitle: task.issueTitle,
|
|
131
|
+
issueBody: task.issueBody,
|
|
132
|
+
prUrl: task.prUrl,
|
|
133
|
+
branch,
|
|
134
|
+
agent: task.agent,
|
|
135
|
+
resumeAgentSessionId: task.agentSessionId,
|
|
136
|
+
agentConfig,
|
|
117
137
|
};
|
|
118
|
-
|
|
119
|
-
if (!
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
this.opts.q.setAgentSessionId(task.id, resumeId);
|
|
123
|
-
this.log(`[scheduler] task #${task.id} discovered prior session ${resumeId}`);
|
|
124
|
-
}
|
|
138
|
+
const sent = this.opts.registry.sendControl(conn.workerId, assign);
|
|
139
|
+
if (!sent) {
|
|
140
|
+
this.log(`[scheduler] failed to send task.assign for #${task.id} to ${conn.workerId}`);
|
|
141
|
+
return false;
|
|
125
142
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
this.opts.q.setTaskStatus(task.id, "running");
|
|
133
|
-
this.opts.q.insertSession({
|
|
134
|
-
id: sessionId,
|
|
135
|
-
taskId: task.id,
|
|
136
|
-
pid: session.pid,
|
|
137
|
-
worktreePath: wt.path,
|
|
138
|
-
});
|
|
139
|
-
this.log(`[scheduler] task #${task.id} ${task.agent}/${task.kind} ${task.repoSlug}#${task.issueNumber} -> session ${sessionId} (pid ${session.pid}) in ${wt.path}${spec.resumed ? ` [resumed ${resumeId}]` : ""}`);
|
|
140
|
-
this.opts.bus.emit("task:updated", { task: this.opts.q.getTask(task.id) });
|
|
141
|
-
if (!resumeId) {
|
|
142
|
-
void adapter
|
|
143
|
-
.captureSessionId({ cwd: wt.path })
|
|
144
|
-
.then((agentSessionId) => {
|
|
145
|
-
if (!agentSessionId) {
|
|
146
|
-
this.log(`[scheduler] task #${task.id}: could not capture agent session id (timed out)`);
|
|
147
|
-
return;
|
|
148
|
-
}
|
|
149
|
-
const cur = this.opts.q.getTask(task.id);
|
|
150
|
-
if (!cur || cur.agentSessionId === agentSessionId)
|
|
151
|
-
return;
|
|
152
|
-
this.opts.q.setAgentSessionId(task.id, agentSessionId);
|
|
153
|
-
this.log(`[scheduler] task #${task.id} agent session = ${agentSessionId}`);
|
|
154
|
-
const updated = this.opts.q.getTask(task.id);
|
|
155
|
-
if (updated)
|
|
156
|
-
this.opts.bus.emit("task:updated", { task: updated });
|
|
157
|
-
})
|
|
158
|
-
.catch((err) => {
|
|
159
|
-
this.log(`[scheduler] task #${task.id} captureSessionId error: ${err.message}`);
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
// PR-URL detection only applies to implement tasks. Review tasks already
|
|
163
|
-
// know their PR URL (set at insert time), and the agent isn't expected to
|
|
164
|
-
// create a new PR — if it does, we ignore that.
|
|
165
|
-
let detectedPrUrl = null;
|
|
166
|
-
if (!isReview) {
|
|
167
|
-
let tail = "";
|
|
168
|
-
const onChunk = (chunk) => {
|
|
169
|
-
if (detectedPrUrl)
|
|
170
|
-
return;
|
|
171
|
-
tail = (tail + chunk.toString("utf8")).slice(-4096);
|
|
172
|
-
const url = parsePrUrl(tail);
|
|
173
|
-
if (!url)
|
|
174
|
-
return;
|
|
175
|
-
detectedPrUrl = url;
|
|
176
|
-
this.opts.q.setTaskStatus(task.id, "pr_open", { prUrl: url, exitCode: 0 });
|
|
177
|
-
const updated = this.opts.q.getTask(task.id);
|
|
178
|
-
if (updated)
|
|
179
|
-
this.opts.bus.emit("task:updated", { task: updated });
|
|
180
|
-
this.log(`[scheduler] task #${task.id} -> pr_open (${url}); session ${sessionId} still attached`);
|
|
181
|
-
session.off("data", onChunk);
|
|
182
|
-
this.wake();
|
|
183
|
-
};
|
|
184
|
-
session.on("data", onChunk);
|
|
185
|
-
}
|
|
186
|
-
session.once("exit", ({ exitCode }) => {
|
|
187
|
-
this.opts.q.endSession(sessionId);
|
|
188
|
-
const current = this.opts.q.getTask(task.id);
|
|
189
|
-
const preserveStatuses = new Set(["completed", "pr_open", "pr_closed", "merged", "cleaned"]);
|
|
190
|
-
if (current && preserveStatuses.has(current.status)) {
|
|
191
|
-
this.log(`[scheduler] task #${task.id} session ended (exit ${exitCode}); status ${current.status} preserved`);
|
|
192
|
-
this.opts.bus.emit("task:updated", { task: current });
|
|
193
|
-
this.wake();
|
|
194
|
-
return;
|
|
195
|
-
}
|
|
196
|
-
const prUrl = isReview
|
|
197
|
-
? task.prUrl ?? null
|
|
198
|
-
: detectedPrUrl ?? parsePrUrl(session.scrollback().toString("utf8"));
|
|
199
|
-
const status = exitCode === 0 ? "completed" : "failed";
|
|
200
|
-
this.opts.q.setTaskStatus(task.id, status, { exitCode, prUrl });
|
|
201
|
-
this.log(`[scheduler] task #${task.id} ended with code ${exitCode}${prUrl ? ` (PR: ${prUrl})` : ""}`);
|
|
202
|
-
const updated = this.opts.q.getTask(task.id);
|
|
203
|
-
if (updated)
|
|
204
|
-
this.opts.bus.emit("task:updated", { task: updated });
|
|
205
|
-
this.wake();
|
|
206
|
-
});
|
|
143
|
+
this.opts.q.setTaskAssigned(task.id, conn.workerId);
|
|
144
|
+
this.log(`[scheduler] task #${task.id} ${task.agent}/${task.kind} ${task.repoSlug}#${task.issueNumber} -> worker ${conn.hostname} (${conn.workerId})`);
|
|
145
|
+
const updated = this.opts.q.getTask(task.id);
|
|
146
|
+
if (updated)
|
|
147
|
+
this.opts.bus.emit("task:updated", { task: updated });
|
|
148
|
+
return true;
|
|
207
149
|
}
|
|
208
150
|
}
|
|
151
|
+
function countMatching(rows, agent, kind) {
|
|
152
|
+
let n = 0;
|
|
153
|
+
for (const r of rows)
|
|
154
|
+
if (r.agent === agent && r.kind === kind)
|
|
155
|
+
n += 1;
|
|
156
|
+
return n;
|
|
157
|
+
}
|
|
209
158
|
//# sourceMappingURL=scheduler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../../../src/server/workers/scheduler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../../../src/server/workers/scheduler.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAa3D;;;;;;;;;;GAUG;AACH,MAAM,OAAO,SAAS;IAKS;IAJrB,OAAO,GAAG,KAAK,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC;IACb,GAAG,CAAwB;IAE5C,YAA6B,IAAsB;QAAtB,SAAI,GAAJ,IAAI,CAAkB;QACjD,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACrC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED,+DAA+D;IAC/D,IAAI;QACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;gBAC3B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3B,CAAC,QAAQ,IAAI,CAAC,aAAa,EAAE;QAC/B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;QACxC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAA6D,CAAC;QAC5G,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,KAAK,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAe,EAAE,CAAC;gBACzD,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,GAAG,IAAI,CAAC;oBAAE,SAAS;gBACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;gBACnE,sEAAsE;gBACtE,sEAAsE;gBACtE,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpG,IAAI,cAAc,IAAI,GAAG;oBAAE,SAAS;gBACpC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACtD,MAAM,KAAK,GAAG,GAAG,GAAG,cAAc,CAAC;gBACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC1C,IAAI,CAAC;wBACH,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;wBAC/B,IAAI,CAAC,EAAE,EAAE,CAAC;4BACR,8DAA8D;4BAC9D,+DAA+D;4BAC/D,0BAA0B;4BAC1B,MAAM;wBACR,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,IAAI,CAAC,GAAG,CAAC,yCAAyC,IAAI,CAAC,EAAE,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;wBACxF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;oBACjE,CAAC;gBACH,CAAC;gBACD,0EAA0E;gBAC1E,KAAK,QAAQ,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,uBAAuB;QAC7B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;YACjE,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,+BAA+B,CAAC,CAAC;YAC/F,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,QAAQ,CAAC,IAAa;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;QACxC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,QAAQ,sBAAsB,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QAExB,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,WAAW,EAAE,CAAC;QAC5G,MAAM,MAAM,GAAkB;YAC5B,IAAI,EAAE,aAAa;YACnB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM;YACN,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,oBAAoB,EAAE,IAAI,CAAC,cAAc;YACzC,WAAW;SACZ,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,+CAA+C,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG,CACN,qBAAqB,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,WAAW,cAAc,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,GAAG,CAC7I,CAAC;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,OAAO;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,SAAS,aAAa,CAAC,IAAe,EAAE,KAAgB,EAAE,IAAc;IACtE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,IAAI,CAAC,CAAC,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;YAAE,CAAC,IAAI,CAAC,CAAC;IACvE,OAAO,CAAC,CAAC;AACX,CAAC"}
|