wicked-crew 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -0
- package/dist/api/audit.d.ts +13 -0
- package/dist/api/audit.d.ts.map +1 -1
- package/dist/api/audit.js +18 -2
- package/dist/api/audit.js.map +1 -1
- package/dist/api/endpoint-manifest-live.d.ts +29 -0
- package/dist/api/endpoint-manifest-live.d.ts.map +1 -0
- package/dist/api/endpoint-manifest-live.js +72 -0
- package/dist/api/endpoint-manifest-live.js.map +1 -0
- package/dist/api/endpoint-manifest.d.ts +107 -0
- package/dist/api/endpoint-manifest.d.ts.map +1 -0
- package/dist/api/endpoint-manifest.js +108 -0
- package/dist/api/endpoint-manifest.js.map +1 -0
- package/dist/api/guidance-index.d.ts +39 -0
- package/dist/api/guidance-index.d.ts.map +1 -0
- package/dist/api/guidance-index.js +67 -0
- package/dist/api/guidance-index.js.map +1 -0
- package/dist/api/open-path.d.ts +16 -0
- package/dist/api/open-path.d.ts.map +1 -1
- package/dist/api/open-path.js +22 -0
- package/dist/api/open-path.js.map +1 -1
- package/dist/api/requirements.d.ts +7 -0
- package/dist/api/requirements.d.ts.map +1 -1
- package/dist/api/requirements.js +23 -2
- package/dist/api/requirements.js.map +1 -1
- package/dist/api/retry-index.d.ts +30 -0
- package/dist/api/retry-index.d.ts.map +1 -0
- package/dist/api/retry-index.js +45 -0
- package/dist/api/retry-index.js.map +1 -0
- package/dist/api/routes.d.ts +53 -1
- package/dist/api/routes.d.ts.map +1 -1
- package/dist/api/routes.js +399 -30
- package/dist/api/routes.js.map +1 -1
- package/dist/api/run-files.d.ts +63 -0
- package/dist/api/run-files.d.ts.map +1 -0
- package/dist/api/run-files.js +271 -0
- package/dist/api/run-files.js.map +1 -0
- package/dist/api/server.d.ts +79 -0
- package/dist/api/server.d.ts.map +1 -1
- package/dist/api/server.js +183 -8
- package/dist/api/server.js.map +1 -1
- package/dist/api/stall-watchdog.d.ts +62 -0
- package/dist/api/stall-watchdog.d.ts.map +1 -0
- package/dist/api/stall-watchdog.js +138 -0
- package/dist/api/stall-watchdog.js.map +1 -0
- package/dist/cli/index.js +89 -15
- package/dist/cli/index.js.map +1 -1
- package/dist/core/adapter.d.ts +36 -10
- package/dist/core/adapter.d.ts.map +1 -1
- package/dist/core/adapter.js +245 -31
- package/dist/core/adapter.js.map +1 -1
- package/dist/core/bridge-reaper.d.ts +134 -0
- package/dist/core/bridge-reaper.d.ts.map +1 -0
- package/dist/core/bridge-reaper.js +286 -0
- package/dist/core/bridge-reaper.js.map +1 -0
- package/dist/core/deliver.d.ts +118 -0
- package/dist/core/deliver.d.ts.map +1 -0
- package/dist/core/deliver.js +241 -0
- package/dist/core/deliver.js.map +1 -0
- package/dist/core/deliverable-floor.d.ts +155 -0
- package/dist/core/deliverable-floor.d.ts.map +1 -0
- package/dist/core/deliverable-floor.js +248 -0
- package/dist/core/deliverable-floor.js.map +1 -0
- package/dist/core/exec.d.ts +2 -0
- package/dist/core/exec.d.ts.map +1 -1
- package/dist/core/exec.js.map +1 -1
- package/dist/core/types.d.ts +79 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +3 -0
- package/dist/core/types.js.map +1 -1
- package/dist/interactive/bridge-pool.d.ts +52 -0
- package/dist/interactive/bridge-pool.d.ts.map +1 -1
- package/dist/interactive/bridge-pool.js +93 -12
- package/dist/interactive/bridge-pool.js.map +1 -1
- package/dist/interactive/chat-events.d.ts +207 -0
- package/dist/interactive/chat-events.d.ts.map +1 -0
- package/dist/interactive/chat-events.js +769 -0
- package/dist/interactive/chat-events.js.map +1 -0
- package/dist/interactive/demo-events.d.ts +283 -0
- package/dist/interactive/demo-events.d.ts.map +1 -0
- package/dist/interactive/demo-events.js +889 -0
- package/dist/interactive/demo-events.js.map +1 -0
- package/dist/interactive/draft-events.d.ts +87 -7
- package/dist/interactive/draft-events.d.ts.map +1 -1
- package/dist/interactive/draft-events.js +352 -49
- package/dist/interactive/draft-events.js.map +1 -1
- package/dist/interactive/edit-events.d.ts +25 -2
- package/dist/interactive/edit-events.d.ts.map +1 -1
- package/dist/interactive/edit-events.js +88 -9
- package/dist/interactive/edit-events.js.map +1 -1
- package/dist/interactive/repo-snapshot.d.ts +100 -0
- package/dist/interactive/repo-snapshot.d.ts.map +1 -0
- package/dist/interactive/repo-snapshot.js +289 -0
- package/dist/interactive/repo-snapshot.js.map +1 -0
- package/dist/projects/graph-paths.d.ts +122 -0
- package/dist/projects/graph-paths.d.ts.map +1 -0
- package/dist/projects/graph-paths.js +175 -0
- package/dist/projects/graph-paths.js.map +1 -0
- package/dist/projects/graph.d.ts +191 -0
- package/dist/projects/graph.d.ts.map +1 -0
- package/dist/projects/graph.js +829 -0
- package/dist/projects/graph.js.map +1 -0
- package/dist/projects/routes.d.ts +17 -0
- package/dist/projects/routes.d.ts.map +1 -1
- package/dist/projects/routes.js +139 -0
- package/dist/projects/routes.js.map +1 -1
- package/dist/qe/ledger.d.ts +3 -2
- package/dist/qe/ledger.d.ts.map +1 -1
- package/dist/qe/ledger.js +5 -4
- package/dist/qe/ledger.js.map +1 -1
- package/dist/studio/assets/index-D-BFUYnY.js +530 -0
- package/dist/studio/assets/index-D6S9zUtO.css +32 -0
- package/dist/studio/index.html +5 -3
- package/endpoint-manifest.json +624 -0
- package/package.json +8 -5
- package/dist/studio/assets/index-CCwXa1cn.js +0 -428
- package/dist/studio/assets/index-HWxo0h41.css +0 -32
package/dist/api/server.js
CHANGED
|
@@ -8,18 +8,27 @@ import { GateCache } from './gate-cache.js';
|
|
|
8
8
|
import { ElicitationCache } from './elicitation-cache.js';
|
|
9
9
|
import { registerAuthHooks, resolveAuth } from './auth.js';
|
|
10
10
|
import { AuditLog } from './audit.js';
|
|
11
|
+
import { RetryIndex } from './retry-index.js';
|
|
12
|
+
import { GuidanceIndex } from './guidance-index.js';
|
|
11
13
|
import { registerClient, broadcast } from '../events/bus.js';
|
|
12
14
|
import { TerminalHub, registerTerminalWs } from '../events/terminals.js';
|
|
13
15
|
import { QeGateCache, startQeGateSubscriber } from '../qe/gate-events.js';
|
|
14
16
|
import { startInteractiveDraftSubscriber } from '../interactive/draft-events.js';
|
|
15
17
|
import { startInteractiveEditSubscriber } from '../interactive/edit-events.js';
|
|
18
|
+
import { startInteractiveChatSubscriber } from '../interactive/chat-events.js';
|
|
19
|
+
import { startInteractiveDemoSubscriber } from '../interactive/demo-events.js';
|
|
20
|
+
import { resolveInteractiveRoot } from '../interactive/bridge-root.js';
|
|
21
|
+
import { ProjectSettingsStore } from '../projects/settings.js';
|
|
16
22
|
import { startProjectBus, MEMBERSHIP_ATTACHED, membershipAttachedKey } from '../projects/events.js';
|
|
17
23
|
import { startInteractiveWsRelay, registerInteractiveEventRoutes } from '../interactive/ws-relay.js';
|
|
18
24
|
import { MembershipIndex } from '../projects/membership-index.js';
|
|
19
25
|
import { writeRunEvidencePointer } from '../projects/charter.js';
|
|
20
26
|
import { CoreAdapter } from '../core/adapter.js';
|
|
21
27
|
import { SeatHealthTracker, startSeatHealthProbe } from './seat-health.js';
|
|
28
|
+
import { installEndpointManifestHook } from './endpoint-manifest.js';
|
|
29
|
+
import { WorkerStallWatchdog } from './stall-watchdog.js';
|
|
22
30
|
import { applyWorkerConfigRoot } from './seat-signin.js';
|
|
31
|
+
import { DEFAULT_WORKER_STALL_MINUTES } from '../core/types.js';
|
|
23
32
|
// Allow the studio (a separate localhost origin, e.g. :4200) to call the
|
|
24
33
|
// daemon's REST API. Restricted to loopback origins — the daemon only binds
|
|
25
34
|
// 127.0.0.1, so this never widens exposure beyond the local machine.
|
|
@@ -36,6 +45,11 @@ export function defaultStudioRoot() {
|
|
|
36
45
|
}
|
|
37
46
|
export async function createServer(adapter, options) {
|
|
38
47
|
const app = Fastify({ logger: { level: process.env['LOG_LEVEL'] ?? 'info' } });
|
|
48
|
+
// TH-11: accumulate the route table as it registers — the endpoint manifest's one source of
|
|
49
|
+
// truth. Installed FIRST because fastify only fires onRoute for routes added after the hook
|
|
50
|
+
// exists. Read via `app.endpointManifest` (scripts/generate-endpoint-manifest.ts + the drift
|
|
51
|
+
// test); costs one array push per route at boot, nothing per request.
|
|
52
|
+
app.decorate('endpointManifest', installEndpointManifestHook(app));
|
|
39
53
|
const gateCache = new GateCache();
|
|
40
54
|
const elicitationCache = new ElicitationCache();
|
|
41
55
|
const terminals = new TerminalHub();
|
|
@@ -68,6 +82,19 @@ export async function createServer(adapter, options) {
|
|
|
68
82
|
// post-commit half (index tag + membership.attached emit) via `fileRun`.
|
|
69
83
|
const membershipIndex = new MembershipIndex();
|
|
70
84
|
await membershipIndex.hydrate(adapter, (m) => app.log.warn(m));
|
|
85
|
+
// Per-project crew-side settings (DES-MERGE-001 §7.1's `interactiveRoot`). ONE instance,
|
|
86
|
+
// created here so the chat seam's docs-root resolution below and the routes (project PATCH +
|
|
87
|
+
// interactive proxy) all read/write the same store — two instances over one file would let a
|
|
88
|
+
// PATCH land in one while the other keeps serving the stale root.
|
|
89
|
+
const projectSettings = new ProjectSettingsStore();
|
|
90
|
+
// Retry lineage (CREW-UX-3): hydrated from the audit trail — the durable record the launch
|
|
91
|
+
// route writes — so a restarted daemon still echoes `retry_of` on prior runs' DTOs.
|
|
92
|
+
const retryIndex = new RetryIndex();
|
|
93
|
+
await retryIndex.hydrate(audit, (m) => app.log.warn(m));
|
|
94
|
+
// Operator guidance (CREW-UX-7, DES-UX-002 §7.2): same durable pattern — hydrated from the
|
|
95
|
+
// trail's `guidance.set` entries so notes survive a daemon restart.
|
|
96
|
+
const guidanceIndex = new GuidanceIndex();
|
|
97
|
+
await guidanceIndex.hydrate(audit, (m) => app.log.warn(m));
|
|
71
98
|
const projectBus = options?.projectEvents?.disabled === true
|
|
72
99
|
? null
|
|
73
100
|
: await startProjectBus({
|
|
@@ -118,6 +145,11 @@ export async function createServer(adapter, options) {
|
|
|
118
145
|
// under the task #88 rule that event actors are never caller-supplied.
|
|
119
146
|
{ project_id: projectId, member: { kind: 'crew.run', ref: runId }, actor: 'interactive' }, membershipAttachedKey(projectId, 'crew.run', runId, Date.now()));
|
|
120
147
|
};
|
|
148
|
+
/** The docs root a project's interactive docs live under — the SAME per-project settings
|
|
149
|
+
* resolution the project routes, the interactive proxy, and the chat seam share
|
|
150
|
+
* (DES-MERGE-001 §7.1/§7.2). Shared by the edit seam (demo-kind gate, CREW-UX-9) and the
|
|
151
|
+
* demo seam (spec install + manifest reads). */
|
|
152
|
+
const interactiveDocsRoot = (projectId) => resolveInteractiveRoot(projectId !== undefined ? projectSettings.get(projectId) : null);
|
|
121
153
|
// Arm the opt-in QE gate-event subscription (crew's bus seam). Failure to
|
|
122
154
|
// arm is LOUD but non-fatal: the acceptance route never depends on the bus.
|
|
123
155
|
if (options?.qeGateEvents?.enabled === true) {
|
|
@@ -134,12 +166,54 @@ export async function createServer(adapter, options) {
|
|
|
134
166
|
});
|
|
135
167
|
}
|
|
136
168
|
}
|
|
169
|
+
// ── A STUB ENGINE NEVER ANSWERS ANOTHER PRODUCT'S TRAFFIC (crew#309) ────────────────────────
|
|
170
|
+
//
|
|
171
|
+
// The four seams below are ANSWERERS: each opens a durable wicked-bus subscription and replies
|
|
172
|
+
// to wicked-interactive's events by LAUNCHING A GOVERNED RUN. Under `serve --stub` the engine is
|
|
173
|
+
// `Core.spawnStub` — a `StubDispatcher` (every seat votes for the first roster option, no
|
|
174
|
+
// subprocess) plus a `StubStepRunner` (fixed text, no CLI) — so such a run resolves every phase
|
|
175
|
+
// Ok in under a millisecond and narrates the entire governed lifecycle onto interactive's
|
|
176
|
+
// vocabulary: "Convening a 6-seat council…", "Council picked claude for outline…", "Gate
|
|
177
|
+
// approved outline — moving on…", "Gate approved the draft…". Nothing ran. Nothing was written.
|
|
178
|
+
// Two gate approvals were announced anyway.
|
|
179
|
+
//
|
|
180
|
+
// Worse than fabricating locally: the durable cursor is keyed by PLUGIN NAME
|
|
181
|
+
// (`wicked-crew-interactive-draft`), so on the shared bus a stub daemon and the production
|
|
182
|
+
// daemon are ONE consumer group — a frame claimed by the stub is a frame the real daemon never
|
|
183
|
+
// sees. That is what wicked-crew#309 recorded: bus rows 242632–242643 carry a complete draft
|
|
184
|
+
// narration ending in a gate approval, 52ms wide, whose run id
|
|
185
|
+
// (3c106511-6340-4465-b029-30cb5b004416) appears in NO store, NO event log and NO audit trail on
|
|
186
|
+
// the daemon the operator was watching — `GET /runs/:id` 404s on a run the bus says was
|
|
187
|
+
// gate-approved. Both units were assigned the same first-roster seat (`claude`) within 1ms of
|
|
188
|
+
// convening, which is the StubDispatcher's signature; the real runs 5 minutes later took ~95s to
|
|
189
|
+
// vote and split their seats (`pi` for outline, `claude` for draft).
|
|
190
|
+
//
|
|
191
|
+
// So: refuse to arm, loudly. The deny is scoped to the ANSWERERS on purpose — the QE seam above
|
|
192
|
+
// only fills a freshness cache and the project seam only relays, neither launches work nor
|
|
193
|
+
// narrates governance, so neither can fabricate a verdict. Offline/deterministic runs are NOT
|
|
194
|
+
// lost by this: every harness in `e2e/` already does the correct thing, keeping the REAL engine
|
|
195
|
+
// (`stub: false`) and registering a scripted stub SEAT in the roster — which exercises planning,
|
|
196
|
+
// dispatch, gates and the durable event log for real and simply pins what the worker returns.
|
|
197
|
+
const stubEngine = adapter.stub === true;
|
|
198
|
+
const refuseStubSeam = (seam) => {
|
|
199
|
+
if (!stubEngine)
|
|
200
|
+
return false;
|
|
201
|
+
app.log.warn(`${seam} subscription REFUSED: this daemon runs the deterministic stub engine (--stub), ` +
|
|
202
|
+
`whose runs complete every phase instantly with no CLI and no artifact. Answering ` +
|
|
203
|
+
`wicked-interactive on the bus would narrate a full governed lifecycle — gate approvals ` +
|
|
204
|
+
`included — for work that never happened, and would take the frame away from the ` +
|
|
205
|
+
`production daemon sharing this bus (crew#309). Run without --stub to answer, or keep the ` +
|
|
206
|
+
`real engine and register a scripted stub SEAT for a deterministic run.`);
|
|
207
|
+
return true;
|
|
208
|
+
};
|
|
137
209
|
// Arm the opt-in interactive-draft answering seam (task #86 spike). Same posture as the QE
|
|
138
210
|
// seam: failure to arm is LOUD but non-fatal — interactive's assist loop is the fallback
|
|
139
|
-
// answerer, and this daemon must boot on a machine whose bus is broken.
|
|
140
|
-
|
|
211
|
+
// answerer, and this daemon must boot on a machine whose bus is broken. The handle is kept:
|
|
212
|
+
// the chat seam below consults its in-flight docs (CREW-UX-5 per-doc serialization).
|
|
213
|
+
let draftSub = null;
|
|
214
|
+
if (options?.interactiveDraftEvents?.enabled === true && !refuseStubSeam('interactive-draft')) {
|
|
141
215
|
const o = options.interactiveDraftEvents;
|
|
142
|
-
|
|
216
|
+
draftSub = await startInteractiveDraftSubscriber(adapter, {
|
|
143
217
|
...(o.dbPath !== undefined ? { dbPath: o.dbPath } : {}),
|
|
144
218
|
...(o.pollIntervalMs !== undefined ? { pollIntervalMs: o.pollIntervalMs } : {}),
|
|
145
219
|
...(o.heartbeatMs !== undefined ? { heartbeatMs: o.heartbeatMs } : {}),
|
|
@@ -149,35 +223,118 @@ export async function createServer(adapter, options) {
|
|
|
149
223
|
onRunFiled: fileRun,
|
|
150
224
|
log: (m) => app.log.warn(m),
|
|
151
225
|
});
|
|
152
|
-
if (
|
|
226
|
+
if (draftSub !== null) {
|
|
227
|
+
const sub = draftSub;
|
|
153
228
|
app.log.info('interactive-draft subscription armed (filter wicked.interactive.doc.created)');
|
|
154
229
|
app.addHook('onClose', async () => {
|
|
155
230
|
await sub.stop();
|
|
156
231
|
});
|
|
157
232
|
}
|
|
158
233
|
}
|
|
234
|
+
// Declared BEFORE the edit seam it is probed by: the demo seam arms after (it must, so the
|
|
235
|
+
// per-doc serialization contract can see the edit seam's in-flight set), but the edit seam's
|
|
236
|
+
// demo-kind gate needs to know at EVENT time whether the demo seam actually came up. A
|
|
237
|
+
// closure over this binding reads the settled value; a boolean captured here would not.
|
|
238
|
+
let demoSub = null;
|
|
159
239
|
// Arm the opt-in interactive STRUCTURAL-edit answering seam (task #86 final leg). Same
|
|
160
240
|
// posture as the draft seam: failure to arm is LOUD but non-fatal — interactive's assist
|
|
161
241
|
// loop is the fallback answerer, and this daemon must boot on a machine whose bus is broken.
|
|
162
|
-
|
|
242
|
+
let editSub = null;
|
|
243
|
+
if (options?.interactiveEditEvents?.enabled === true && !refuseStubSeam('interactive-edit')) {
|
|
163
244
|
const o = options.interactiveEditEvents;
|
|
164
|
-
|
|
245
|
+
editSub = await startInteractiveEditSubscriber(adapter, {
|
|
165
246
|
...(o.dbPath !== undefined ? { dbPath: o.dbPath } : {}),
|
|
166
247
|
...(o.pollIntervalMs !== undefined ? { pollIntervalMs: o.pollIntervalMs } : {}),
|
|
167
248
|
...(o.heartbeatMs !== undefined ? { heartbeatMs: o.heartbeatMs } : {}),
|
|
168
249
|
...(o.ledgerPath !== undefined ? { ledgerPath: o.ledgerPath } : {}),
|
|
169
250
|
...(o.editDir !== undefined ? { editDir: o.editDir } : {}),
|
|
170
251
|
...(o.clisJson !== undefined ? { clisJson: o.clisJson } : {}),
|
|
252
|
+
// The demo-kind gate (CREW-UX-9): a demo doc's step feedback is the demo seam's — but
|
|
253
|
+
// only when that seam is actually up. Probed per event (the demo seam arms below), so an
|
|
254
|
+
// un-armed demo seam gets an honest error status instead of a silent, unanswerable drop.
|
|
255
|
+
resolveDocsRoot: o.resolveDocsRoot ?? interactiveDocsRoot,
|
|
256
|
+
demoSeamArmed: () => demoSub !== null,
|
|
171
257
|
onRunFiled: fileRun,
|
|
172
258
|
log: (m) => app.log.warn(m),
|
|
173
259
|
});
|
|
174
|
-
if (
|
|
260
|
+
if (editSub !== null) {
|
|
261
|
+
const sub = editSub;
|
|
175
262
|
app.log.info('interactive-edit subscription armed (filter wicked.interactive.feedback.processed)');
|
|
176
263
|
app.addHook('onClose', async () => {
|
|
177
264
|
await sub.stop();
|
|
178
265
|
});
|
|
179
266
|
}
|
|
180
267
|
}
|
|
268
|
+
// Arm the opt-in interactive DEMO answering seam (CREW-UX-9 — the retired assist agent's
|
|
269
|
+
// Step 8). Same posture as the sibling seams: failure to arm is LOUD but non-fatal — a demo
|
|
270
|
+
// doc then keeps its placeholder, and this daemon must boot on a machine whose bus is
|
|
271
|
+
// broken. Armed beside the edit seam because the two split `feedback.processed` by the doc
|
|
272
|
+
// manifest's kind (demo → re-author the spec here; everything else → structural edit there).
|
|
273
|
+
if (options?.interactiveDemoEvents?.enabled === true && !refuseStubSeam('interactive-demo')) {
|
|
274
|
+
const o = options.interactiveDemoEvents;
|
|
275
|
+
demoSub = await startInteractiveDemoSubscriber(adapter, {
|
|
276
|
+
...(o.dbPath !== undefined ? { dbPath: o.dbPath } : {}),
|
|
277
|
+
...(o.pollIntervalMs !== undefined ? { pollIntervalMs: o.pollIntervalMs } : {}),
|
|
278
|
+
...(o.heartbeatMs !== undefined ? { heartbeatMs: o.heartbeatMs } : {}),
|
|
279
|
+
...(o.ledgerPath !== undefined ? { ledgerPath: o.ledgerPath } : {}),
|
|
280
|
+
...(o.demoDir !== undefined ? { demoDir: o.demoDir } : {}),
|
|
281
|
+
...(o.clisJson !== undefined ? { clisJson: o.clisJson } : {}),
|
|
282
|
+
resolveDocsRoot: o.resolveDocsRoot ?? interactiveDocsRoot,
|
|
283
|
+
onRunFiled: fileRun,
|
|
284
|
+
log: (m) => app.log.warn(m),
|
|
285
|
+
});
|
|
286
|
+
if (demoSub !== null) {
|
|
287
|
+
const sub = demoSub;
|
|
288
|
+
app.log.info('interactive-demo subscription armed (filters wicked.interactive.doc.created + feedback.processed, kind:demo)');
|
|
289
|
+
app.addHook('onClose', async () => {
|
|
290
|
+
await sub.stop();
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
// Arm the opt-in interactive CHAT answering seam (CREW-UX-5 — the iteration ask). Same
|
|
295
|
+
// posture again: failure to arm is LOUD but non-fatal. Armed AFTER the sibling seams so the
|
|
296
|
+
// per-doc serialization contract (no draft/edit/chat run races another on one doc) can
|
|
297
|
+
// consult their in-flight sets; docs roots resolve through the SAME per-project settings
|
|
298
|
+
// store the project routes and the interactive proxy share.
|
|
299
|
+
if (options?.interactiveChatEvents?.enabled === true && !refuseStubSeam('interactive-chat')) {
|
|
300
|
+
const o = options.interactiveChatEvents;
|
|
301
|
+
const sub = await startInteractiveChatSubscriber(adapter, {
|
|
302
|
+
...(o.dbPath !== undefined ? { dbPath: o.dbPath } : {}),
|
|
303
|
+
...(o.pollIntervalMs !== undefined ? { pollIntervalMs: o.pollIntervalMs } : {}),
|
|
304
|
+
...(o.heartbeatMs !== undefined ? { heartbeatMs: o.heartbeatMs } : {}),
|
|
305
|
+
...(o.ledgerPath !== undefined ? { ledgerPath: o.ledgerPath } : {}),
|
|
306
|
+
...(o.chatDir !== undefined ? { chatDir: o.chatDir } : {}),
|
|
307
|
+
...(o.clisJson !== undefined ? { clisJson: o.clisJson } : {}),
|
|
308
|
+
...(o.queueSweepMs !== undefined ? { queueSweepMs: o.queueSweepMs } : {}),
|
|
309
|
+
...(o.landingGateMs !== undefined ? { landingGateMs: o.landingGateMs } : {}),
|
|
310
|
+
resolveDocsRoot: o.resolveDocsRoot ??
|
|
311
|
+
((projectId) => resolveInteractiveRoot(projectId !== undefined ? projectSettings.get(projectId) : null)),
|
|
312
|
+
isDocBusy: (documentId) => (draftSub?.inFlightDocs().includes(documentId) ?? false) ||
|
|
313
|
+
(editSub?.inFlightDocs().includes(documentId) ?? false),
|
|
314
|
+
onRunFiled: fileRun,
|
|
315
|
+
log: (m) => app.log.warn(m),
|
|
316
|
+
});
|
|
317
|
+
if (sub !== null) {
|
|
318
|
+
app.log.info('interactive-chat subscription armed (filter wicked.interactive.chat.posted)');
|
|
319
|
+
app.addHook('onClose', async () => {
|
|
320
|
+
await sub.stop();
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
// The worker stall watchdog (crew#287). Built BEFORE the single CoreEvent subscription below
|
|
325
|
+
// so every relayed frame stamps its run's liveness clock; armed (sweep interval) further down
|
|
326
|
+
// beside the seat-health probe, under the same test-runner gate. Detection only: its sole
|
|
327
|
+
// outputs are one synthetic `workerStalled` /ws frame per quiet period and a warn log.
|
|
328
|
+
const stallWatchdog = new WorkerStallWatchdog({
|
|
329
|
+
listExecuting: async () => (await adapter.sessionsDetail())
|
|
330
|
+
.filter((v) => v.session.status === 'executing')
|
|
331
|
+
.map((v) => ({ id: v.session.id, ord: v.session.unit_ix })),
|
|
332
|
+
broadcast: (frame) => broadcast(frame),
|
|
333
|
+
stallMinutes: async () => options?.stallWatchdog?.stallMinutes ??
|
|
334
|
+
(await adapter.getSettings()).workerStallMinutes ??
|
|
335
|
+
DEFAULT_WORKER_STALL_MINUTES,
|
|
336
|
+
log: (m) => app.log.warn(m),
|
|
337
|
+
});
|
|
181
338
|
// The daemon's single CoreEvent subscription fans out here: cache gate prompts
|
|
182
339
|
// (§3.3), cache elicitation prompts (DES-002), route terminal frames to their owning
|
|
183
340
|
// per-terminal socket (by id, DES-TERMINAL-001 §6), then forward every frame — tagged
|
|
@@ -187,10 +344,17 @@ export async function createServer(adapter, options) {
|
|
|
187
344
|
// Unregistered on close: a process can build more than one server over the same
|
|
188
345
|
// adapter (tests do), and a closed server's caches must stop consuming events —
|
|
189
346
|
// otherwise every discarded server keeps folding state forever (listener leak).
|
|
347
|
+
const stallWatchdogArmed = options?.stallWatchdog?.enabled ??
|
|
348
|
+
!(process.env['VITEST'] !== undefined || process.env['NODE_ENV'] === 'test');
|
|
190
349
|
const offEvent = adapter.onEvent((event) => {
|
|
191
350
|
gateCache.ingest(event);
|
|
192
351
|
elicitationCache.ingest(event);
|
|
193
352
|
seatHealth.ingest(event);
|
|
353
|
+
// Only feed the watchdog when its sweep is (or will be) armed: sweeping is what
|
|
354
|
+
// prunes its per-run maps, so ingesting while disabled grows without bound
|
|
355
|
+
// (Copilot on #301).
|
|
356
|
+
if (stallWatchdogArmed)
|
|
357
|
+
stallWatchdog.ingest(event);
|
|
194
358
|
terminals.route(event);
|
|
195
359
|
const session = typeof event.session === 'string' ? event.session : undefined;
|
|
196
360
|
const projectId = session !== undefined ? membershipIndex.projectOf(session) : undefined;
|
|
@@ -235,6 +399,16 @@ export async function createServer(adapter, options) {
|
|
|
235
399
|
probe.stop();
|
|
236
400
|
});
|
|
237
401
|
}
|
|
402
|
+
// Arm the stall watchdog's sweep (crew#287). Same gate as the probe: ON in the daemon, OFF
|
|
403
|
+
// under a test runner unless a test opts in — a suite building servers over stub adapters
|
|
404
|
+
// must not have a background interval calling `sessionsDetail()` on them.
|
|
405
|
+
const stallCfg = options?.stallWatchdog;
|
|
406
|
+
if (stallWatchdogArmed) {
|
|
407
|
+
stallWatchdog.start(stallCfg?.sweepIntervalMs);
|
|
408
|
+
app.addHook('onClose', async () => {
|
|
409
|
+
stallWatchdog.stop();
|
|
410
|
+
});
|
|
411
|
+
}
|
|
238
412
|
app.addHook('onRequest', async (req, reply) => {
|
|
239
413
|
const origin = req.headers.origin;
|
|
240
414
|
// Loopback origins are always allowed (the studio on another localhost
|
|
@@ -288,7 +462,8 @@ export async function createServer(adapter, options) {
|
|
|
288
462
|
bus: projectBus,
|
|
289
463
|
index: membershipIndex,
|
|
290
464
|
log: (m) => app.log.warn(m),
|
|
291
|
-
|
|
465
|
+
settings: projectSettings,
|
|
466
|
+
}, { audit, authMode: auth.mode }, { seatHealth, retryIndex, guidanceIndex });
|
|
292
467
|
// The UI-emittable direction of the interactive seam. Registered unconditionally (a null relay
|
|
293
468
|
// answers 503, not 404) and BEFORE the static/SPA fallback below, like every other API route.
|
|
294
469
|
registerInteractiveEventRoutes(app, interactiveRelay);
|
package/dist/api/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/api/server.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAClD,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAoB,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,gCAAgC,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,MAAM,4BAA4B,CAAC;AACrG,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAkB,MAAM,kBAAkB,CAAC;AAC3F,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,yEAAyE;AACzE,4EAA4E;AAC5E,qEAAqE;AACrE,MAAM,eAAe,GAAG,yCAAyC,CAAC;AAElE;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9D,CAAC;AAgHD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAoB,EACpB,OAA6B;IAE7B,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;IAChD,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,2FAA2F;IAC3F,oFAAoF;IACpF,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE3C,6EAA6E;IAC7E,2EAA2E;IAC3E,4EAA4E;IAC5E,yEAAyE;IACzE,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,0CAA0C;IACjE,CAAC,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yFAAyF;IACzF,yFAAyF;IACzF,2FAA2F;IAC3F,0DAA0D;IAC1D,qBAAqB,CAAC,CAAC,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAExE,0FAA0F;IAC1F,4FAA4F;IAC5F,4FAA4F;IAC5F,qFAAqF;IACrF,yEAAyE;IACzE,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,UAAU,GACd,OAAO,EAAE,aAAa,EAAE,QAAQ,KAAK,IAAI;QACvC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,eAAe,CAAC;YACpB,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,SAAS;gBAC9C,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE;gBAC1C,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,cAAc,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,CAAC,cAAc,EAAE;gBAC1D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;IACT,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACrF,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6FAA6F;IAC7F,0FAA0F;IAC1F,gGAAgG;IAChG,2FAA2F;IAC3F,6EAA6E;IAC7E,MAAM,gBAAgB,GACpB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,KAAK,IAAI;QAC5C,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,uBAAuB,CAAC;YAC5B,GAAG,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,KAAK,SAAS;gBACnD,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,MAAM,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,EAAE,kBAAkB,EAAE,cAAc,KAAK,SAAS;gBAC3D,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,kBAAkB,CAAC,cAAc,EAAE;gBAC/D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;IACT,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC9F,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;4EAEwE;IACxE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAQ,EAAE;QACzD,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACtC,UAAU,EAAE,IAAI,CACd,mBAAmB;QACnB,uEAAuE;QACvE,qEAAqE;QACrE,uEAAuE;QACvE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,EACzF,qBAAqB,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAChE,CAAC;IACJ,CAAC,CAAC;IAEF,0EAA0E;IAC1E,4EAA4E;IAC5E,IAAI,OAAO,EAAE,YAAY,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;QACxD,MAAM,GAAG,GAAG,MAAM,qBAAqB,CAAC,WAAW,EAAE;YACnD,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACvE,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2FAA2F;IAC3F,yFAAyF;IACzF,wEAAwE;IACxE,IAAI,OAAO,EAAE,sBAAsB,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC;QACzC,MAAM,GAAG,GAAG,MAAM,+BAA+B,CAAC,OAAO,EAAE;YACzD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAC;YAC7F,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uFAAuF;IACvF,yFAAyF;IACzF,6FAA6F;IAC7F,IAAI,OAAO,EAAE,qBAAqB,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,OAAO,EAAE;YACxD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;YACnG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,qFAAqF;IACrF,sFAAsF;IACtF,kFAAkF;IAClF,sEAAsE;IACtE,EAAE;IACF,gFAAgF;IAChF,gFAAgF;IAChF,gFAAgF;IAChF,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACzC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,MAAM,SAAS,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAE,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,SAAS,EAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChG,4FAA4F;QAC5F,qEAAqE;QACrE,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,IAAI,OAAO,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC1F,KAAK,CAAC,KAAK,IAAI,EAAE;gBACf,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;oBACzD,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;oBAC/C,MAAM,QAAQ,GACZ,OAAO,KAAK,IAAI;wBACd,CAAC,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;wBAChF,CAAC,CAAC,IAAI,CAAC;oBACX,oFAAoF;oBACpF,qFAAqF;oBACrF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,uBAAuB,CAC3B,OAAO,EACP,OAAO,EAAE,KAAK,IAAI,WAAW,SAAS,EAAE,EACxC,OAAO,EACP,QAAQ,EACR,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CACvB,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,0CAA0C,OAAO,YAC/C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;IACH,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAChC,QAAQ,EAAE,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,2FAA2F;IAC3F,2FAA2F;IAC3F,sFAAsF;IACtF,MAAM,QAAQ,GAAG,OAAO,EAAE,eAAe,CAAC;IAC1C,MAAM,eAAe,GACnB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC;IAC5E,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,oBAAoB,CAChC,UAAU,EACV,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAiB,EACzC;YACE,GAAG,CAAC,QAAQ,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,QAAQ,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CACF,CAAC;QACF,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAClC,uEAAuE;QACvE,2EAA2E;QAC3E,uEAAuE;QACvE,wEAAwE;QACxE,uEAAuE;QACvE,uEAAuE;QACvE,MAAM,OAAO,GACX,MAAM,KAAK,SAAS;YACpB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC3B,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU;oBACvB,CAAC,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,KAAK,CAAC,MAAM,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAC;YACpD,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC/B,KAAK,CAAC,MAAM,CAAC,8BAA8B,EAAE,mCAAmC,CAAC,CAAC;YAClF,KAAK,CAAC,MAAM,CAAC,8BAA8B,EAAE,6BAA6B,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,gFAAgF;IAChF,kEAAkE;IAClE,GAAG,CAAC,oBAAoB,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QACvF,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAc,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAY,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAErC,2EAA2E;IAC3E,iEAAiE;IACjE,6EAA6E;IAC7E,yEAAyE;IACzE,0EAA0E;IAC1E,2EAA2E;IAC3E,uDAAuD;IACvD,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAE7B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7C,6EAA6E;QAC7E,cAAc,CAAC,MAA8B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5C,cAAc,CACZ,GAAG,EACH,OAAO,EACP,SAAS,EACT,gBAAgB,EAChB,WAAW,EACX;QACE,GAAG,EAAE,UAAU;QACf,KAAK,EAAE,eAAe;QACtB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;KAC5B,EACD,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAC9B,EAAE,UAAU,EAAE,CACf,CAAC;IAEF,+FAA+F;IAC/F,8FAA8F;IAC9F,8BAA8B,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAEtD,4EAA4E;IAC5E,4EAA4E;IAC5E,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;IAC9D,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE;YAChC,IAAI,EAAE,UAAU;YAChB,yEAAyE;YACzE,yEAAyE;YACzE,0EAA0E;YAC1E,yEAAyE;YACzE,yEAAyE;YACzE,QAAQ,EAAE,IAAI;YACd,uEAAuE;YACvE,qDAAqD;YACrD,YAAY,EAAE,KAAK;YACnB,KAAK,EAAE,CAAC,YAAY,CAAC;YACrB,iFAAiF;YACjF,sFAAsF;YACtF,yFAAyF;YACzF,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBAC9B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACvC,IAAI,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC9B,uEAAuE;oBACvE,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBAC5C,CAAC;qBAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBAClC,mDAAmD;oBACnD,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,qCAAqC,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,0EAA0E;QAC1E,2EAA2E;QAC3E,kEAAkE;QAClE,GAAG,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACpC,IACE,GAAG,CAAC,MAAM,KAAK,KAAK;gBACpB,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAC5B,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAC1B,CAAC;gBACD,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBAC1C,8DAA8D;gBAC9D,4DAA4D;gBAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,wEAAwE;QACxE,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,8BAA8B,UAAU,qCAAqC,CAC9E,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAoB,EACpB,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,WAAW,EAClB,OAA6B;IAE7B,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACtE,MAAM,SAAS,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAmC,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC;IAC1E,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AACnD,CAAC"}
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/api/server.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAClD,OAAO,aAAa,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAoB,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,+BAA+B,EAAE,MAAM,gCAAgC,CAAC;AACjF,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,MAAM,4BAA4B,CAAC;AACrG,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAkB,MAAM,kBAAkB,CAAC;AAC3F,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAEhE,yEAAyE;AACzE,4EAA4E;AAC5E,qEAAqE;AACrE,MAAM,eAAe,GAAG,yCAAyC,CAAC;AAElE;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9D,CAAC;AA+LD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAoB,EACpB,OAA6B;IAE7B,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/E,4FAA4F;IAC5F,4FAA4F;IAC5F,6FAA6F;IAC7F,sEAAsE;IACtE,GAAG,CAAC,QAAQ,CAAC,kBAAkB,EAAE,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;IAChD,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,2FAA2F;IAC3F,oFAAoF;IACpF,MAAM,UAAU,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE3C,6EAA6E;IAC7E,2EAA2E;IAC3E,4EAA4E;IAC5E,yEAAyE;IACzE,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,0CAA0C;IACjE,CAAC,CAAC,CAAC;IAEH,2FAA2F;IAC3F,yFAAyF;IACzF,yFAAyF;IACzF,2FAA2F;IAC3F,0DAA0D;IAC1D,qBAAqB,CAAC,CAAC,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAExE,0FAA0F;IAC1F,4FAA4F;IAC5F,4FAA4F;IAC5F,qFAAqF;IACrF,yEAAyE;IACzE,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAC9C,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,yFAAyF;IACzF,6FAA6F;IAC7F,6FAA6F;IAC7F,kEAAkE;IAClE,MAAM,eAAe,GAAG,IAAI,oBAAoB,EAAE,CAAC;IACnD,2FAA2F;IAC3F,oFAAoF;IACpF,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;IACpC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,2FAA2F;IAC3F,oEAAoE;IACpE,MAAM,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;IAC1C,MAAM,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,UAAU,GACd,OAAO,EAAE,aAAa,EAAE,QAAQ,KAAK,IAAI;QACvC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,eAAe,CAAC;YACpB,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,SAAS;gBAC9C,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE;gBAC1C,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,cAAc,KAAK,SAAS;gBACtD,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,CAAC,cAAc,EAAE;gBAC1D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;IACT,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACrF,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6FAA6F;IAC7F,0FAA0F;IAC1F,gGAAgG;IAChG,2FAA2F;IAC3F,6EAA6E;IAC7E,MAAM,gBAAgB,GACpB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,KAAK,IAAI;QAC5C,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,uBAAuB,CAAC;YAC5B,GAAG,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,KAAK,SAAS;gBACnD,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,kBAAkB,CAAC,MAAM,EAAE;gBAC/C,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,OAAO,EAAE,kBAAkB,EAAE,cAAc,KAAK,SAAS;gBAC3D,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,kBAAkB,CAAC,cAAc,EAAE;gBAC/D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;IACT,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC9F,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;4EAEwE;IACxE,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAQ,EAAE;QACzD,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACtC,UAAU,EAAE,IAAI,CACd,mBAAmB;QACnB,uEAAuE;QACvE,qEAAqE;QACrE,uEAAuE;QACvE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,EACzF,qBAAqB,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAChE,CAAC;IACJ,CAAC,CAAC;IAEF;;;qDAGiD;IACjD,MAAM,mBAAmB,GAAG,CAAC,SAA6B,EAAU,EAAE,CACpE,sBAAsB,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE1F,0EAA0E;IAC1E,4EAA4E;IAC5E,IAAI,OAAO,EAAE,YAAY,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC;QACxD,MAAM,GAAG,GAAG,MAAM,qBAAqB,CAAC,WAAW,EAAE;YACnD,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACvE,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,+FAA+F;IAC/F,EAAE;IACF,+FAA+F;IAC/F,iGAAiG;IACjG,0FAA0F;IAC1F,gGAAgG;IAChG,0FAA0F;IAC1F,yFAAyF;IACzF,gGAAgG;IAChG,4CAA4C;IAC5C,EAAE;IACF,6EAA6E;IAC7E,2FAA2F;IAC3F,+FAA+F;IAC/F,6FAA6F;IAC7F,+DAA+D;IAC/D,iGAAiG;IACjG,wFAAwF;IACxF,8FAA8F;IAC9F,iGAAiG;IACjG,qEAAqE;IACrE,EAAE;IACF,gGAAgG;IAChG,2FAA2F;IAC3F,8FAA8F;IAC9F,gGAAgG;IAChG,iGAAiG;IACjG,8FAA8F;IAC9F,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC;IACzC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAW,EAAE;QAC/C,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAC;QAC9B,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,GAAG,IAAI,kFAAkF;YACvF,mFAAmF;YACnF,yFAAyF;YACzF,kFAAkF;YAClF,2FAA2F;YAC3F,wEAAwE,CAC3E,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,2FAA2F;IAC3F,yFAAyF;IACzF,4FAA4F;IAC5F,qFAAqF;IACrF,IAAI,QAAQ,GAAgE,IAAI,CAAC;IACjF,IAAI,OAAO,EAAE,sBAAsB,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC9F,MAAM,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC;QACzC,QAAQ,GAAG,MAAM,+BAA+B,CAAC,OAAO,EAAE;YACxD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,QAAQ,CAAC;YACrB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAC;YAC7F,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2FAA2F;IAC3F,6FAA6F;IAC7F,uFAAuF;IACvF,wFAAwF;IACxF,IAAI,OAAO,GAA+D,IAAI,CAAC;IAE/E,uFAAuF;IACvF,yFAAyF;IACzF,6FAA6F;IAC7F,IAAI,OAAO,GAA+D,IAAI,CAAC;IAC/E,IAAI,OAAO,EAAE,qBAAqB,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5F,MAAM,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;QACxC,OAAO,GAAG,MAAM,8BAA8B,CAAC,OAAO,EAAE;YACtD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,sFAAsF;YACtF,yFAAyF;YACzF,yFAAyF;YACzF,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,mBAAmB;YACzD,aAAa,EAAE,GAAG,EAAE,CAAC,OAAO,KAAK,IAAI;YACrC,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,OAAO,CAAC;YACpB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;YACnG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yFAAyF;IACzF,4FAA4F;IAC5F,sFAAsF;IACtF,2FAA2F;IAC3F,6FAA6F;IAC7F,IAAI,OAAO,EAAE,qBAAqB,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5F,MAAM,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;QACxC,OAAO,GAAG,MAAM,8BAA8B,CAAC,OAAO,EAAE;YACtD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,eAAe,EAAE,CAAC,CAAC,eAAe,IAAI,mBAAmB;YACzD,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,OAAO,CAAC;YACpB,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,8GAA8G,CAC/G,CAAC;YACF,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uFAAuF;IACvF,4FAA4F;IAC5F,uFAAuF;IACvF,yFAAyF;IACzF,4DAA4D;IAC5D,IAAI,OAAO,EAAE,qBAAqB,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5F,MAAM,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,OAAO,EAAE;YACxD,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,GAAG,CAAC,CAAC,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,eAAe,EACb,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,SAAS,EAAE,EAAE,CACb,sBAAsB,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5F,SAAS,EAAE,CAAC,UAAU,EAAE,EAAE,CACxB,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;gBACxD,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YACzD,UAAU,EAAE,OAAO;YACnB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,6EAA6E,CAAC,CAAC;YAC5F,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,8FAA8F;IAC9F,0FAA0F;IAC1F,uFAAuF;IACvF,MAAM,aAAa,GAAG,IAAI,mBAAmB,CAAC;QAC5C,aAAa,EAAE,KAAK,IAAI,EAAE,CACxB,CAAC,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC;aAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC;aAC/C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC;QACtC,YAAY,EAAE,KAAK,IAAI,EAAE,CACvB,OAAO,EAAE,aAAa,EAAE,YAAY;YACpC,CAAC,MAAM,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,kBAAkB;YAChD,4BAA4B;QAC9B,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;KAC5B,CAAC,CAAC;IAEH,+EAA+E;IAC/E,qFAAqF;IACrF,sFAAsF;IACtF,kFAAkF;IAClF,sEAAsE;IACtE,EAAE;IACF,gFAAgF;IAChF,gFAAgF;IAChF,gFAAgF;IAChF,MAAM,kBAAkB,GACtB,OAAO,EAAE,aAAa,EAAE,OAAO;QAC/B,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QACzC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,gFAAgF;QAChF,2EAA2E;QAC3E,qBAAqB;QACrB,IAAI,kBAAkB;YAAE,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpD,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,MAAM,SAAS,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzF,SAAS,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAE,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,SAAS,EAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChG,4FAA4F;QAC5F,qEAAqE;QACrE,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,IAAI,OAAO,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC1F,KAAK,CAAC,KAAK,IAAI,EAAE;gBACf,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC;oBACzD,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;oBAC/C,MAAM,QAAQ,GACZ,OAAO,KAAK,IAAI;wBACd,CAAC,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;wBAChF,CAAC,CAAC,IAAI,CAAC;oBACX,oFAAoF;oBACpF,qFAAqF;oBACrF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,uBAAuB,CAC3B,OAAO,EACP,OAAO,EAAE,KAAK,IAAI,WAAW,SAAS,EAAE,EACxC,OAAO,EACP,QAAQ,EACR,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CACvB,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,0CAA0C,OAAO,YAC/C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CACjD,EAAE,CACH,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;IACH,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;QAChC,QAAQ,EAAE,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,2FAA2F;IAC3F,2FAA2F;IAC3F,sFAAsF;IACtF,MAAM,QAAQ,GAAG,OAAO,EAAE,eAAe,CAAC;IAC1C,MAAM,eAAe,GACnB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC;IAC5E,IAAI,QAAQ,EAAE,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,oBAAoB,CAChC,UAAU,EACV,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAiB,EACzC;YACE,GAAG,CAAC,QAAQ,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,QAAQ,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC5B,CACF,CAAC;QACF,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAED,2FAA2F;IAC3F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAa,CAAC;IACxC,IAAI,kBAAkB,EAAE,CAAC;QACvB,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QAC/C,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAChC,aAAa,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAClC,uEAAuE;QACvE,2EAA2E;QAC3E,uEAAuE;QACvE,wEAAwE;QACxE,uEAAuE;QACvE,uEAAuE;QACvE,MAAM,OAAO,GACX,MAAM,KAAK,SAAS;YACpB,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC3B,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU;oBACvB,CAAC,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,KAAK,CAAC,MAAM,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAC;YACpD,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC/B,KAAK,CAAC,MAAM,CAAC,8BAA8B,EAAE,mCAAmC,CAAC,CAAC;YAClF,KAAK,CAAC,MAAM,CAAC,8BAA8B,EAAE,6BAA6B,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,gFAAgF;IAChF,kEAAkE;IAClE,GAAG,CAAC,oBAAoB,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QACvF,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACrF,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAc,CAAC,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAY,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAErC,2EAA2E;IAC3E,iEAAiE;IACjE,6EAA6E;IAC7E,yEAAyE;IACzE,0EAA0E;IAC1E,2EAA2E;IAC3E,uDAAuD;IACvD,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAE7B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7C,6EAA6E;QAC7E,cAAc,CAAC,MAA8B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5C,cAAc,CACZ,GAAG,EACH,OAAO,EACP,SAAS,EACT,gBAAgB,EAChB,WAAW,EACX;QACE,GAAG,EAAE,UAAU;QACf,KAAK,EAAE,eAAe;QACtB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3B,QAAQ,EAAE,eAAe;KAC1B,EACD,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAC9B,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,CAC1C,CAAC;IAEF,+FAA+F;IAC/F,8FAA8F;IAC9F,8BAA8B,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAEtD,4EAA4E;IAC5E,4EAA4E;IAC5E,2EAA2E;IAC3E,oEAAoE;IACpE,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;IAC9D,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE;YAChC,IAAI,EAAE,UAAU;YAChB,yEAAyE;YACzE,yEAAyE;YACzE,0EAA0E;YAC1E,yEAAyE;YACzE,yEAAyE;YACzE,QAAQ,EAAE,IAAI;YACd,uEAAuE;YACvE,qDAAqD;YACrD,YAAY,EAAE,KAAK;YACnB,KAAK,EAAE,CAAC,YAAY,CAAC;YACrB,iFAAiF;YACjF,sFAAsF;YACtF,yFAAyF;YACzF,UAAU,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBAC9B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACvC,IAAI,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC9B,uEAAuE;oBACvE,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBAC5C,CAAC;qBAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBAClC,mDAAmD;oBACnD,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,qCAAqC,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,0EAA0E;QAC1E,2EAA2E;QAC3E,kEAAkE;QAClE,GAAG,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACpC,IACE,GAAG,CAAC,MAAM,KAAK,KAAK;gBACpB,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;gBAC5B,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAC1B,CAAC;gBACD,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBAC1C,8DAA8D;gBAC9D,4DAA4D;gBAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YACxD,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,wEAAwE;QACxE,GAAG,CAAC,GAAG,CAAC,IAAI,CACV,8BAA8B,UAAU,qCAAqC,CAC9E,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAoB,EACpB,IAAI,GAAG,IAAI,EACX,IAAI,GAAG,WAAW,EAClB,OAA6B;IAE7B,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACtE,MAAM,SAAS,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAmC,SAAS,IAAI,SAAS,EAAE,CAAC,CAAC;IAC1E,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { CoreEvent } from '../core/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The synthetic /ws frame (crew#287). LOCAL type: `wicked-crew-api-types` 0.6.0 does not name
|
|
4
|
+
* it yet — the contract's permissive `CoreEvent` (index signature; DES-STUDIO-001 §5.1) is what
|
|
5
|
+
* lets it ride the wire to consumers today. NOTE for the next api-types release: add this frame
|
|
6
|
+
* (and `SystemSettings.workerStallMinutes`) to the published contract. A `type` alias on
|
|
7
|
+
* purpose, not an `interface`: only anonymous object types satisfy `CoreEvent`'s index
|
|
8
|
+
* signature, which is what lets the frame flow into the CoreEvent-typed broadcast seam.
|
|
9
|
+
*/
|
|
10
|
+
export type WorkerStalledFrame = {
|
|
11
|
+
type: 'workerStalled';
|
|
12
|
+
/** The stalled run's id. */
|
|
13
|
+
session: string;
|
|
14
|
+
/** The current unit ord, when any observed event or the run header named one. */
|
|
15
|
+
ord?: number;
|
|
16
|
+
/** How long the run has been silent when the frame fired, ms. */
|
|
17
|
+
quietForMs: number;
|
|
18
|
+
};
|
|
19
|
+
/** Default sweep cadence — frequent enough that `quietForMs` is at most ~30s stale. */
|
|
20
|
+
export declare const DEFAULT_SWEEP_INTERVAL_MS = 30000;
|
|
21
|
+
/** The slice of a run the sweep needs; the server maps it from `sessionsDetail()`. */
|
|
22
|
+
export interface ExecutingRun {
|
|
23
|
+
id: string;
|
|
24
|
+
/** The run's current unit ord when the caller knows it (`session.unit_ix`). */
|
|
25
|
+
ord?: number;
|
|
26
|
+
}
|
|
27
|
+
export interface StallWatchdogDeps {
|
|
28
|
+
/** Runs whose engine status is `executing`, right now. Read per sweep, never cached. */
|
|
29
|
+
listExecuting: () => Promise<ExecutingRun[]>;
|
|
30
|
+
/** The /ws fan-out for the synthetic frame (`events/bus.ts` `broadcast` in the daemon). */
|
|
31
|
+
broadcast: (frame: WorkerStalledFrame) => void;
|
|
32
|
+
/** Threshold resolver, minutes — read per sweep so a PUT /settings change applies live. */
|
|
33
|
+
stallMinutes?: () => number | Promise<number | undefined> | undefined;
|
|
34
|
+
/** Clock (tests stub it). */
|
|
35
|
+
now?: () => number;
|
|
36
|
+
/** Wired to the daemon's warn logger — a stall is an operator-attention signal. */
|
|
37
|
+
log?: (m: string) => void;
|
|
38
|
+
}
|
|
39
|
+
export declare class WorkerStallWatchdog {
|
|
40
|
+
private readonly deps;
|
|
41
|
+
/** run id → epoch ms of the last engine event observed for it (or the seed, see sweep). */
|
|
42
|
+
private readonly lastEventAt;
|
|
43
|
+
/** run id → last unit ord any of its events named. */
|
|
44
|
+
private readonly lastOrd;
|
|
45
|
+
/** Runs already alerted for the CURRENT quiet period (cleared by any new event). */
|
|
46
|
+
private readonly alerted;
|
|
47
|
+
private readonly now;
|
|
48
|
+
private readonly log;
|
|
49
|
+
private handle;
|
|
50
|
+
private sweeping;
|
|
51
|
+
constructor(deps: StallWatchdogDeps);
|
|
52
|
+
/** Fold one relay frame: any event for a run is proof of life — stamp its clock and RE-ARM. */
|
|
53
|
+
ingest(event: CoreEvent): void;
|
|
54
|
+
/** One detection pass. Public so tests drive it directly; the armed interval calls it too. */
|
|
55
|
+
sweep(): Promise<void>;
|
|
56
|
+
/** Arm the periodic sweep. Unref'd — the daemon's server lifecycle owns shutdown. */
|
|
57
|
+
start(intervalMs?: number): void;
|
|
58
|
+
stop(): void;
|
|
59
|
+
/** Threshold in ms — an unset/invalid setting falls back to the 15-minute default. */
|
|
60
|
+
private thresholdMs;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=stall-watchdog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stall-watchdog.d.ts","sourceRoot":"","sources":["../../src/api/stall-watchdog.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAGlD;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,eAAe,CAAC;IACtB,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,uFAAuF;AACvF,eAAO,MAAM,yBAAyB,QAAS,CAAC;AAEhD,sFAAsF;AACtF,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,+EAA+E;IAC/E,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,wFAAwF;IACxF,aAAa,EAAE,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC7C,2FAA2F;IAC3F,SAAS,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAC/C,2FAA2F;IAC3F,YAAY,CAAC,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IACtE,6BAA6B;IAC7B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,mFAAmF;IACnF,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3B;AAED,qBAAa,mBAAmB;IAYlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAXjC,2FAA2F;IAC3F,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;IACzD,sDAAsD;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,oFAAoF;IACpF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,QAAQ,CAAS;gBAEI,IAAI,EAAE,iBAAiB;IAKpD,+FAA+F;IAC/F,MAAM,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI;IAS9B,8FAA8F;IACxF,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsD5B,qFAAqF;IACrF,KAAK,CAAC,UAAU,SAA4B,GAAG,IAAI;IAUnD,IAAI,IAAI,IAAI;IAKZ,sFAAsF;YACxE,WAAW;CAc1B"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// Worker stall watchdog (crew#287): platform-native liveness detection for executing runs.
|
|
2
|
+
//
|
|
3
|
+
// A wedged worker emits nothing — no unitOutputDelta, no gate frames — and its run sits in
|
|
4
|
+
// `executing` until an operator notices by absence (run 8aa1cd42: a design unit burned 3+ hours
|
|
5
|
+
// with zero output before anyone looked). The daemon already observes every engine event on its
|
|
6
|
+
// single CoreEvent relay, so "time since the last event" is cheap to compute right here:
|
|
7
|
+
//
|
|
8
|
+
// - `ingest` stamps the run's last-observed-event time on EVERY frame carrying a `session`
|
|
9
|
+
// (any CoreEvent counts, `unitOutputDelta` included) and RE-ARMS the alert for that run.
|
|
10
|
+
// - a periodic sweep lists the runs whose engine status is `executing` (`awaiting_human` is
|
|
11
|
+
// quiet BY DESIGN and never counts) and, when a run has been silent past the threshold
|
|
12
|
+
// (`workerStallMinutes` setting, default 15), broadcasts ONE synthetic frame on /ws —
|
|
13
|
+
// `{ type: "workerStalled", session, ord?, quietForMs }` — and logs it at warn.
|
|
14
|
+
// - once per quiet period: after the frame fires, the run stays alerted until a NEW event for
|
|
15
|
+
// it arrives; a second quiet period then emits a second frame.
|
|
16
|
+
//
|
|
17
|
+
// DETECTION ONLY. The watchdog never cancels, mutates, or re-dispatches a run — the operator
|
|
18
|
+
// decides. The synthetic frame goes straight to the /ws fan-out, never back through the engine
|
|
19
|
+
// relay, so it cannot stamp or re-arm itself.
|
|
20
|
+
import { DEFAULT_WORKER_STALL_MINUTES } from '../core/types.js';
|
|
21
|
+
/** Default sweep cadence — frequent enough that `quietForMs` is at most ~30s stale. */
|
|
22
|
+
export const DEFAULT_SWEEP_INTERVAL_MS = 30_000;
|
|
23
|
+
export class WorkerStallWatchdog {
|
|
24
|
+
deps;
|
|
25
|
+
/** run id → epoch ms of the last engine event observed for it (or the seed, see sweep). */
|
|
26
|
+
lastEventAt = new Map();
|
|
27
|
+
/** run id → last unit ord any of its events named. */
|
|
28
|
+
lastOrd = new Map();
|
|
29
|
+
/** Runs already alerted for the CURRENT quiet period (cleared by any new event). */
|
|
30
|
+
alerted = new Set();
|
|
31
|
+
now;
|
|
32
|
+
log;
|
|
33
|
+
handle = null;
|
|
34
|
+
sweeping = false;
|
|
35
|
+
constructor(deps) {
|
|
36
|
+
this.deps = deps;
|
|
37
|
+
this.now = deps.now ?? Date.now;
|
|
38
|
+
this.log = deps.log ?? (() => undefined);
|
|
39
|
+
}
|
|
40
|
+
/** Fold one relay frame: any event for a run is proof of life — stamp its clock and RE-ARM. */
|
|
41
|
+
ingest(event) {
|
|
42
|
+
const session = typeof event.session === 'string' && event.session.length > 0 ? event.session : undefined;
|
|
43
|
+
if (session === undefined)
|
|
44
|
+
return;
|
|
45
|
+
this.lastEventAt.set(session, this.now());
|
|
46
|
+
if (typeof event.ord === 'number')
|
|
47
|
+
this.lastOrd.set(session, event.ord);
|
|
48
|
+
this.alerted.delete(session);
|
|
49
|
+
}
|
|
50
|
+
/** One detection pass. Public so tests drive it directly; the armed interval calls it too. */
|
|
51
|
+
async sweep() {
|
|
52
|
+
if (this.sweeping)
|
|
53
|
+
return; // a slow engine read must never stack sweeps
|
|
54
|
+
this.sweeping = true;
|
|
55
|
+
try {
|
|
56
|
+
let executing;
|
|
57
|
+
try {
|
|
58
|
+
executing = await this.deps.listExecuting();
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
this.log(`[stall-watchdog] run listing failed, skipping sweep: ${String(err)}`);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const ids = new Set(executing.map((r) => r.id));
|
|
65
|
+
// Prune state for runs no longer executing (completed / cancelled / awaiting_human):
|
|
66
|
+
// their silence is not a stall, and a later return to executing starts a fresh clock.
|
|
67
|
+
for (const key of [...this.lastEventAt.keys()]) {
|
|
68
|
+
if (!ids.has(key)) {
|
|
69
|
+
this.lastEventAt.delete(key);
|
|
70
|
+
this.lastOrd.delete(key);
|
|
71
|
+
this.alerted.delete(key);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
const thresholdMs = await this.thresholdMs();
|
|
75
|
+
const now = this.now();
|
|
76
|
+
for (const run of executing) {
|
|
77
|
+
const last = this.lastEventAt.get(run.id);
|
|
78
|
+
if (last === undefined) {
|
|
79
|
+
// First observation with no event on record — e.g. the run was already executing
|
|
80
|
+
// (and possibly already wedged) before this daemon booted. Silence before the relay
|
|
81
|
+
// existed is unknowable, so the quiet clock starts HERE; the threshold still trips
|
|
82
|
+
// one full quiet period later, which is exactly the restart-survival the issue asks.
|
|
83
|
+
this.lastEventAt.set(run.id, now);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const quietForMs = now - last;
|
|
87
|
+
if (quietForMs < thresholdMs || this.alerted.has(run.id))
|
|
88
|
+
continue;
|
|
89
|
+
this.alerted.add(run.id);
|
|
90
|
+
const ord = this.lastOrd.get(run.id) ?? run.ord;
|
|
91
|
+
this.deps.broadcast({
|
|
92
|
+
type: 'workerStalled',
|
|
93
|
+
session: run.id,
|
|
94
|
+
...(ord !== undefined ? { ord } : {}),
|
|
95
|
+
quietForMs,
|
|
96
|
+
});
|
|
97
|
+
this.log(`[stall-watchdog] run ${run.id}${ord !== undefined ? ` (unit ${ord})` : ''} silent for ` +
|
|
98
|
+
`${(quietForMs / 60_000).toFixed(1)} min — workerStalled frame broadcast ` +
|
|
99
|
+
`(detection only: the run was NOT touched; the operator decides)`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
this.sweeping = false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/** Arm the periodic sweep. Unref'd — the daemon's server lifecycle owns shutdown. */
|
|
107
|
+
start(intervalMs = DEFAULT_SWEEP_INTERVAL_MS) {
|
|
108
|
+
if (this.handle !== null)
|
|
109
|
+
return;
|
|
110
|
+
this.handle = setInterval(() => {
|
|
111
|
+
this.sweep().catch((err) => {
|
|
112
|
+
this.log(`[stall-watchdog] sweep failed: ${String(err)}`);
|
|
113
|
+
});
|
|
114
|
+
}, intervalMs);
|
|
115
|
+
this.handle.unref?.();
|
|
116
|
+
}
|
|
117
|
+
stop() {
|
|
118
|
+
if (this.handle !== null)
|
|
119
|
+
clearInterval(this.handle);
|
|
120
|
+
this.handle = null;
|
|
121
|
+
}
|
|
122
|
+
/** Threshold in ms — an unset/invalid setting falls back to the 15-minute default. */
|
|
123
|
+
async thresholdMs() {
|
|
124
|
+
let minutes;
|
|
125
|
+
try {
|
|
126
|
+
minutes = await this.deps.stallMinutes?.();
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
this.log(`[stall-watchdog] threshold read failed, using default: ${String(err)}`);
|
|
130
|
+
minutes = undefined;
|
|
131
|
+
}
|
|
132
|
+
const m = typeof minutes === 'number' && Number.isFinite(minutes) && minutes > 0
|
|
133
|
+
? minutes
|
|
134
|
+
: DEFAULT_WORKER_STALL_MINUTES;
|
|
135
|
+
return m * 60_000;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
//# sourceMappingURL=stall-watchdog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stall-watchdog.js","sourceRoot":"","sources":["../../src/api/stall-watchdog.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,EAAE;AACF,2FAA2F;AAC3F,gGAAgG;AAChG,gGAAgG;AAChG,yFAAyF;AACzF,EAAE;AACF,2FAA2F;AAC3F,2FAA2F;AAC3F,4FAA4F;AAC5F,yFAAyF;AACzF,wFAAwF;AACxF,kFAAkF;AAClF,8FAA8F;AAC9F,iEAAiE;AACjE,EAAE;AACF,6FAA6F;AAC7F,+FAA+F;AAC/F,8CAA8C;AAG9C,OAAO,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAoBhE,uFAAuF;AACvF,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAsBhD,MAAM,OAAO,mBAAmB;IAYD;IAX7B,2FAA2F;IAC1E,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzD,sDAAsD;IACrC,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrD,oFAAoF;IACnE,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5B,GAAG,CAAe;IAClB,GAAG,CAAsB;IAClC,MAAM,GAA0B,IAAI,CAAC;IACrC,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAA6B,IAAuB;QAAvB,SAAI,GAAJ,IAAI,CAAmB;QAClD,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QAChC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAS,EAAE,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAED,+FAA+F;IAC/F,MAAM,CAAC,KAAgB;QACrB,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5F,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ;YAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACxE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAED,8FAA8F;IAC9F,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,6CAA6C;QACxE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,IAAI,SAAyB,CAAC;YAC9B,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,wDAAwD,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChF,OAAO;YACT,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChD,qFAAqF;YACrF,sFAAsF;YACtF,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACzB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,iFAAiF;oBACjF,oFAAoF;oBACpF,mFAAmF;oBACnF,qFAAqF;oBACrF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;oBAClC,SAAS;gBACX,CAAC;gBACD,MAAM,UAAU,GAAG,GAAG,GAAG,IAAI,CAAC;gBAC9B,IAAI,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAAE,SAAS;gBACnE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBAClB,IAAI,EAAE,eAAe;oBACrB,OAAO,EAAE,GAAG,CAAC,EAAE;oBACf,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrC,UAAU;iBACX,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,CACN,wBAAwB,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,cAAc;oBACtF,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,uCAAuC;oBAC1E,iEAAiE,CACpE,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAED,qFAAqF;IACrF,KAAK,CAAC,UAAU,GAAG,yBAAyB;QAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI;YAAE,OAAO;QACjC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBAClC,IAAI,CAAC,GAAG,CAAC,kCAAkC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;IACxB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI;YAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,sFAAsF;IAC9E,KAAK,CAAC,WAAW;QACvB,IAAI,OAA2B,CAAC;QAChC,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,0DAA0D,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAClF,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;QACD,MAAM,CAAC,GACL,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YACpE,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,4BAA4B,CAAC;QACnC,OAAO,CAAC,GAAG,MAAM,CAAC;IACpB,CAAC;CACF"}
|