sensorium-mcp 3.1.0-refactor.18 → 3.1.0-refactor.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/agent-settings.d.ts +6 -0
- package/dist/config/agent-settings.d.ts.map +1 -1
- package/dist/config/agent-settings.js +22 -0
- package/dist/config/agent-settings.js.map +1 -1
- package/dist/config/detail-grant.d.ts +39 -0
- package/dist/config/detail-grant.d.ts.map +1 -0
- package/dist/config/detail-grant.js +61 -0
- package/dist/config/detail-grant.js.map +1 -0
- package/dist/config/index.d.ts +1 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +3 -3
- package/dist/config/index.js.map +1 -1
- package/dist/config/settings-store.d.ts.map +1 -1
- package/dist/config/settings-store.js +2 -3
- package/dist/config/settings-store.js.map +1 -1
- package/dist/dashboard/routes/copilot.d.ts.map +1 -1
- package/dist/dashboard/routes/copilot.js +17 -1
- package/dist/dashboard/routes/copilot.js.map +1 -1
- package/dist/dashboard/routes/threads-read.d.ts.map +1 -1
- package/dist/dashboard/routes/threads-read.js +15 -3
- package/dist/dashboard/routes/threads-read.js.map +1 -1
- package/dist/dashboard/routes/threads.d.ts.map +1 -1
- package/dist/dashboard/routes/threads.js +13 -4
- package/dist/dashboard/routes/threads.js.map +1 -1
- package/dist/dashboard/routes.d.ts.map +1 -1
- package/dist/dashboard/routes.js +11 -4
- package/dist/dashboard/routes.js.map +1 -1
- package/dist/dashboard/spa.html +11 -11
- package/dist/data/daily-marker.js +3 -3
- package/dist/data/daily-marker.js.map +1 -1
- package/dist/data/file-storage.d.ts.map +1 -1
- package/dist/data/file-storage.js +3 -3
- package/dist/data/file-storage.js.map +1 -1
- package/dist/data/memory/bootstrap.d.ts.map +1 -1
- package/dist/data/memory/bootstrap.js +2 -3
- package/dist/data/memory/bootstrap.js.map +1 -1
- package/dist/data/memory/migration-runner.d.ts.map +1 -1
- package/dist/data/memory/migration-runner.js +2 -3
- package/dist/data/memory/migration-runner.js.map +1 -1
- package/dist/data/memory/schema.js +2 -2
- package/dist/data/memory/schema.js.map +1 -1
- package/dist/index.js +54 -21
- package/dist/index.js.map +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js +21 -2
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/paths.d.ts +29 -0
- package/dist/lib/paths.d.ts.map +1 -0
- package/dist/lib/paths.js +36 -0
- package/dist/lib/paths.js.map +1 -0
- package/dist/server/factory.js +16 -6
- package/dist/server/factory.js.map +1 -1
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +22 -7
- package/dist/server/http-server.js.map +1 -1
- package/dist/services/agent-spawn.service.d.ts +3 -3
- package/dist/services/agent-spawn.service.d.ts.map +1 -1
- package/dist/services/agent-spawn.service.js +30 -9
- package/dist/services/agent-spawn.service.js.map +1 -1
- package/dist/services/background-runner.d.ts.map +1 -1
- package/dist/services/background-runner.js +22 -12
- package/dist/services/background-runner.js.map +1 -1
- package/dist/services/broker-client.d.ts +209 -0
- package/dist/services/broker-client.d.ts.map +1 -0
- package/dist/services/broker-client.js +359 -0
- package/dist/services/broker-client.js.map +1 -0
- package/dist/services/claude-spawn.service.d.ts +1 -1
- package/dist/services/claude-spawn.service.d.ts.map +1 -1
- package/dist/services/claude-spawn.service.js +25 -25
- package/dist/services/claude-spawn.service.js.map +1 -1
- package/dist/services/codex-spawn.service.d.ts +1 -1
- package/dist/services/codex-spawn.service.d.ts.map +1 -1
- package/dist/services/codex-spawn.service.js +19 -24
- package/dist/services/codex-spawn.service.js.map +1 -1
- package/dist/services/copilot-spawn.service.d.ts +1 -1
- package/dist/services/copilot-spawn.service.d.ts.map +1 -1
- package/dist/services/copilot-spawn.service.js +45 -36
- package/dist/services/copilot-spawn.service.js.map +1 -1
- package/dist/services/dispatcher/broker.d.ts +4 -3
- package/dist/services/dispatcher/broker.d.ts.map +1 -1
- package/dist/services/dispatcher/broker.js +23 -11
- package/dist/services/dispatcher/broker.js.map +1 -1
- package/dist/services/dispatcher/lock.js +2 -2
- package/dist/services/dispatcher/lock.js.map +1 -1
- package/dist/services/dispatcher/poller.d.ts.map +1 -1
- package/dist/services/dispatcher/poller.js +24 -5
- package/dist/services/dispatcher/poller.js.map +1 -1
- package/dist/services/flight-recorder.service.d.ts.map +1 -1
- package/dist/services/flight-recorder.service.js +43 -5
- package/dist/services/flight-recorder.service.js.map +1 -1
- package/dist/services/keeper.service.d.ts +9 -0
- package/dist/services/keeper.service.d.ts.map +1 -1
- package/dist/services/keeper.service.js +158 -34
- package/dist/services/keeper.service.js.map +1 -1
- package/dist/services/maintenance-signal.d.ts.map +1 -1
- package/dist/services/maintenance-signal.js +2 -3
- package/dist/services/maintenance-signal.js.map +1 -1
- package/dist/services/process.service.d.ts +76 -61
- package/dist/services/process.service.d.ts.map +1 -1
- package/dist/services/process.service.js +224 -608
- package/dist/services/process.service.js.map +1 -1
- package/dist/services/reconnect-snapshot.service.js +2 -2
- package/dist/services/reconnect-snapshot.service.js.map +1 -1
- package/dist/services/scheduler.d.ts +1 -0
- package/dist/services/scheduler.d.ts.map +1 -1
- package/dist/services/scheduler.js +69 -14
- package/dist/services/scheduler.js.map +1 -1
- package/dist/services/self-update.service.d.ts +14 -3
- package/dist/services/self-update.service.d.ts.map +1 -1
- package/dist/services/self-update.service.js +143 -74
- package/dist/services/self-update.service.js.map +1 -1
- package/dist/services/spawn-common.d.ts +49 -23
- package/dist/services/spawn-common.d.ts.map +1 -1
- package/dist/services/spawn-common.js +262 -47
- package/dist/services/spawn-common.js.map +1 -1
- package/dist/services/spawn-exit-watcher.d.ts +43 -0
- package/dist/services/spawn-exit-watcher.d.ts.map +1 -0
- package/dist/services/spawn-exit-watcher.js +127 -0
- package/dist/services/spawn-exit-watcher.js.map +1 -0
- package/dist/services/thread-health.service.d.ts +1 -1
- package/dist/services/thread-health.service.d.ts.map +1 -1
- package/dist/services/thread-health.service.js +11 -7
- package/dist/services/thread-health.service.js.map +1 -1
- package/dist/services/thread-lifecycle.service.js +2 -2
- package/dist/services/thread-lifecycle.service.js.map +1 -1
- package/dist/services/worker-cleanup.service.d.ts.map +1 -1
- package/dist/services/worker-cleanup.service.js +25 -99
- package/dist/services/worker-cleanup.service.js.map +1 -1
- package/dist/test-bootstrap.d.ts +39 -0
- package/dist/test-bootstrap.d.ts.map +1 -0
- package/dist/test-bootstrap.js +78 -0
- package/dist/test-bootstrap.js.map +1 -0
- package/dist/tools/defs/session-defs.js +1 -1
- package/dist/tools/defs/session-defs.js.map +1 -1
- package/dist/tools/defs/utility-defs.d.ts.map +1 -1
- package/dist/tools/defs/utility-defs.js +4 -3
- package/dist/tools/defs/utility-defs.js.map +1 -1
- package/dist/tools/delegate-tool.d.ts +1 -1
- package/dist/tools/delegate-tool.d.ts.map +1 -1
- package/dist/tools/delegate-tool.js +37 -6
- package/dist/tools/delegate-tool.js.map +1 -1
- package/dist/tools/session-tools.d.ts.map +1 -1
- package/dist/tools/session-tools.js +26 -9
- package/dist/tools/session-tools.js.map +1 -1
- package/dist/tools/utility-tools.js +1 -1
- package/dist/tools/utility-tools.js.map +1 -1
- package/dist/tools/wait/message-delivery.d.ts.map +1 -1
- package/dist/tools/wait/message-delivery.js +1 -2
- package/dist/tools/wait/message-delivery.js.map +1 -1
- package/package.json +2 -2
|
@@ -10,10 +10,14 @@ import { log } from "../../lib/logger.js";
|
|
|
10
10
|
import { errorMessage } from "../../lib/utils.js";
|
|
11
11
|
import { appendToThread, ensureDirs, injectOperatorMessage, OFFSET_FILE, readOffset, resolveThreadForTopic, writeOffset, writeReactionFile, } from "./broker.js";
|
|
12
12
|
// ── /mode + /prompts commands: handled specially (side-effect / keyboard) ──
|
|
13
|
-
import { getPredefinedPrompts, setThreadConversationMode, } from "../../config/index.js";
|
|
13
|
+
import { getPredefinedPrompts, setThreadConversationMode, grantDetailWindow, revokeDetailGrant, DETAIL_GRANT_MINUTES, } from "../../config/index.js";
|
|
14
14
|
import { registerTopic } from "../../sessions/topic-registry.js";
|
|
15
15
|
import { readLock, refreshLock, removeLock, tryAcquireLock, tryAcquireLockWithRetry, } from "./lock.js";
|
|
16
16
|
const MODE_PATTERN = /^\/mode\s+(concise|voice)\s*$/i;
|
|
17
|
+
// `/detail off` (also `/detailed off`) revokes the grant; `/detail` (also
|
|
18
|
+
// `/detailed`) opens one. Off is matched first so it never falls through to on.
|
|
19
|
+
const DETAIL_OFF_PATTERN = /^\/detail(?:ed)?\s+off\s*$/i;
|
|
20
|
+
const DETAIL_ON_PATTERN = /^\/detail(?:ed)?\s*$/i;
|
|
17
21
|
// Verbatim /prompts command (optionally addressed as /prompts@botname).
|
|
18
22
|
// Intentionally exact-match, never a "contains" test — a stray "/prompts"
|
|
19
23
|
// inside a longer sentence must reach the agent as normal text.
|
|
@@ -292,14 +296,29 @@ async function pollOnce(telegram, chatId) {
|
|
|
292
296
|
},
|
|
293
297
|
};
|
|
294
298
|
try {
|
|
295
|
-
//
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
+
// Operator commands: apply the side-effect, then replace the stored text
|
|
300
|
+
// with a confirmation so the agent sees the outcome, not the raw command.
|
|
301
|
+
if (stored.message.text && typeof threadId === "number") {
|
|
302
|
+
const trimmed = stored.message.text.trim();
|
|
303
|
+
const modeMatch = trimmed.match(MODE_PATTERN);
|
|
304
|
+
if (modeMatch) {
|
|
305
|
+
// /mode — persists conversation mode to settings.
|
|
299
306
|
const mode = modeMatch[1].toLowerCase();
|
|
300
307
|
setThreadConversationMode(threadId, mode);
|
|
301
308
|
stored.message.text = `Conversation mode switched to "${mode}".`;
|
|
302
309
|
}
|
|
310
|
+
else if (DETAIL_OFF_PATTERN.test(trimmed)) {
|
|
311
|
+
// /detail off — close the detail window immediately.
|
|
312
|
+
revokeDetailGrant(threadId);
|
|
313
|
+
stored.message.text = "Detail mode OFF.";
|
|
314
|
+
}
|
|
315
|
+
else if (DETAIL_ON_PATTERN.test(trimmed)) {
|
|
316
|
+
// /detail — grant one over-cap report (or DETAIL_GRANT_MINUTES).
|
|
317
|
+
grantDetailWindow(threadId);
|
|
318
|
+
stored.message.text =
|
|
319
|
+
`Detail mode ON — your next long report (or the next ${DETAIL_GRANT_MINUTES} min) ` +
|
|
320
|
+
"will pass the length cap in full.";
|
|
321
|
+
}
|
|
303
322
|
}
|
|
304
323
|
appendToThread(threadId, stored);
|
|
305
324
|
committedOffset = u.update_id + 1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"poller.js","sourceRoot":"","sources":["../../../src/services/dispatcher/poller.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,cAAc,EACd,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,8EAA8E;AAC9E,OAAO,EACL,oBAAoB,EACpB,yBAAyB,GAE1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAEjE,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,cAAc,EACd,uBAAuB,GACxB,MAAM,WAAW,CAAC;AACnB,MAAM,YAAY,GAAG,gCAAgC,CAAC;AACtD,wEAAwE;AACxE,0EAA0E;AAC1E,gEAAgE;AAChE,MAAM,eAAe,GAAG,+BAA+B,CAAC;AACxD,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E;;yBAEyB;AACzB,MAAM,MAAM,GAAG;IACb,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,SAAwC;CAChD,CAAC;AAEF,8EAA8E;AAC9E,6CAA6C;AAC7C,8EAA8E;AAE9E;;;;GAIG;AACH,KAAK,UAAU,mBAAmB,CAChC,QAAwB,EACxB,MAAc,EACd,UAA8B;IAE9B,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,CAAC,kBAAkB,CAC/B,MAAM,EACN,8EAA8E,EAC9E,EAAE,EACF,UAAU,CACX,CAAC;QACF,OAAO;IACT,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,sBAAsB,GAAG,CAAC,EAAE,EAAE;KAClE,CAAC,CAAC;IACH,MAAM,QAAQ,CAAC,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;AACtF,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,oBAAoB,CACjC,QAAwB,EACxB,MAAc,EACd,EAAyB;IAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC7C,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACxE,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACjD,MAAM,QAAQ,GACZ,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,IAAI,CAAC;QACH,uEAAuE;QACvE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,OAAO,EAAE,CAAC;gBACjB,GAAG,CAAC,KAAK,CACP,8CAA8C,QAAQ,KAAK,YAAY,CAAC,OAAO,CAAC,EAAE,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE;YAC3C,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;SACvB,CAAC,CAAC;QACH,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,gDAAgD,QAAQ,KAAK,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,KAAK,UAAU,QAAQ,CAAC,QAAwB,EAAE,MAAc;IAC9D,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;QACrC,oEAAoE;QACpE,mEAAmE;QACnE,GAAG,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC5E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,OAAO;IACT,CAAC;IAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,gEAAgE;IAChE,yDAAyD;IACzD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACrC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5B,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;gBAC/E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;gBACvB,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,oBAAoB,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE7F,qEAAqE;QACrE,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YAC9E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,OAAO;QACT,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,6EAA6E;QAC7E,IAAI,eAAe,GAAG,MAAM,CAAC;QAC7B,IAAI,YAAY,GAAG,IAAI,CAAC;QAExB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,6DAA6D;YAC7D,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE,CAAC;oBAC1D,IAAI,CAAC;wBACH,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC;oBACjE,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,GAAG,CAAC,KAAK,CAAC,gDAAgD,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjF,CAAC;gBACH,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,6DAA6D;YAC7D,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,EAAE,KAAK,CAAC;gBACrF,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,QAAQ,GAAmB;wBAC/B,KAAK;wBACL,SAAS,EAAE,CAAC,CAAC,gBAAgB,CAAC,UAAU;wBACxC,MAAM,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE;wBAClC,IAAI,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI;qBAC9B,CAAC;oBACF,QAAQ,CAAC,YAAY,GAAG;wBACtB,KAAK;wBACL,SAAS,EAAE,CAAC,CAAC,gBAAgB,CAAC,UAAU;wBACxC,IAAI,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI;qBAC9B,CAAC;oBACF,iBAAiB,CAAC,QAAQ,CAAC,CAAC;oBAC5B,GAAG,CAAC,IAAI,CAAC,yBAAyB,KAAK,eAAe,CAAC,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC;gBACzF,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;gBACf,uDAAuD;gBACvD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;gBACzC,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;YAEpB,iEAAiE;YACjE,IAAI,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,iBAAiB,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBACvE,GAAG,CAAC,IAAI,CACN,uCAAuC,CAAC,CAAC,mBAAmB,CAAC,IAAI,OAAO,CAAC,CAAC,iBAAiB,EAAE,CAC9F,CAAC;gBACJ,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;gBAC7E,IAAI,CAAC;oBACH,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBACtE,GAAG,CAAC,IAAI,CACN,iCAAiC,CAAC,CAAC,kBAAkB,CAAC,IAAI,aAAa,CAAC,CAAC,iBAAiB,GAAG,CAC9F,CAAC;gBACJ,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,KAAK,CAAC,qCAAqC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YAED,oEAAoE;YACpE,8DAA8D;YAC9D,2CAA2C;YAC3C,MAAM,UAAU,GACd,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC;YAC/C,MAAM,QAAQ,GACZ,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAErE,wEAAwE;YACxE,IAAI,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC;oBACH,MAAM,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1D,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,KAAK,CAAC,+CAA+C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChF,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAkB;gBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE;oBACP,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU;oBAChC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;oBAC1B,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;oBACpB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;oBAC1B,qEAAqE;oBACrE,wEAAwE;oBACxE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,SAAS;oBACjD,iBAAiB,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB;oBAC9C,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ;wBAC1B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO;4BACnC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS;4BACvC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS;yBACxC;wBACH,CAAC,CAAC,SAAS;oBACb,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK;wBACpB,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;4BAChC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ;4BAClC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS;yBACrC;wBACH,CAAC,CAAC,SAAS;oBACb,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU;wBAC9B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO;4BACrC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM;4BACnC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ;yBACxC;wBACH,CAAC,CAAC,SAAS;oBACb,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;wBACxB,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO;4BAClC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK;4BAC9B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ;yBACrC;wBACH,CAAC,CAAC,SAAS;oBACb,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS;wBAC5B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO;4BACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ;4BACtC,SAAS,EACP,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC;gCAC1D,CAAC,CAAC;oCACE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;yCACnE,OAAO;iCACX;gCACH,CAAC,CAAC,SAAS;yBAChB;wBACH,CAAC,CAAC,SAAS;oBACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;iBACrB;aACF,CAAC;YAEF,IAAI,CAAC;gBACH,gEAAgE;gBAChE,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;oBACxB,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBACjE,IAAI,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;wBAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAsB,CAAC;wBAC5D,yBAAyB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;wBAC1C,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,kCAAkC,IAAI,IAAI,CAAC;oBACnE,CAAC;gBACH,CAAC;gBACD,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACjC,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CACP,wCAAwC,CAAC,CAAC,SAAS,cAAc,QAAQ,KAAK,YAAY,CAAC,QAAQ,CAAC,EAAE,CACvG,CAAC;gBACF,YAAY,GAAG,KAAK,CAAC;gBACrB,MAAM,CAAC,yCAAyC;YAClD,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,eAAe,GAAG,MAAM,EAAE,CAAC;YAC7B,WAAW,CAAC,eAAe,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uCAAuC;QACvC,IAAI,GAAG,YAAY,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO;QACrE,GAAG,CAAC,KAAK,CAAC,4BAA4B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,aAAa,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAwB,EAAE,MAAc;IAC5E,UAAU,EAAE,CAAC;IACb,MAAM,QAAQ,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACjD,MAAM,iBAAiB,GAAG,MAAM,CAAC;IAEjC,iCAAiC;IACjC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,eAAe,GAAG,GAAG,EAAE;QAC3B,IAAI,iBAAiB;YAAE,OAAO;QAC9B,iBAAiB,GAAG,IAAI,CAAC;QACzB,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YACtB,UAAU,EAAE,CAAC;QACf,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,IAAI,kBAA8D,CAAC;IACnE,MAAM,oBAAoB,GAAG,GAAG,EAAE;QAChC,IAAI,kBAAkB;YAAE,OAAO;QAC/B,kBAAkB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,MAAM,cAAc,EAAE,EAAE,CAAC;gBAC3B,aAAa,CAAC,kBAAkB,CAAC,CAAC;gBAClC,kBAAkB,GAAG,SAAS,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;gBAC9E,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;gBACtB,SAAS,EAAE,CAAC;gBACZ,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACtB,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,GAAG,EAAE;QACrB,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;YACtB,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,MAAM,QAAQ,EAAE,CAAC;gBACrC,IAAI,WAAW,IAAI,WAAW,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnD,GAAG,CAAC,IAAI,CAAC,kCAAkC,WAAW,CAAC,GAAG,8BAA8B,CAAC,CAAC;oBAC1F,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;oBACvB,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,KAAK,CAAC,uCAAuC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACtE,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;gBAClE,CAAC;gBACD,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC;QACF,KAAK,IAAI,EAAE;aACR,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aACjF,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CACN,kFAAkF,CACnF,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,IAAI,QAAQ,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAEtD,qEAAqE;QACrE,mEAAmE;QACnE,gDAAgD;QAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACnE,IAAI,YAAuD,CAAC;YAC5D,IAAI,CAAC;gBACH,6DAA6D;gBAC7D,iEAAiE;gBACjE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBACnE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;oBACvD,WAAW,CAAC,MAAM,CAAC,CAAC;oBACpB,GAAG,CAAC,IAAI,CAAC,kCAAkC,MAAM,GAAG,CAAC,CAAC;gBACxD,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,oEAAoE;gBACpE,mEAAmE;gBACnE,0DAA0D;gBAC1D,GAAG,CAAC,IAAI,CACN,uCAAuC,YAAY,CAAC,GAAG,CAAC,uCAAuC,CAChG,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,IAAI,YAAY;oBAAE,YAAY,CAAC,YAAY,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QAED,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;QAE3F,qEAAqE;QACrE,kEAAkE;QAClE,oBAAoB,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
1
|
+
{"version":3,"file":"poller.js","sourceRoot":"","sources":["../../../src/services/dispatcher/poller.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,cAAc,EACd,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,8EAA8E;AAC9E,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,GAErB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAEjE,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,cAAc,EACd,uBAAuB,GACxB,MAAM,WAAW,CAAC;AACnB,MAAM,YAAY,GAAG,gCAAgC,CAAC;AACtD,0EAA0E;AAC1E,gFAAgF;AAChF,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AACzD,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAClD,wEAAwE;AACxE,0EAA0E;AAC1E,gEAAgE;AAChE,MAAM,eAAe,GAAG,+BAA+B,CAAC;AACxD,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAErC,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E;;yBAEyB;AACzB,MAAM,MAAM,GAAG;IACb,OAAO,EAAE,KAAK;IACd,KAAK,EAAE,SAAwC;CAChD,CAAC;AAEF,8EAA8E;AAC9E,6CAA6C;AAC7C,8EAA8E;AAE9E;;;;GAIG;AACH,KAAK,UAAU,mBAAmB,CAChC,QAAwB,EACxB,MAAc,EACd,UAA8B;IAE9B,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,CAAC,kBAAkB,CAC/B,MAAM,EACN,8EAA8E,EAC9E,EAAE,EACF,UAAU,CACX,CAAC;QACF,OAAO;IACT,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,sBAAsB,GAAG,CAAC,EAAE,EAAE;KAClE,CAAC,CAAC;IACH,MAAM,QAAQ,CAAC,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;AACtF,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,oBAAoB,CACjC,QAAwB,EACxB,MAAc,EACd,EAAyB;IAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC7C,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1C,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACxE,OAAO;IACT,CAAC;IAED,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACjD,MAAM,QAAQ,GACZ,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,IAAI,CAAC;QACH,uEAAuE;QACvE,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,OAAO,EAAE,CAAC;gBACjB,GAAG,CAAC,KAAK,CACP,8CAA8C,QAAQ,KAAK,YAAY,CAAC,OAAO,CAAC,EAAE,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE;YAC3C,OAAO,EAAE,UAAU;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;SACvB,CAAC,CAAC;QACH,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,gDAAgD,QAAQ,KAAK,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,MAAM,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,KAAK,UAAU,QAAQ,CAAC,QAAwB,EAAE,MAAc;IAC9D,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;QACrC,oEAAoE;QACpE,mEAAmE;QACnE,GAAG,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAC5E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,OAAO;IACT,CAAC;IAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAE5B,gEAAgE;IAChE,yDAAyD;IACzD,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACrC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YAC5B,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;gBAC/E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;gBACvB,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,wBAAwB,CAAC,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,oBAAoB,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE7F,qEAAqE;QACrE,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;YAC9E,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,OAAO;QACT,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,6EAA6E;QAC7E,IAAI,eAAe,GAAG,MAAM,CAAC;QAC7B,IAAI,YAAY,GAAG,IAAI,CAAC;QAExB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,6DAA6D;YAC7D,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE,CAAC;oBAC1D,IAAI,CAAC;wBACH,MAAM,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC;oBACjE,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,GAAG,CAAC,KAAK,CAAC,gDAAgD,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjF,CAAC;gBACH,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,6DAA6D;YAC7D,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,EAAE,KAAK,CAAC;gBACrF,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,QAAQ,GAAmB;wBAC/B,KAAK;wBACL,SAAS,EAAE,CAAC,CAAC,gBAAgB,CAAC,UAAU;wBACxC,MAAM,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE;wBAClC,IAAI,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI;qBAC9B,CAAC;oBACF,QAAQ,CAAC,YAAY,GAAG;wBACtB,KAAK;wBACL,SAAS,EAAE,CAAC,CAAC,gBAAgB,CAAC,UAAU;wBACxC,IAAI,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI;qBAC9B,CAAC;oBACF,iBAAiB,CAAC,QAAQ,CAAC,CAAC;oBAC5B,GAAG,CAAC,IAAI,CAAC,yBAAyB,KAAK,eAAe,CAAC,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC;gBACzF,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;gBACf,uDAAuD;gBACvD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;gBACzC,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;YAEpB,iEAAiE;YACjE,IAAI,CAAC,CAAC,mBAAmB,IAAI,CAAC,CAAC,iBAAiB,EAAE,CAAC;gBACjD,IAAI,CAAC;oBACH,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBACvE,GAAG,CAAC,IAAI,CACN,uCAAuC,CAAC,CAAC,mBAAmB,CAAC,IAAI,OAAO,CAAC,CAAC,iBAAiB,EAAE,CAC9F,CAAC;gBACJ,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YAED,4EAA4E;YAC5E,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;gBAC7E,IAAI,CAAC;oBACH,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;oBACtE,GAAG,CAAC,IAAI,CACN,iCAAiC,CAAC,CAAC,kBAAkB,CAAC,IAAI,aAAa,CAAC,CAAC,iBAAiB,GAAG,CAC9F,CAAC;gBACJ,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,GAAG,CAAC,KAAK,CAAC,qCAAqC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpE,CAAC;YACH,CAAC;YAED,oEAAoE;YACpE,8DAA8D;YAC9D,2CAA2C;YAC3C,MAAM,UAAU,GACd,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,OAAO;gBACT,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC;YAC/C,MAAM,QAAQ,GACZ,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAErE,wEAAwE;YACxE,IAAI,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC;oBACH,MAAM,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1D,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,KAAK,CAAC,+CAA+C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChF,CAAC;gBACD,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAkB;gBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE;oBACP,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU;oBAChC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;oBAC1B,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;oBACpB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;oBAC1B,qEAAqE;oBACrE,wEAAwE;oBACxE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,SAAS;oBACjD,iBAAiB,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB;oBAC9C,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ;wBAC1B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO;4BACnC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS;4BACvC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS;yBACxC;wBACH,CAAC,CAAC,SAAS;oBACb,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK;wBACpB,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;4BAChC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ;4BAClC,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS;yBACrC;wBACH,CAAC,CAAC,SAAS;oBACb,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU;wBAC9B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO;4BACrC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM;4BACnC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ;yBACxC;wBACH,CAAC,CAAC,SAAS;oBACb,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO;wBACxB,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO;4BAClC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK;4BAC9B,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ;yBACrC;wBACH,CAAC,CAAC,SAAS;oBACb,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS;wBAC5B,CAAC,CAAC;4BACE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO;4BACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ;4BACtC,SAAS,EACP,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC;gCAC1D,CAAC,CAAC;oCACE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;yCACnE,OAAO;iCACX;gCACH,CAAC,CAAC,SAAS;yBAChB;wBACH,CAAC,CAAC,SAAS;oBACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;iBACrB;aACF,CAAC;YAEF,IAAI,CAAC;gBACH,yEAAyE;gBACzE,0EAA0E;gBAC1E,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBACxD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;oBAC9C,IAAI,SAAS,EAAE,CAAC;wBACd,kDAAkD;wBAClD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,EAAsB,CAAC;wBAC5D,yBAAyB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;wBAC1C,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,kCAAkC,IAAI,IAAI,CAAC;oBACnE,CAAC;yBAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5C,qDAAqD;wBACrD,iBAAiB,CAAC,QAAQ,CAAC,CAAC;wBAC5B,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,kBAAkB,CAAC;oBAC3C,CAAC;yBAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC3C,iEAAiE;wBACjE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;wBAC5B,MAAM,CAAC,OAAO,CAAC,IAAI;4BACjB,uDAAuD,oBAAoB,QAAQ;gCACnF,mCAAmC,CAAC;oBACxC,CAAC;gBACH,CAAC;gBACD,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACjC,eAAe,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CACP,wCAAwC,CAAC,CAAC,SAAS,cAAc,QAAQ,KAAK,YAAY,CAAC,QAAQ,CAAC,EAAE,CACvG,CAAC;gBACF,YAAY,GAAG,KAAK,CAAC;gBACrB,MAAM,CAAC,yCAAyC;YAClD,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,eAAe,GAAG,MAAM,EAAE,CAAC;YAC7B,WAAW,CAAC,eAAe,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uCAAuC;QACvC,IAAI,GAAG,YAAY,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO;QACrE,GAAG,CAAC,KAAK,CAAC,4BAA4B,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;QACzB,aAAa,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAwB,EAAE,MAAc;IAC5E,UAAU,EAAE,CAAC;IACb,MAAM,QAAQ,GAAG,MAAM,uBAAuB,EAAE,CAAC;IACjD,MAAM,iBAAiB,GAAG,MAAM,CAAC;IAEjC,iCAAiC;IACjC,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,eAAe,GAAG,GAAG,EAAE;QAC3B,IAAI,iBAAiB;YAAE,OAAO;QAC9B,iBAAiB,GAAG,IAAI,CAAC;QACzB,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YACtB,UAAU,EAAE,CAAC;QACf,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,IAAI,kBAA8D,CAAC;IACnE,MAAM,oBAAoB,GAAG,GAAG,EAAE;QAChC,IAAI,kBAAkB;YAAE,OAAO;QAC/B,kBAAkB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,MAAM,cAAc,EAAE,EAAE,CAAC;gBAC3B,aAAa,CAAC,kBAAkB,CAAC,CAAC;gBAClC,kBAAkB,GAAG,SAAS,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;gBAC9E,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;gBACtB,SAAS,EAAE,CAAC;gBACZ,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACtB,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,GAAG,EAAE;QACrB,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;YACtB,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;gBACtB,MAAM,WAAW,GAAG,MAAM,QAAQ,EAAE,CAAC;gBACrC,IAAI,WAAW,IAAI,WAAW,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC;oBACnD,GAAG,CAAC,IAAI,CAAC,kCAAkC,WAAW,CAAC,GAAG,8BAA8B,CAAC,CAAC;oBAC1F,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;oBACvB,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACnC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,KAAK,CAAC,uCAAuC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACtE,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;gBAClE,CAAC;gBACD,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC;QACF,KAAK,IAAI,EAAE;aACR,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aACjF,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CACN,kFAAkF,CACnF,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,IAAI,QAAQ,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QAEtD,qEAAqE;QACrE,mEAAmE;QACnE,gDAAgD;QAChD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACnE,IAAI,YAAuD,CAAC;YAC5D,IAAI,CAAC;gBACH,6DAA6D;gBAC7D,iEAAiE;gBACjE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBACtE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBACnE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;oBACvD,WAAW,CAAC,MAAM,CAAC,CAAC;oBACpB,GAAG,CAAC,IAAI,CAAC,kCAAkC,MAAM,GAAG,CAAC,CAAC;gBACxD,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,oEAAoE;gBACpE,mEAAmE;gBACnE,0DAA0D;gBAC1D,GAAG,CAAC,IAAI,CACN,uCAAuC,YAAY,CAAC,GAAG,CAAC,uCAAuC,CAChG,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,IAAI,YAAY;oBAAE,YAAY,CAAC,YAAY,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QAED,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;QAE3F,qEAAqE;QACrE,kEAAkE;QAClE,oBAAoB,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flight-recorder.service.d.ts","sourceRoot":"","sources":["../../src/services/flight-recorder.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;
|
|
1
|
+
{"version":3,"file":"flight-recorder.service.d.ts","sourceRoot":"","sources":["../../src/services/flight-recorder.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,4FAA4F;AAC5F,eAAO,MAAM,mBAAmB,QAAoC,CAAC;AAkBrE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,+EAA+E;IAC/E,kBAAkB,IAAI,IAAI,CAAC;CAC5B;AAqED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,qBAA0B,GAAG,oBAAoB,CA6G7F"}
|
|
@@ -14,14 +14,25 @@
|
|
|
14
14
|
* lag with its own performance.now() delta, the worker measures beat
|
|
15
15
|
* staleness with its own. Both are monotonic within their thread.
|
|
16
16
|
*/
|
|
17
|
-
import { homedir } from "node:os";
|
|
18
|
-
import { join } from "node:path";
|
|
19
17
|
import { Worker } from "node:worker_threads";
|
|
20
18
|
import { log } from "../lib/logger.js";
|
|
19
|
+
import { dataPath } from "../lib/paths.js";
|
|
21
20
|
/** Where the supervisor looks for the heartbeat (keep in sync with SupervisorWorker.cs). */
|
|
22
|
-
export const HEARTBEAT_FILE_PATH =
|
|
21
|
+
export const HEARTBEAT_FILE_PATH = dataPath("server-heartbeat.json");
|
|
23
22
|
const DEFAULT_BEAT_INTERVAL_MS = 1_000;
|
|
24
23
|
const DEFAULT_WRITE_INTERVAL_MS = 5_000;
|
|
24
|
+
/**
|
|
25
|
+
* Stall persistence (2026-07-28 forensics): the heartbeat file is overwritten
|
|
26
|
+
* every write interval, so a lag SPIKE leaves no durable record — the 15:11
|
|
27
|
+
* freeze had to be reconstructed from telemetry gaps. Spikes above this
|
|
28
|
+
* threshold are appended to a dedicated spike log by the WORKER (immune to the
|
|
29
|
+
* stall, survives a mid-stall kill) and WARN-logged by the main thread once the
|
|
30
|
+
* loop unblocks (records the full stall duration).
|
|
31
|
+
*/
|
|
32
|
+
const LAG_SPIKE_THRESHOLD_MS = 1_000;
|
|
33
|
+
/** Min gap between spike-log appends so a long stall writes a line per write
|
|
34
|
+
* interval, not an unbounded stream. */
|
|
35
|
+
const SPIKE_APPEND_MIN_GAP_MS = 5_000;
|
|
25
36
|
/**
|
|
26
37
|
* How many times a dead heartbeat worker is respawned before giving up. The
|
|
27
38
|
* heartbeat is the supervisor's stalled-vs-dead evidence — losing it silently
|
|
@@ -33,10 +44,11 @@ const MAX_WORKER_RESPAWNS = 3;
|
|
|
33
44
|
const WORKER_SOURCE = `
|
|
34
45
|
const { parentPort, workerData } = require("node:worker_threads");
|
|
35
46
|
const fs = require("node:fs");
|
|
36
|
-
const { filePath, beatIntervalMs, writeIntervalMs } = workerData;
|
|
47
|
+
const { filePath, beatIntervalMs, writeIntervalMs, spikeFilePath, spikeThresholdMs, spikeMinGapMs } = workerData;
|
|
37
48
|
|
|
38
49
|
let lastBeatAt = performance.now();
|
|
39
50
|
let lastMainLagMs = 0;
|
|
51
|
+
let lastSpikeAppendAt = 0;
|
|
40
52
|
|
|
41
53
|
parentPort.on("message", (m) => {
|
|
42
54
|
// Test hook: lets the respawn path be exercised through a real worker crash.
|
|
@@ -50,6 +62,19 @@ setInterval(() => {
|
|
|
50
62
|
const msSinceLastBeat = performance.now() - lastBeatAt;
|
|
51
63
|
const overdueMs = Math.max(0, msSinceLastBeat - beatIntervalMs);
|
|
52
64
|
const eventLoopLagMs = Math.round(Math.max(lastMainLagMs, overdueMs));
|
|
65
|
+
// Durable stall evidence: the heartbeat file below is overwritten each tick,
|
|
66
|
+
// so append spikes to their own log — written from THIS thread, which keeps
|
|
67
|
+
// running (and keeps appending a rising number) while the main loop is
|
|
68
|
+
// frozen, and survives the process being killed mid-stall.
|
|
69
|
+
if (eventLoopLagMs >= spikeThresholdMs && performance.now() - lastSpikeAppendAt >= spikeMinGapMs) {
|
|
70
|
+
lastSpikeAppendAt = performance.now();
|
|
71
|
+
try {
|
|
72
|
+
fs.appendFileSync(
|
|
73
|
+
spikeFilePath,
|
|
74
|
+
JSON.stringify({ ts: new Date().toISOString(), pid: process.pid, eventLoopLagMs }) + "\\n",
|
|
75
|
+
);
|
|
76
|
+
} catch { /* best-effort */ }
|
|
77
|
+
}
|
|
53
78
|
const payload = JSON.stringify({
|
|
54
79
|
ts: Date.now(),
|
|
55
80
|
pid: process.pid,
|
|
@@ -98,6 +123,12 @@ export function startFlightRecorder(options = {}) {
|
|
|
98
123
|
// Timer drift = how late this 1s tick fired — the event-loop lag.
|
|
99
124
|
const lagMs = Math.max(0, now - lastFire - beatIntervalMs);
|
|
100
125
|
lastFire = now;
|
|
126
|
+
// This code runs AFTER the loop unblocked, so lagMs here is the FULL
|
|
127
|
+
// stall duration — record it where post-mortems will look (server.log),
|
|
128
|
+
// with what the loop was late for. The worker already streamed the
|
|
129
|
+
// rising lag to the spike file while the stall was in progress.
|
|
130
|
+
if (lagMs >= LAG_SPIKE_THRESHOLD_MS)
|
|
131
|
+
log.warn(`[flight-recorder] event-loop stall ended: the loop was blocked for ~${Math.round(lagMs)}ms`);
|
|
101
132
|
w.postMessage({ lagMs });
|
|
102
133
|
}, beatIntervalMs);
|
|
103
134
|
beatTimer.unref();
|
|
@@ -131,7 +162,14 @@ export function startFlightRecorder(options = {}) {
|
|
|
131
162
|
function spawnWorker() {
|
|
132
163
|
const w = new Worker(WORKER_SOURCE, {
|
|
133
164
|
eval: true,
|
|
134
|
-
workerData: {
|
|
165
|
+
workerData: {
|
|
166
|
+
filePath,
|
|
167
|
+
beatIntervalMs,
|
|
168
|
+
writeIntervalMs,
|
|
169
|
+
spikeFilePath: filePath.replace(/\.json$/, "") + "-spikes.log",
|
|
170
|
+
spikeThresholdMs: LAG_SPIKE_THRESHOLD_MS,
|
|
171
|
+
spikeMinGapMs: SPIKE_APPEND_MIN_GAP_MS,
|
|
172
|
+
},
|
|
135
173
|
});
|
|
136
174
|
worker = w;
|
|
137
175
|
w.unref();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flight-recorder.service.js","sourceRoot":"","sources":["../../src/services/flight-recorder.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"flight-recorder.service.js","sourceRoot":"","sources":["../../src/services/flight-recorder.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,4FAA4F;AAC5F,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAC,uBAAuB,CAAC,CAAC;AAErE,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,KAAK,CAAC;AACrC;yCACyC;AACzC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAgBtC;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,2EAA2E;AAC3E,uEAAuE;AACvE,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwDrB,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAiC,EAAE;IACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,mBAAmB,CAAC;IACzD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;IAE7E,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,SAAS,GAA0B,IAAI,CAAC;IAC5C,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,SAAS,QAAQ;QACf,IAAI,SAAS;YAAE,aAAa,CAAC,SAAS,CAAC,CAAC;QACxC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,SAAS,SAAS,CAAC,CAAS;QAC1B,IAAI,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACjC,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;YAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,kEAAkE;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,GAAG,cAAc,CAAC,CAAC;YAC3D,QAAQ,GAAG,GAAG,CAAC;YACf,qEAAqE;YACrE,wEAAwE;YACxE,mEAAmE;YACnE,gEAAgE;YAChE,IAAI,KAAK,IAAI,sBAAsB;gBACjC,GAAG,CAAC,IAAI,CACN,uEAAuE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAC7F,CAAC;YACJ,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3B,CAAC,EAAE,cAAc,CAAC,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED,4EAA4E;IAC5E,4EAA4E;IAC5E,uEAAuE;IACvE,qEAAqE;IACrE,SAAS,YAAY,CAAC,KAAa;QACjC,QAAQ,EAAE,CAAC;QACX,MAAM,GAAG,IAAI,CAAC;QACd,IAAI,OAAO;YAAE,OAAO;QACpB,IAAI,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACpC,GAAG,CAAC,KAAK,CACP,4CAA4C,KAAK,WAAW,QAAQ,cAAc;gBAChF,qEAAqE,CACxE,CAAC;YACF,OAAO;QACT,CAAC;QACD,QAAQ,EAAE,CAAC;QACX,GAAG,CAAC,KAAK,CACP,4CAA4C,KAAK,mBAAmB,QAAQ,IAAI,mBAAmB,GAAG,CACvG,CAAC;QACF,IAAI,CAAC;YACH,WAAW,EAAE,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,oEAAoE;YACpE,uEAAuE;YACvE,MAAM,GAAG,IAAI,CAAC;YACd,GAAG,CAAC,KAAK,CAAC,+DAA+D,GAAG,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,SAAS,WAAW;QAClB,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE;YAClC,IAAI,EAAE,IAAI;YACV,UAAU,EAAE;gBACV,QAAQ;gBACR,cAAc;gBACd,eAAe;gBACf,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,aAAa;gBAC9D,gBAAgB,EAAE,sBAAsB;gBACxC,aAAa,EAAE,uBAAuB;aACvC;SACF,CAAC,CAAC;QACH,MAAM,GAAG,CAAC,CAAC;QACX,CAAC,CAAC,KAAK,EAAE,CAAC;QACV,uEAAuE;QACvE,0DAA0D;QAC1D,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACpB,IAAI,MAAM,KAAK,CAAC;gBAAE,YAAY,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACpB,IAAI,MAAM,KAAK,CAAC;gBAAE,YAAY,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,SAAS,CAAC,CAAC,CAAC,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,WAAW,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,iCAAiC,QAAQ,UAAU,eAAe,IAAI,CAAC,CAAC;IACnF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,IAAI,CAAC;QACd,QAAQ,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,qEAAqE,GAAG,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,KAAK,CAAC,IAAI;YACR,OAAO,GAAG,IAAI,CAAC;YACf,QAAQ,EAAE,CAAC;YACX,IAAI,MAAM;gBAAE,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACrD,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,kBAAkB;YAChB,MAAM,EAAE,WAAW,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -15,6 +15,15 @@ export declare class KeeperService {
|
|
|
15
15
|
start(): void;
|
|
16
16
|
stop(): void;
|
|
17
17
|
private syncKeepers;
|
|
18
|
+
/** One keeper cycle with persistence: whenever a cycle changed any durable
|
|
19
|
+
* policy counter, push the new state to the broker's registry so it survives
|
|
20
|
+
* an MCP restart (plan §4.3). Single choke point — the cycle body has many
|
|
21
|
+
* early returns and each mutation site would otherwise need its own save. */
|
|
22
|
+
private runKeeperCycle;
|
|
23
|
+
private static durableStateOf;
|
|
24
|
+
/** Restore persisted counters when a keeper starts (max-merge so counters that
|
|
25
|
+
* already advanced in this session are never rolled back). */
|
|
26
|
+
private hydrateFromRegistry;
|
|
18
27
|
private checkAndRestart;
|
|
19
28
|
private notifyDeath;
|
|
20
29
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"keeper.service.d.ts","sourceRoot":"","sources":["../../src/services/keeper.service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"keeper.service.d.ts","sourceRoot":"","sources":["../../src/services/keeper.service.ts"],"names":[],"mappings":"AAeA,OAAO,EAAe,KAAK,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAGzF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAmK5D,qBAAa,aAAa;IAKtB,OAAO,CAAC,IAAI;IAJd,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAA+C;gBAGtD,IAAI,EAAE;QACZ,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;QACnD,eAAe,EAAE,sBAAsB,CAAC;QACxC,QAAQ,EAAE;YACR,WAAW,CACT,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;SAClB,CAAC;QACF,MAAM,EAAE,MAAM,CAAC;KAChB;IAGH,KAAK,IAAI,IAAI;IAMb,IAAI,IAAI,IAAI;IAWZ,OAAO,CAAC,WAAW;IAwEnB;;;kFAG8E;YAChE,cAAc;IAc5B,OAAO,CAAC,MAAM,CAAC,cAAc;IAY7B;mEAC+D;YACjD,mBAAmB;YAkCnB,eAAe;YA2Uf,WAAW;CAkB1B"}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { log } from "../lib/logger.js";
|
|
4
|
-
import { findAliveThreadViaPidFile, isProcessAlive,
|
|
4
|
+
import { findAliveThreadViaPidFile, isProcessAlive, isThreadAlive, killThreadTree, PROCESS_LOGS_DIR, THREAD_LOGS_DIR, } from "./process.service.js";
|
|
5
5
|
import { getThreadSessionCount } from "../sessions/mcp-registry.js";
|
|
6
6
|
import { readThreadHeartbeat } from "../data/file-storage.js";
|
|
7
7
|
import { dispatchSpawn } from "./agent-spawn.service.js";
|
|
8
8
|
import { getKeepAliveThreads, getThread } from "../data/memory/thread-registry.js";
|
|
9
9
|
import { ThreadState } from "./thread-lifecycle.service.js";
|
|
10
10
|
import { errorMessage } from "../lib/utils.js";
|
|
11
|
+
import { brokerGetKeeperState, brokerSaveKeeperState } from "./broker-client.js";
|
|
11
12
|
const KEEPER_CHECK_INTERVAL_MS = 120_000; // 2 min
|
|
12
13
|
const KEEPER_MAX_RETRIES = 5;
|
|
13
14
|
const KEEPER_COOLDOWN_MS = 300_000; // 5 min
|
|
@@ -65,7 +66,7 @@ function findLatestThreadLogFile(threadId) {
|
|
|
65
66
|
* NEVER affect the kill. This runs only when the keeper has ALREADY decided the
|
|
66
67
|
* thread is stuck; it triggers no new kills.
|
|
67
68
|
*/
|
|
68
|
-
function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
|
|
69
|
+
async function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
|
|
69
70
|
try {
|
|
70
71
|
const dir = join(PROCESS_LOGS_DIR, "wedge-snapshots");
|
|
71
72
|
mkdirSync(dir, { recursive: true });
|
|
@@ -76,7 +77,7 @@ function writeWedgeSnapshot(threadId, pid, reason, heartbeat, now) {
|
|
|
76
77
|
lines.push(`timestamp: ${new Date(now).toISOString()}`);
|
|
77
78
|
let alive;
|
|
78
79
|
try {
|
|
79
|
-
alive = isProcessAlive(pid);
|
|
80
|
+
alive = await isProcessAlive(pid);
|
|
80
81
|
}
|
|
81
82
|
catch (err) {
|
|
82
83
|
alive = `error: ${errorMessage(err)}`;
|
|
@@ -167,7 +168,7 @@ export class KeeperService {
|
|
|
167
168
|
// Add new keepers
|
|
168
169
|
for (const thread of threads) {
|
|
169
170
|
if (!this.keepers.has(thread.threadId)) {
|
|
170
|
-
|
|
171
|
+
const entry = {
|
|
171
172
|
threadId: thread.threadId,
|
|
172
173
|
retryCount: 0,
|
|
173
174
|
fastExitCount: 0,
|
|
@@ -180,15 +181,22 @@ export class KeeperService {
|
|
|
180
181
|
survivedKillCount: 0,
|
|
181
182
|
lastUnkillableNotifyMs: 0,
|
|
182
183
|
lastLogSizeBytes: 0,
|
|
183
|
-
|
|
184
|
+
lastHeartbeatMs: 0,
|
|
185
|
+
};
|
|
186
|
+
this.keepers.set(thread.threadId, entry);
|
|
184
187
|
log.info(`[keeper] Started keeper for thread ${thread.threadId} ('${thread.name}')`);
|
|
188
|
+
// Hydrate backoff counters from the broker's registry (plan §4.3): they
|
|
189
|
+
// used to reset to zero on every MCP restart — during the Jul 25 wave 15
|
|
190
|
+
// resets in 2h19m meant no backoff ever matured. Fire-and-forget; until
|
|
191
|
+
// it lands the entry runs on defaults (today's behavior).
|
|
192
|
+
void this.hydrateFromRegistry(entry);
|
|
185
193
|
}
|
|
186
194
|
}
|
|
187
195
|
// Check all active keepers (skip if already in-flight to prevent duplicate spawns)
|
|
188
196
|
for (const entry of this.keepers.values()) {
|
|
189
197
|
if (!entry.stopped && !entry.checking) {
|
|
190
198
|
entry.checking = true;
|
|
191
|
-
this.
|
|
199
|
+
this.runKeeperCycle(entry)
|
|
192
200
|
.catch((err) => {
|
|
193
201
|
log.warn(`[keeper] checkAndRestart error for thread ${entry.threadId}: ${errorMessage(err)}`);
|
|
194
202
|
})
|
|
@@ -198,6 +206,64 @@ export class KeeperService {
|
|
|
198
206
|
}
|
|
199
207
|
}
|
|
200
208
|
}
|
|
209
|
+
/** One keeper cycle with persistence: whenever a cycle changed any durable
|
|
210
|
+
* policy counter, push the new state to the broker's registry so it survives
|
|
211
|
+
* an MCP restart (plan §4.3). Single choke point — the cycle body has many
|
|
212
|
+
* early returns and each mutation site would otherwise need its own save. */
|
|
213
|
+
async runKeeperCycle(entry) {
|
|
214
|
+
const before = KeeperService.durableStateOf(entry);
|
|
215
|
+
try {
|
|
216
|
+
await this.checkAndRestart(entry);
|
|
217
|
+
}
|
|
218
|
+
finally {
|
|
219
|
+
const after = KeeperService.durableStateOf(entry);
|
|
220
|
+
if (JSON.stringify(before) !== JSON.stringify(after)) {
|
|
221
|
+
void brokerSaveKeeperState(after).catch(() => {
|
|
222
|
+
/* unreachable broker already WARN-logged by the client */
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
static durableStateOf(entry) {
|
|
228
|
+
return {
|
|
229
|
+
threadId: entry.threadId,
|
|
230
|
+
retryCount: entry.retryCount,
|
|
231
|
+
fastExitCount: entry.fastExitCount,
|
|
232
|
+
fastExitEscalation: entry.fastExitEscalation,
|
|
233
|
+
cooldownUntilMs: entry.cooldownUntil > 0 ? entry.cooldownUntil : null,
|
|
234
|
+
survivedKillCount: entry.survivedKillCount,
|
|
235
|
+
lastStartAtMs: entry.lastStartTime > 0 ? entry.lastStartTime : null,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
/** Restore persisted counters when a keeper starts (max-merge so counters that
|
|
239
|
+
* already advanced in this session are never rolled back). */
|
|
240
|
+
async hydrateFromRegistry(entry) {
|
|
241
|
+
try {
|
|
242
|
+
const state = await brokerGetKeeperState(entry.threadId);
|
|
243
|
+
if (!state || entry.stopped)
|
|
244
|
+
return;
|
|
245
|
+
entry.retryCount = Math.max(entry.retryCount, state.retryCount);
|
|
246
|
+
entry.fastExitCount = Math.max(entry.fastExitCount, state.fastExitCount);
|
|
247
|
+
entry.fastExitEscalation = Math.max(entry.fastExitEscalation, state.fastExitEscalation);
|
|
248
|
+
entry.survivedKillCount = Math.max(entry.survivedKillCount, state.survivedKillCount);
|
|
249
|
+
if (state.cooldownUntilMs !== null && state.cooldownUntilMs > entry.cooldownUntil)
|
|
250
|
+
entry.cooldownUntil = state.cooldownUntilMs;
|
|
251
|
+
if (state.lastStartAtMs !== null && entry.lastStartTime === 0)
|
|
252
|
+
entry.lastStartTime = state.lastStartAtMs;
|
|
253
|
+
if (state.retryCount > 0 ||
|
|
254
|
+
state.fastExitCount > 0 ||
|
|
255
|
+
state.fastExitEscalation > 0 ||
|
|
256
|
+
state.survivedKillCount > 0 ||
|
|
257
|
+
(state.cooldownUntilMs !== null && state.cooldownUntilMs > Date.now())) {
|
|
258
|
+
log.info(`[keeper] Thread ${entry.threadId}: restored persisted keeper state (retry=${state.retryCount}, fastExit=${state.fastExitCount}, escalation=${state.fastExitEscalation}, survivedKill=${state.survivedKillCount}${state.cooldownUntilMs !== null && state.cooldownUntilMs > Date.now()
|
|
259
|
+
? `, cooldown until ${new Date(state.cooldownUntilMs).toISOString()}`
|
|
260
|
+
: ""})`);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
catch (err) {
|
|
264
|
+
log.debug(`[keeper] keeper-state hydration failed for thread ${entry.threadId}: ${errorMessage(err)}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
201
267
|
async checkAndRestart(entry) {
|
|
202
268
|
const now = Date.now();
|
|
203
269
|
// 1. Check cooldown
|
|
@@ -206,29 +272,71 @@ export class KeeperService {
|
|
|
206
272
|
return;
|
|
207
273
|
}
|
|
208
274
|
// 2. Check if thread is running — reads PID file directly (authoritative OS evidence)
|
|
209
|
-
const
|
|
210
|
-
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
//
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
//
|
|
275
|
+
const aliveProc = await findAliveThreadViaPidFile(entry.threadId);
|
|
276
|
+
const alivePid = aliveProc?.pid;
|
|
277
|
+
// 2a. PID file shows no live tracked process. A fresh heartbeat USED to be
|
|
278
|
+
// trusted alone as "someone is actively serving this thread" — but with the
|
|
279
|
+
// tracked PID gone, a heartbeat < 1h old is NOT proof of life: the heartbeat
|
|
280
|
+
// file is only rewritten in the wait poll loop and on tool calls, so a long
|
|
281
|
+
// final model turn freezes it, and if the process then dies (e.g. Claude's
|
|
282
|
+
// 5-hour org limit 429s it) the frozen-but-recent heartbeat lingers for up to
|
|
283
|
+
// STUCK_THRESHOLD_MS. That is exactly how thread 1327's death was silently
|
|
284
|
+
// swallowed for ~67 minutes on 2026-07-27: the limbo guard fired every ~2-min
|
|
285
|
+
// cycle while BOTH the restart AND the operator-facing 💀 notification were
|
|
286
|
+
// suppressed, until the frozen heartbeat finally crossed 1h.
|
|
287
|
+
//
|
|
288
|
+
// The ONLY signals that prove a worker is alive RIGHT NOW are CROSS-CYCLE
|
|
289
|
+
// deltas measured between keeper cycles (~2 min apart) — symmetric to the
|
|
290
|
+
// 2026-07-26 stuck-branch fix, which likewise learned that a stale heartbeat
|
|
291
|
+
// alone does not prove/disprove a wedge because a long model turn freezes it:
|
|
292
|
+
// • heartbeat ADVANCED since the previous cycle — a live untracked worker
|
|
293
|
+
// idle in the long-poll rewrites it every ~2s, so it keeps advancing, OR
|
|
294
|
+
// • stdout GREW in bytes since the previous cycle (sizeGrew) — a live worker
|
|
295
|
+
// mid model turn emits tokens even while its heartbeat is frozen.
|
|
296
|
+
// A genuinely dead worker shows BOTH frozen: no heartbeat advance AND no
|
|
297
|
+
// stdout growth → fall through to the normal restart + notifyDeath path so the
|
|
298
|
+
// operator is warned within ~2 keeper cycles instead of ~1 hour. (mtime < 1h
|
|
299
|
+
// is deliberately NOT used here: a just-died process leaves a log whose mtime
|
|
300
|
+
// is recent for up to an hour — the same failure mode as the frozen heartbeat.)
|
|
219
301
|
if (alivePid === undefined) {
|
|
220
302
|
const heartbeat = readThreadHeartbeat(entry.threadId);
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
303
|
+
// Upper sanity bound: no heartbeat, a pre-restart heartbeat from an older
|
|
304
|
+
// server generation, or a heartbeat already older than 1h is definitely not
|
|
305
|
+
// a live untracked worker → fall through to the cleanup + restart path.
|
|
306
|
+
const heartbeatPlausiblyLive = heartbeat !== null && heartbeat > SERVER_START_TIME && now - heartbeat < STUCK_THRESHOLD_MS;
|
|
307
|
+
if (heartbeatPlausiblyLive) {
|
|
308
|
+
const streamLog = findLatestThreadLog(entry.threadId);
|
|
309
|
+
const heartbeatAdvanced = entry.lastHeartbeatMs > 0 && heartbeat > entry.lastHeartbeatMs;
|
|
310
|
+
const sizeGrew = entry.lastLogSizeBytes > 0 &&
|
|
311
|
+
streamLog !== null &&
|
|
312
|
+
streamLog.sizeBytes > entry.lastLogSizeBytes;
|
|
313
|
+
// First cycle of this episode: no baseline yet, so no cross-cycle delta can
|
|
314
|
+
// be measured. Record the baselines and grant exactly ONE ~2-min grace
|
|
315
|
+
// cycle (vs. the old up-to-1-hour hole) before demanding evidence of life.
|
|
316
|
+
const noBaseline = entry.lastHeartbeatMs === 0 && entry.lastLogSizeBytes === 0;
|
|
317
|
+
if (noBaseline || heartbeatAdvanced || sizeGrew) {
|
|
318
|
+
// Still (or possibly) a live untracked worker — refresh baselines so the
|
|
319
|
+
// next cycle measures a fresh delta, then skip the restart.
|
|
320
|
+
entry.lastHeartbeatMs = heartbeat;
|
|
321
|
+
if (streamLog !== null)
|
|
322
|
+
entry.lastLogSizeBytes = streamLog.sizeBytes;
|
|
323
|
+
const evidence = noBaseline
|
|
324
|
+
? "first cycle — baselining"
|
|
325
|
+
: heartbeatAdvanced
|
|
326
|
+
? "heartbeat advanced"
|
|
327
|
+
: "stdout grew";
|
|
328
|
+
if (!entry.limboWarned) {
|
|
329
|
+
log.warn(`[keeper] Thread ${entry.threadId}: no live tracked PID but a worker still shows life (${evidence}, heartbeat ${Math.round((now - heartbeat) / 1000)}s ago) — a live worker is still serving it under an untracked PID. Skipping restart to avoid a duplicate process.`);
|
|
330
|
+
entry.limboWarned = true;
|
|
331
|
+
}
|
|
332
|
+
else {
|
|
333
|
+
log.debug(`[keeper] Thread ${entry.threadId}: still in untracked-live-worker limbo (${evidence}, heartbeat ${Math.round((now - heartbeat) / 1000)}s ago) — skipping restart.`);
|
|
334
|
+
}
|
|
335
|
+
return;
|
|
230
336
|
}
|
|
231
|
-
|
|
337
|
+
// Heartbeat recent but FROZEN across cycles AND stdout static → the worker
|
|
338
|
+
// is dead (the 2026-07-27 thread-1327 case). Do NOT skip — fall through to
|
|
339
|
+
// the cleanup + normal restart + notifyDeath path below.
|
|
232
340
|
}
|
|
233
341
|
// Process is genuinely gone — any prior survived-kill episode is over.
|
|
234
342
|
entry.survivedKillCount = 0;
|
|
@@ -236,6 +344,8 @@ export class KeeperService {
|
|
|
236
344
|
// A restart writes a fresh (empty) stdout log; drop the byte baseline so the
|
|
237
345
|
// next episode never compares sizes across two different log files.
|
|
238
346
|
entry.lastLogSizeBytes = 0;
|
|
347
|
+
// Drop the heartbeat liveness baseline too — the next episode measures fresh.
|
|
348
|
+
entry.lastHeartbeatMs = 0;
|
|
239
349
|
}
|
|
240
350
|
// Not in limbo (PID alive, or heartbeat stale/absent) — clear the flag so a
|
|
241
351
|
// future episode logs afresh.
|
|
@@ -271,12 +381,16 @@ export class KeeperService {
|
|
|
271
381
|
const streamLog = heartbeatStale ? findLatestThreadLog(entry.threadId) : null;
|
|
272
382
|
const streamAgeMs = streamLog !== null ? now - streamLog.mtimeMs : null;
|
|
273
383
|
const mtimeFresh = streamAgeMs !== null && streamAgeMs >= 0 && streamAgeMs < STUCK_THRESHOLD_MS;
|
|
274
|
-
const sizeGrew = streamLog !== null &&
|
|
384
|
+
const sizeGrew = streamLog !== null &&
|
|
385
|
+
entry.lastLogSizeBytes > 0 &&
|
|
386
|
+
streamLog.sizeBytes > entry.lastLogSizeBytes;
|
|
275
387
|
if (streamLog !== null)
|
|
276
388
|
entry.lastLogSizeBytes = streamLog.sizeBytes;
|
|
277
389
|
const streamFresh = mtimeFresh || sizeGrew;
|
|
278
390
|
if (heartbeatStale && streamFresh) {
|
|
279
|
-
log.info(`[keeper] Thread ${entry.threadId} heartbeat is stale (${heartbeat !== null ? Math.round((now - heartbeat) / 60000) + "m" : "none"}) but its stdout stream shows life (${mtimeFresh
|
|
391
|
+
log.info(`[keeper] Thread ${entry.threadId} heartbeat is stale (${heartbeat !== null ? Math.round((now - heartbeat) / 60000) + "m" : "none"}) but its stdout stream shows life (${mtimeFresh
|
|
392
|
+
? `mtime ${Math.round(streamAgeMs / 1000)}s ago`
|
|
393
|
+
: "grew in bytes"}) — agent is mid-turn (streaming), not wedged. Skipping kill.`);
|
|
280
394
|
}
|
|
281
395
|
const isStuck = heartbeatStale && !streamFresh;
|
|
282
396
|
if (isStuck) {
|
|
@@ -298,11 +412,21 @@ export class KeeperService {
|
|
|
298
412
|
// DIAGNOSTIC (logging-only): forensic snapshot BEFORE the kill so we can
|
|
299
413
|
// see whether the stuck agent was still MCP-connected or already orphaned,
|
|
300
414
|
// plus the tail of its stdout. Best-effort; never blocks/affects the kill.
|
|
301
|
-
writeWedgeSnapshot(entry.threadId, alivePid, reason, heartbeat, now);
|
|
302
|
-
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
|
|
415
|
+
await writeWedgeSnapshot(entry.threadId, alivePid, reason, heartbeat, now);
|
|
416
|
+
// THREAD-scoped kill: the recorded alivePid is a launcher that has long
|
|
417
|
+
// since exited while its forked child lives on in the thread's job, so a
|
|
418
|
+
// killProcessTree(alivePid) routes to the broker's KillByPid AlreadyDead
|
|
419
|
+
// fast-path and never touches the live child — then isProcessAlive(launcherPid)
|
|
420
|
+
// reads false and the keeper falsely concludes success → duplicate spawn +
|
|
421
|
+
// orphaned wedged child. Route by threadId (broker KillByThread → job kill of
|
|
422
|
+
// the real child); pass the launcher pid + PID-file startedAt as the
|
|
423
|
+
// identity-checked degraded fallback.
|
|
424
|
+
await killThreadTree(entry.threadId, "keeper-stuck", alivePid, aliveProc?.startedAt);
|
|
425
|
+
// Verify kill succeeded before restarting — if the thread still has a live
|
|
426
|
+
// member, do NOT spawn a second instance. Keeper will retry next cycle.
|
|
427
|
+
// THREAD-scoped so a broker-unreachable/unknown verdict treats the thread as
|
|
428
|
+
// alive (never a false "killed" → never a duplicate spawn).
|
|
429
|
+
if (await isThreadAlive(entry.threadId, alivePid, aliveProc?.startedAt)) {
|
|
306
430
|
entry.survivedKillCount++;
|
|
307
431
|
log.warn(`[keeper] Thread ${entry.threadId} PID=${alivePid} survived kill (${entry.survivedKillCount}x) — skipping restart (will retry next cycle)`);
|
|
308
432
|
// Escalate: a process that repeatedly refuses to die needs operator
|
|
@@ -390,7 +514,7 @@ export class KeeperService {
|
|
|
390
514
|
? `last keeper-start ${Math.round((now - entry.lastStartTime) / 1000)}s ago`
|
|
391
515
|
: "never started by this server (inherited PID file)";
|
|
392
516
|
log.info(`[keeper] Thread ${entry.threadId} not running — restarting (attempt ${entry.retryCount + 1}/${KEEPER_MAX_RETRIES}, ${lastStartLabel})`);
|
|
393
|
-
const result = dispatchSpawn((thread.client || "claude"), thread.name, entry.threadId, this.deps.threadLifecycle, thread.workingDirectory ?? undefined);
|
|
517
|
+
const result = await dispatchSpawn((thread.client || "claude"), thread.name, entry.threadId, this.deps.threadLifecycle, thread.workingDirectory ?? undefined);
|
|
394
518
|
if ("pid" in result) {
|
|
395
519
|
entry.lastStartTime = Date.now();
|
|
396
520
|
entry.retryCount = 0;
|