sensorium-mcp 3.1.0-refactor.1 → 3.1.0-refactor.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Install-Sensorium.ps1 +113 -20
- package/dist/config/agent-settings.d.ts +7 -0
- package/dist/config/agent-settings.d.ts.map +1 -1
- package/dist/config/agent-settings.js +14 -0
- package/dist/config/agent-settings.js.map +1 -1
- package/dist/dashboard/routes/claude.d.ts.map +1 -1
- package/dist/dashboard/routes/claude.js +9 -1
- package/dist/dashboard/routes/claude.js.map +1 -1
- package/dist/dashboard/routes/threads.js +1 -1
- package/dist/dashboard/routes/threads.js.map +1 -1
- package/dist/dashboard/spa.html +11 -11
- package/dist/data/memory/narrative.d.ts.map +1 -1
- package/dist/data/memory/narrative.js +5 -1
- package/dist/data/memory/narrative.js.map +1 -1
- package/dist/data/memory/reflection-dedup.js +2 -2
- package/dist/data/memory/reflection-dedup.js.map +1 -1
- package/dist/data/memory/reflection.js +5 -1
- package/dist/data/memory/reflection.js.map +1 -1
- package/dist/data/memory/topic-shift.js +2 -2
- package/dist/data/memory/topic-shift.js.map +1 -1
- package/dist/index.js +19 -9
- package/dist/index.js.map +1 -1
- package/dist/integrations/openai/chat.d.ts +7 -1
- package/dist/integrations/openai/chat.d.ts.map +1 -1
- package/dist/integrations/openai/chat.js +19 -2
- package/dist/integrations/openai/chat.js.map +1 -1
- package/dist/integrations/openai/speech.d.ts +4 -1
- package/dist/integrations/openai/speech.d.ts.map +1 -1
- package/dist/integrations/openai/speech.js +6 -1
- package/dist/integrations/openai/speech.js.map +1 -1
- package/dist/lib/markdown.d.ts +0 -13
- package/dist/lib/markdown.d.ts.map +1 -1
- package/dist/lib/markdown.js +45 -0
- package/dist/lib/markdown.js.map +1 -1
- package/dist/lib/usage-metering/index.d.ts +13 -0
- package/dist/lib/usage-metering/index.d.ts.map +1 -0
- package/dist/lib/usage-metering/index.js +10 -0
- package/dist/lib/usage-metering/index.js.map +1 -0
- package/dist/lib/usage-metering/meter.d.ts +34 -0
- package/dist/lib/usage-metering/meter.d.ts.map +1 -0
- package/dist/lib/usage-metering/meter.js +28 -0
- package/dist/lib/usage-metering/meter.js.map +1 -0
- package/dist/lib/usage-metering/pricing.d.ts +39 -0
- package/dist/lib/usage-metering/pricing.d.ts.map +1 -0
- package/dist/lib/usage-metering/pricing.js +49 -0
- package/dist/lib/usage-metering/pricing.js.map +1 -0
- package/dist/lib/usage-metering/sink.d.ts +24 -0
- package/dist/lib/usage-metering/sink.d.ts.map +1 -0
- package/dist/lib/usage-metering/sink.js +56 -0
- package/dist/lib/usage-metering/sink.js.map +1 -0
- package/dist/server/factory.js +12 -9
- package/dist/server/factory.js.map +1 -1
- package/dist/server/http-server.d.ts +1 -0
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +45 -3
- package/dist/server/http-server.js.map +1 -1
- package/dist/services/claude-spawn.service.d.ts.map +1 -1
- package/dist/services/claude-spawn.service.js +78 -4
- package/dist/services/claude-spawn.service.js.map +1 -1
- package/dist/services/codex-spawn.service.d.ts.map +1 -1
- package/dist/services/codex-spawn.service.js.map +1 -1
- package/dist/services/consolidation.service.d.ts.map +1 -1
- package/dist/services/consolidation.service.js +7 -1
- package/dist/services/consolidation.service.js.map +1 -1
- package/dist/services/copilot-spawn.service.d.ts.map +1 -1
- package/dist/services/copilot-spawn.service.js +8 -1
- package/dist/services/copilot-spawn.service.js.map +1 -1
- package/dist/services/dispatcher/poller.d.ts.map +1 -1
- package/dist/services/dispatcher/poller.js +12 -0
- package/dist/services/dispatcher/poller.js.map +1 -1
- package/dist/services/keeper.service.d.ts.map +1 -1
- package/dist/services/keeper.service.js +97 -1
- package/dist/services/keeper.service.js.map +1 -1
- package/dist/services/memory-pruning.service.d.ts.map +1 -1
- package/dist/services/memory-pruning.service.js +2 -0
- package/dist/services/memory-pruning.service.js.map +1 -1
- package/dist/services/process.service.d.ts.map +1 -1
- package/dist/services/process.service.js +30 -0
- package/dist/services/process.service.js.map +1 -1
- package/dist/services/spawn-common.d.ts.map +1 -1
- package/dist/services/spawn-common.js +9 -2
- package/dist/services/spawn-common.js.map +1 -1
- package/dist/services/thread-config.service.d.ts.map +1 -1
- package/dist/services/thread-config.service.js +1 -1
- package/dist/services/thread-config.service.js.map +1 -1
- package/dist/services/thread-health.service.d.ts +19 -1
- package/dist/services/thread-health.service.d.ts.map +1 -1
- package/dist/services/thread-health.service.js +74 -4
- package/dist/services/thread-health.service.js.map +1 -1
- package/dist/services/thread-lifecycle.service.d.ts.map +1 -1
- package/dist/services/thread-lifecycle.service.js +5 -0
- package/dist/services/thread-lifecycle.service.js.map +1 -1
- package/dist/services/worker-cleanup.service.d.ts +12 -0
- package/dist/services/worker-cleanup.service.d.ts.map +1 -1
- package/dist/services/worker-cleanup.service.js +278 -67
- package/dist/services/worker-cleanup.service.js.map +1 -1
- package/dist/sessions/mcp-registry.d.ts +6 -0
- package/dist/sessions/mcp-registry.d.ts.map +1 -1
- package/dist/sessions/mcp-registry.js +8 -0
- package/dist/sessions/mcp-registry.js.map +1 -1
- package/dist/tools/defs/session-defs.d.ts.map +1 -1
- package/dist/tools/defs/session-defs.js +20 -2
- package/dist/tools/defs/session-defs.js.map +1 -1
- package/dist/tools/delegate-tool.d.ts.map +1 -1
- package/dist/tools/delegate-tool.js +84 -4
- package/dist/tools/delegate-tool.js.map +1 -1
- package/dist/tools/memory-tools.d.ts +1 -1
- package/dist/tools/memory-tools.d.ts.map +1 -1
- package/dist/tools/memory-tools.js +4 -4
- package/dist/tools/memory-tools.js.map +1 -1
- package/dist/tools/utility-tools.d.ts.map +1 -1
- package/dist/tools/utility-tools.js +1 -1
- package/dist/tools/utility-tools.js.map +1 -1
- package/dist/tools/wait/drive-handler.js +1 -1
- package/dist/tools/wait/drive-handler.js.map +1 -1
- package/dist/tools/wait/media-processor.d.ts.map +1 -1
- package/dist/tools/wait/media-processor.js +2 -2
- package/dist/tools/wait/media-processor.js.map +1 -1
- package/dist/tools/wait/message-delivery.js +1 -1
- package/dist/tools/wait/message-delivery.js.map +1 -1
- package/dist/tools/wait/poll-loop.d.ts.map +1 -1
- package/dist/tools/wait/poll-loop.js +18 -0
- package/dist/tools/wait/poll-loop.js.map +1 -1
- package/package.json +3 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"thread-lifecycle.service.js","sourceRoot":"","sources":["../../src/services/thread-lifecycle.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAIjC,MAAM,CAAN,IAAY,WAUX;AAVD,WAAY,WAAW;IACrB,kCAAmB,CAAA;IACnB,oCAAqB,CAAA;IACrB,gCAAiB,CAAA;IACjB,kCAAmB,CAAA;IACnB,8BAAe,CAAA;IACf,kCAAmB,CAAA;IACnB,gCAAiB,CAAA;IACjB,oCAAqB,CAAA;IACrB,kCAAmB,CAAA;AACrB,CAAC,EAVW,WAAW,KAAX,WAAW,QAUtB;AA0CD,MAAM,iBAAiB,GAAkD;IACvE,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAChG,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/F,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;QAC5B,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,OAAO;QACnB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,QAAQ;QACpB,WAAW,CAAC,OAAO;KACpB,CAAC;IACF,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,EAAE,4DAA4D;IAChG,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,CAAC;QAC3B,WAAW,CAAC,OAAO;QACnB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,QAAQ;KACrB,CAAC;IACF,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC1E,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC9F,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;CACvD,CAAC;AAEF,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACpD,KAAK,MAAM,QAAQ,IAAI;QACrB,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,QAAQ,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,QAAQ,OAAO,CAAC;QAC3C,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,QAAQ,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"thread-lifecycle.service.js","sourceRoot":"","sources":["../../src/services/thread-lifecycle.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAIjC,MAAM,CAAN,IAAY,WAUX;AAVD,WAAY,WAAW;IACrB,kCAAmB,CAAA;IACnB,oCAAqB,CAAA;IACrB,gCAAiB,CAAA;IACjB,kCAAmB,CAAA;IACnB,8BAAe,CAAA;IACf,kCAAmB,CAAA;IACnB,gCAAiB,CAAA;IACjB,oCAAqB,CAAA;IACrB,kCAAmB,CAAA;AACrB,CAAC,EAVW,WAAW,KAAX,WAAW,QAUtB;AA0CD,MAAM,iBAAiB,GAAkD;IACvE,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAChG,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/F,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;QAC5B,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,OAAO;QACnB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,QAAQ;QACpB,WAAW,CAAC,OAAO;KACpB,CAAC;IACF,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,CAAC,EAAE,4DAA4D;IAChG,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,CAAC;QAC3B,WAAW,CAAC,OAAO;QACnB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,QAAQ;KACrB,CAAC;IACF,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC1E,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC9F,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;CACvD,CAAC;AAEF,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACpD,KAAK,MAAM,QAAQ,IAAI;QACrB,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,QAAQ,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,QAAQ,OAAO,CAAC;QAC3C,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,QAAQ,QAAQ,CAAC;QAC1C,iEAAiE;QACjE,2EAA2E;QAC3E,2EAA2E;QAC3E,yCAAyC;QACzC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,QAAQ,kBAAkB,CAAC;KAClD,EAAE,CAAC;QACF,IAAI,CAAC;YACH,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,yDAAyD;QAC3D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,OAAO,sBAAsB;IAEd;IACA;IACA;IACA;IAJnB,YACmB,gBAAmC,EACnC,iBAAqC,EACrC,eAA+C,EAC/C,MAA6B;QAH7B,qBAAgB,GAAhB,gBAAgB,CAAmB;QACnC,sBAAiB,GAAjB,iBAAiB,CAAoB;QACrC,oBAAe,GAAf,eAAe,CAAgC;QAC/C,WAAM,GAAN,MAAM,CAAuB;IAC7C,CAAC;IAEJ,cAAc,CAAC,EAAY,EAAE,KAA0B;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE7E,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE;gBACrD,eAAe,EAAE,KAAK,CAAC,eAAe;aACvC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,CAAC,QAAQ,qBAAqB,CAAC,CAAC;QAChG,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,cAAc,CAAC,EAAY,EAAE,QAAgB,EAAE,UAA+B,EAAE;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACnE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE;YAC/C,GAAG,OAAO;YACV,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAC/D,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACnE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,QAAQ,GAAG,CAAC,CAAC;QAC9E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,WAAW,CAAC,EAAY,EAAE,QAAgB,EAAE,UAA4B,EAAE;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QAChE,gFAAgF;QAChF,2EAA2E;QAC3E,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;YACrF,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAEzE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE;YAC/C,GAAG,OAAO;YACV,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAC/D,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;QAChE,yEAAyE;QACzE,4EAA4E;QAC5E,uEAAuE;QACvE,uEAAuE;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,QAAQ,GAAG,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,UAAU,CAAC,EAAY,EAAE,QAAgB;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC/D,4EAA4E;QAC5E,sEAAsE;QACtE,2EAA2E;QAC3E,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC;QAC/E,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QAE/D,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE;YAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC;YACxC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC/D,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,CAAC,OAAO,CAAC,SAAS;YAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,KAAK,QAAQ,GAAG,CAAC,CAAC;QAChF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,aAAa,CAAC,EAAY,EAAE,QAAgB;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAClE,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,QAAQ;YAAE,OAAO,OAAO,CAAC;QAC5D,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QAE7E,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE;YAC/C,MAAM,EAAE,WAAW,CAAC,QAAQ;YAC5B,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAClE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,QAAQ,GAAG,CAAC,CAAC;QAC/E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,UAAU,CAAC,EAAY,EAAE,KAAsB;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAExE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE;YACrD,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAErF,IAAI,KAAK,CAAC,eAAe,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC7C,KAAK,IAAI,CAAC,eAAe;iBACtB,WAAW,CACV,KAAK,CAAC,MAAM,EACZ,uDAAuD,EACvD,SAAS,EACT,KAAK,CAAC,eAAe,CACtB;iBACA,KAAK,CAAC,GAAG,EAAE;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,wEAAwE,KAAK,CAAC,QAAQ,EAAE,CACzF,CAAC;YACJ,CAAC,CAAC,CAAC;QACP,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,4CAA4C,KAAK,CAAC,QAAQ,OAAO,KAAK,CAAC,eAAe,GAAG,CAC1F,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,cAAc,CAAC,EAAY,EAAE,QAAgB;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,gBAAgB,CAAC,EAAY,EAAE,QAAgB,EAAE,WAAwB;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,kBAAkB,CAAC,CAAC;QAEvE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE;YAC/C,MAAM,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC;YAC1C,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,WAAW,KAAK,QAAQ,GAAG,CAAC,CAAC;QACxF,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,EAAY,EAAE,QAAgB,EAAE,MAAc;QAClE,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,YAAY,QAAQ,oBAAoB,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CACtB,OAAyC,EACzC,IAAiB,EACjB,MAAc;QAEd,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,WAAW,CAAC,MAAM,IAAI,IAAI,KAAK,WAAW,CAAC,OAAO,EAAE,CAAC;gBAChE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,iEAAiE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,wBAAwB,OAAO,OAAO,IAAI,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAEO,iBAAiB,CACvB,MAA0B,EAC1B,IAAY,EACZ,QAAgB,EAChB,OAA6B;QAE7B,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1D,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrE,KAAK,MAAM,KAAK,IAAI,OAAO,IAAI,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,IAAI;gBAAE,SAAS;YACvC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC3D,IAAI,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,KAA2B;QAClD,OAAO,KAAoB,CAAC;IAC9B,CAAC;IAEO,gBAAgB,CAAC,KAAkB;QACzC,IAAI,KAAK,KAAK,WAAW,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF,CAAC;QACJ,CAAC;QACD,OAAO,KAA6B,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -1,4 +1,15 @@
|
|
|
1
1
|
import { type ThreadLifecycleService } from "./thread-lifecycle.service.js";
|
|
2
|
+
export declare const WORKER_GRACE_MS: number;
|
|
3
|
+
/** True while a worker is inside its TTL grace window (winding down, but its MCP
|
|
4
|
+
* socket is still open so it can deliver a final result). Read by the poll loop
|
|
5
|
+
* (to tell the worker to wrap up) and by send_message_to_thread (to honestly
|
|
6
|
+
* reject new orchestrator work rather than silently queue it to a dying worker). */
|
|
7
|
+
export declare function isWorkerInGrace(threadId: number): boolean;
|
|
8
|
+
/** Drop a worker's grace-map entry without running teardown. Used by the terminal
|
|
9
|
+
* self-exit path in handleProcessExit, which returns without calling
|
|
10
|
+
* decommissionWorker (the DB row is already Archived/Expired) and would otherwise
|
|
11
|
+
* leak the in-memory deadline forever. Best-effort and safe if no entry exists. */
|
|
12
|
+
export declare function clearWorkerGrace(threadId: number): void;
|
|
2
13
|
export declare function decommissionWorker(thread: {
|
|
3
14
|
threadId: number;
|
|
4
15
|
pid?: number;
|
|
@@ -13,6 +24,7 @@ export declare function decommissionWorker(thread: {
|
|
|
13
24
|
chatId: string;
|
|
14
25
|
threadLifecycle: ThreadLifecycleService;
|
|
15
26
|
}): Promise<void>;
|
|
27
|
+
export declare const DEFAULT_WORKER_TTL_MS: number;
|
|
16
28
|
export declare function cleanupExpiredWorkers(db: ReturnType<typeof import("../data/memory/index.js").initMemoryDb>, telegram: {
|
|
17
29
|
deleteForumTopic(chatId: string, threadId: number): Promise<void>;
|
|
18
30
|
sendMessage?(chatId: string, text: string, parseMode?: string, threadId?: number): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-cleanup.service.d.ts","sourceRoot":"","sources":["../../src/services/worker-cleanup.service.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAe,KAAK,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"worker-cleanup.service.d.ts","sourceRoot":"","sources":["../../src/services/worker-cleanup.service.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAe,KAAK,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAyBzF,eAAO,MAAM,eAAe,QAGxB,CAAC;AAQL;;;qFAGqF;AACrF,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;oFAGoF;AACpF,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAEvD;AAED,wBAAsB,kBAAkB,CACtC,MAAM,EAAE;IACN,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,IAAI,EAAE;IACJ,EAAE,EAAE,UAAU,CAAC,cAAc,yBAAyB,EAAE,YAAY,CAAC,CAAC;IACtE,QAAQ,EAAE;QAAE,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,sBAAsB,CAAC;CACzC,GACA,OAAO,CAAC,IAAI,CAAC,CAsGf;AAoED,eAAO,MAAM,qBAAqB,QAAiB,CAAC;AA2JpD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,CAAC,cAAc,yBAAyB,EAAE,YAAY,CAAC,EACrE,QAAQ,EAAE;IACR,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,WAAW,CAAC,CACV,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB,EACD,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,sBAAsB,EACvC,KAAK,GAAE,MAA8B,GACpC,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAwThD"}
|
|
@@ -2,7 +2,7 @@ import { unlinkSync, readFileSync, writeFileSync, renameSync } from "node:fs";
|
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { archiveNotesForThread } from "../data/memory/note-archival.js";
|
|
4
4
|
import { closeThreadSessions } from "../sessions/mcp-registry.js";
|
|
5
|
-
import { getExplicitTelegramTopicId } from "../data/memory/thread-registry.js";
|
|
5
|
+
import { getExplicitTelegramTopicId, updateThread } from "../data/memory/thread-registry.js";
|
|
6
6
|
import { errorMessage } from "../lib/utils.js";
|
|
7
7
|
import { spawnedThreads, readPidFiles, killProcessTree, isProcessAlive, clearTaskConsumed, deletePendingTaskMailbox, PROCESS_BASE_DIR, } from "./process.service.js";
|
|
8
8
|
import { ThreadState } from "./thread-lifecycle.service.js";
|
|
@@ -12,7 +12,50 @@ import { log } from "../lib/logger.js";
|
|
|
12
12
|
function topicAlreadyGone(msg) {
|
|
13
13
|
return /TOPIC_ID_INVALID|message thread not found/i.test(msg);
|
|
14
14
|
}
|
|
15
|
+
// ── Worker TTL grace window ─────────────────────────────────────────────────
|
|
16
|
+
// When a worker crosses its 1h TTL we do NOT force-kill it on the spot. The
|
|
17
|
+
// first sweep that finds it past-TTL instead puts it into a GRACE window: its
|
|
18
|
+
// MCP socket is left OPEN so the worker can finish its current turn and deliver
|
|
19
|
+
// its final result to the orchestrator (via send_message_to_thread). A LATER
|
|
20
|
+
// sweep, once the grace deadline has passed, runs the full teardown.
|
|
21
|
+
//
|
|
22
|
+
// WORKER_GRACE_MS is a MINIMUM, not a precise teardown deadline. Phase 1 (enter
|
|
23
|
+
// grace) and Phase 2 (teardown) each require a separate cleanup sweep, and the
|
|
24
|
+
// sweep runs on a fixed cadence (WORKER_CLEANUP_INTERVAL_MS in background-runner,
|
|
25
|
+
// ~5 min). So the grace window is quantized up to the next sweep boundary: the
|
|
26
|
+
// effective teardown of a worker that never delivers is bounded by TTL + grace
|
|
27
|
+
// rounded up to ~1–2 sweep intervals — worst case on the order of 5–10 min past
|
|
28
|
+
// TTL, never unbounded. A well-behaved worker self-exits within seconds of the
|
|
29
|
+
// wrap-up signal and never waits for Phase 2. The TTL itself is unchanged; grace
|
|
30
|
+
// only adds a bounded clean-shutdown tail. Override 60s via WORKER_GRACE_SECONDS.
|
|
31
|
+
export const WORKER_GRACE_MS = (() => {
|
|
32
|
+
const s = Number(process.env.WORKER_GRACE_SECONDS);
|
|
33
|
+
return Number.isFinite(s) && s > 0 ? s * 1000 : 60_000;
|
|
34
|
+
})();
|
|
35
|
+
// threadId → absolute ms deadline after which the full teardown runs. In-memory
|
|
36
|
+
// only: on a server restart the map is empty, so a still-past-TTL worker simply
|
|
37
|
+
// re-enters grace on the next sweep and is cleaned up then — it can never get
|
|
38
|
+
// stuck un-cleaned. The hard TTL+grace cap resets at worst once per restart.
|
|
39
|
+
const workerGraceDeadlines = new Map();
|
|
40
|
+
/** True while a worker is inside its TTL grace window (winding down, but its MCP
|
|
41
|
+
* socket is still open so it can deliver a final result). Read by the poll loop
|
|
42
|
+
* (to tell the worker to wrap up) and by send_message_to_thread (to honestly
|
|
43
|
+
* reject new orchestrator work rather than silently queue it to a dying worker). */
|
|
44
|
+
export function isWorkerInGrace(threadId) {
|
|
45
|
+
return workerGraceDeadlines.has(threadId);
|
|
46
|
+
}
|
|
47
|
+
/** Drop a worker's grace-map entry without running teardown. Used by the terminal
|
|
48
|
+
* self-exit path in handleProcessExit, which returns without calling
|
|
49
|
+
* decommissionWorker (the DB row is already Archived/Expired) and would otherwise
|
|
50
|
+
* leak the in-memory deadline forever. Best-effort and safe if no entry exists. */
|
|
51
|
+
export function clearWorkerGrace(threadId) {
|
|
52
|
+
workerGraceDeadlines.delete(threadId);
|
|
53
|
+
}
|
|
15
54
|
export async function decommissionWorker(thread, deps) {
|
|
55
|
+
// Grace is over the moment full teardown begins — drop any deadline so the map
|
|
56
|
+
// never retains a stale entry (also covers the self-exit path where the worker
|
|
57
|
+
// exited during grace and handleProcessExit decommissions it directly).
|
|
58
|
+
workerGraceDeadlines.delete(thread.threadId);
|
|
16
59
|
// 0. Mark thread as Exiting so the state machine reflects teardown-in-progress
|
|
17
60
|
try {
|
|
18
61
|
deps.threadLifecycle.transitionThread(deps.db, thread.threadId, ThreadState.Exiting);
|
|
@@ -20,23 +63,54 @@ export async function decommissionWorker(thread, deps) {
|
|
|
20
63
|
catch (err) {
|
|
21
64
|
log.warn(`[decommission] transitionThread failed for ${thread.threadId}: ${errorMessage(err)}`);
|
|
22
65
|
}
|
|
23
|
-
// 1.
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
66
|
+
// 1. Kill the process FIRST (no-op if already dead). We must confirm the agent
|
|
67
|
+
// is gone before touching its MCP transport. Closing the socket of a LIVE
|
|
68
|
+
// agent we could NOT kill (EPERM on a server's own Windows child) does NOT
|
|
69
|
+
// make a mid-task agent self-exit — it just ORPHANS it: alive, but with no
|
|
70
|
+
// MCP connection → wedged forever. So kill, then close only if confirmed dead.
|
|
71
|
+
// Routed through the same circuit breaker as the backstop reaper: this path
|
|
72
|
+
// re-runs for an unkillable worker on every other sweep (Phase 1 re-enters
|
|
73
|
+
// grace, Phase 2 kills again), so leaving it unthrottled would keep feeding
|
|
74
|
+
// the very starvation the breaker exists to stop. Skipping the kill lands on
|
|
75
|
+
// the identical "still alive" branch below as a failed kill would.
|
|
76
|
+
let killIssued = false;
|
|
77
|
+
if (thread.pid !== undefined) {
|
|
78
|
+
killIssued = await killWorkerWithBackoff(thread.pid, thread.threadId, "decommission", thread.startedAt, Date.now());
|
|
31
79
|
}
|
|
32
|
-
|
|
33
|
-
|
|
80
|
+
// 2. Tear down MCP transport(s) ONLY once the process is confirmed dead. A dead
|
|
81
|
+
// process's socket is already gone, so this is bookkeeping (and marks the
|
|
82
|
+
// close expected so http-server's onclose does not re-kill). If the kill
|
|
83
|
+
// EPERM-failed and the agent is still alive, LEAVE the transport OPEN so the
|
|
84
|
+
// live agent stays connected and functional instead of being orphaned — an
|
|
85
|
+
// idle unkillable worker simply stays parked in wait, which is harmless.
|
|
86
|
+
const processGone = thread.pid === undefined || !isProcessAlive(thread.pid, thread.startedAt);
|
|
87
|
+
if (processGone) {
|
|
88
|
+
// Process confirmed gone — drop the unkillable anchor and the breaker entry so
|
|
89
|
+
// neither outlives the thread (and a future thread reusing this id starts clean).
|
|
90
|
+
clearUnkillableAttempt(thread.threadId);
|
|
91
|
+
try {
|
|
92
|
+
const closed = closeThreadSessions(thread.threadId);
|
|
93
|
+
if (closed > 0)
|
|
94
|
+
log.info(`[decommission] Closed ${closed} MCP transport(s) for thread ${thread.threadId}.`);
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
log.warn(`[decommission] closeThreadSessions failed for ${thread.threadId}: ${errorMessage(err)}`);
|
|
98
|
+
}
|
|
34
99
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
100
|
+
else {
|
|
101
|
+
// Kill EPERM-failed and the agent is still alive. Abort teardown entirely:
|
|
102
|
+
// deleting the topic, archiving notes/DB, or dropping in-memory tracking now
|
|
103
|
+
// would erase a RUNNING worker's topic and untrack it while it keeps going.
|
|
104
|
+
// Leave everything intact (transport already left open above) and return — a
|
|
105
|
+
// later sweep re-runs decommission and completes it once the process is dead
|
|
106
|
+
// (the keeper escalates a truly unkillable process in the meantime).
|
|
107
|
+
log.warn(`[decommission] Worker thread ${thread.threadId} survived kill (still alive) — aborting teardown; topic, notes, DB row and tracking preserved until the process is confirmed dead.`);
|
|
108
|
+
// Charge the breaker only for a kill this call actually issued.
|
|
109
|
+
if (thread.pid !== undefined && killIssued)
|
|
110
|
+
recordKillFailure(thread.pid, thread.threadId);
|
|
111
|
+
return;
|
|
38
112
|
}
|
|
39
|
-
// 3. Delete Telegram topic
|
|
113
|
+
// 3. Delete Telegram topic (reached only when the process is confirmed dead).
|
|
40
114
|
try {
|
|
41
115
|
const topicId = getExplicitTelegramTopicId(deps.db, thread.threadId) ?? thread.threadId;
|
|
42
116
|
await deps.telegram.deleteForumTopic(deps.chatId, topicId);
|
|
@@ -129,9 +203,8 @@ async function finalizeDeadWorker(threadId, pidFilePath, db, telegram, chatId, t
|
|
|
129
203
|
}
|
|
130
204
|
}
|
|
131
205
|
}
|
|
132
|
-
const DEFAULT_WORKER_TTL_MS = 60 * 60 * 1000;
|
|
206
|
+
export const DEFAULT_WORKER_TTL_MS = 60 * 60 * 1000;
|
|
133
207
|
const UNKILLABLE_RENOTIFY_MS = 60 * 60 * 1000; // re-ping the operator at most hourly once the alert threshold is crossed
|
|
134
|
-
let orphanSweepDone = false;
|
|
135
208
|
// How long a worker must resist every kill before we bother the operator. Wedged
|
|
136
209
|
// workers almost always self-thaw and die within seconds-to-minutes (Windows S0
|
|
137
210
|
// Modern Standby freezes background processes; taskkill EPERMs until they wake),
|
|
@@ -192,17 +265,91 @@ function recordFirstUnkillableAttempt(threadId, now) {
|
|
|
192
265
|
* reuses the id starts its clock fresh. */
|
|
193
266
|
function clearUnkillableAttempt(threadId) {
|
|
194
267
|
unkillableNotifiedAt.delete(threadId);
|
|
268
|
+
killBackoff.delete(threadId);
|
|
195
269
|
const store = loadUnkillableStore();
|
|
196
270
|
if (store.delete(threadId))
|
|
197
271
|
persistUnkillableStore();
|
|
198
272
|
}
|
|
273
|
+
// ── Kill-retry circuit breaker ─────────────────────────────────────────────
|
|
274
|
+
// A process that EPERMs is not going to yield to the same taskkill five minutes
|
|
275
|
+
// later, but the sweep used to re-issue it every single cycle regardless. Each
|
|
276
|
+
// attempt is a spawned taskkill plus a process-tree walk, and with a couple of
|
|
277
|
+
// wedged workers that was enough to starve the sweep itself — the scheduler went
|
|
278
|
+
// quiet for hours while the loop did nothing but re-kill the unkillable.
|
|
279
|
+
//
|
|
280
|
+
// So: try hard a few times (most wedged workers thaw and die within minutes),
|
|
281
|
+
// then back off to an occasional retry. Liveness is still checked every sweep —
|
|
282
|
+
// only the kill is throttled — so a worker that thaws and exits on its own is
|
|
283
|
+
// still reaped promptly.
|
|
284
|
+
const UNKILLABLE_MAX_FAST_ATTEMPTS = 5;
|
|
285
|
+
const UNKILLABLE_RETRY_BACKOFF_MS = 30 * 60 * 1000;
|
|
286
|
+
// An entry in real backoff is retried at least every UNKILLABLE_RETRY_BACKOFF_MS,
|
|
287
|
+
// so its lastAttemptAt is never more than ~30m stale. Anything older belongs to a
|
|
288
|
+
// thread that left by a path which never observes its death (pid file removed by
|
|
289
|
+
// hand, exit handler that skips decommission) and would otherwise sit in the map
|
|
290
|
+
// for the life of the process.
|
|
291
|
+
const KILL_BACKOFF_TTL_MS = 24 * 60 * 60 * 1000;
|
|
292
|
+
/** threadId → the pid we have been failing to kill, how many times, and when last. */
|
|
293
|
+
const killBackoff = new Map();
|
|
294
|
+
/** Drop entries no sweep has touched in a day — see {@link KILL_BACKOFF_TTL_MS}. */
|
|
295
|
+
function pruneKillBackoff(now) {
|
|
296
|
+
for (const [threadId, state] of killBackoff)
|
|
297
|
+
if (now - state.lastAttemptAt > KILL_BACKOFF_TTL_MS)
|
|
298
|
+
killBackoff.delete(threadId);
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Kill a worker's process tree unless the circuit breaker says to sit this sweep
|
|
302
|
+
* out. Returns true when a kill was actually issued — the caller MUST pass that
|
|
303
|
+
* back to {@link recordKillFailure} if the process survives, and must NOT infer
|
|
304
|
+
* it from timestamps: sweeps can overlap (setInterval does not wait for the
|
|
305
|
+
* previous run), so "the recorded attempt is mine" is not something a shared
|
|
306
|
+
* clock can answer.
|
|
307
|
+
*/
|
|
308
|
+
async function killWorkerWithBackoff(pid, threadId, caller, startedAt, now) {
|
|
309
|
+
const state = killBackoff.get(threadId);
|
|
310
|
+
// The failure history belongs to a PID, not to a thread id. A thread that is
|
|
311
|
+
// torn down and respawned gets a brand-new process which has resisted nothing
|
|
312
|
+
// and deserves the full fast-attempt budget — otherwise it inherits its
|
|
313
|
+
// predecessor's 30-minute skip and lingers for no reason.
|
|
314
|
+
const staleEntry = state !== undefined && state.pid !== pid;
|
|
315
|
+
if (state !== undefined &&
|
|
316
|
+
!staleEntry &&
|
|
317
|
+
state.failures >= UNKILLABLE_MAX_FAST_ATTEMPTS &&
|
|
318
|
+
now - state.lastAttemptAt < UNKILLABLE_RETRY_BACKOFF_MS) {
|
|
319
|
+
log.debug(`[worker-cleanup] Worker pid=${pid} thread=${threadId} in kill backoff (${state.failures} failures, last attempt ${Math.round((now - state.lastAttemptAt) / 60000)}m ago) — skipping kill this sweep (caller=${caller}).`);
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
killBackoff.set(threadId, {
|
|
323
|
+
pid,
|
|
324
|
+
failures: staleEntry ? 0 : (state?.failures ?? 0),
|
|
325
|
+
lastAttemptAt: now,
|
|
326
|
+
});
|
|
327
|
+
await killProcessTree(pid, threadId, caller, startedAt);
|
|
328
|
+
return true;
|
|
329
|
+
}
|
|
330
|
+
/** Count a survived kill against the breaker. Call ONLY when a kill was issued
|
|
331
|
+
* (i.e. {@link killWorkerWithBackoff} returned true) and the pid is still alive. */
|
|
332
|
+
function recordKillFailure(pid, threadId) {
|
|
333
|
+
const state = killBackoff.get(threadId);
|
|
334
|
+
// A concurrent sweep may have re-keyed the entry onto a different pid; that
|
|
335
|
+
// process's budget is not ours to spend.
|
|
336
|
+
if (state === undefined || state.pid !== pid)
|
|
337
|
+
return;
|
|
338
|
+
state.failures += 1;
|
|
339
|
+
if (state.failures === UNKILLABLE_MAX_FAST_ATTEMPTS) {
|
|
340
|
+
log.warn(`[worker-cleanup] Worker pid=${pid} thread=${threadId} survived ${state.failures} kills — backing off to one retry per ${UNKILLABLE_RETRY_BACKOFF_MS / 60000}m so the sweep is not starved.`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
199
343
|
export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycle, ttlMs = DEFAULT_WORKER_TTL_MS) {
|
|
200
344
|
const result = { cleaned: 0, errors: [] };
|
|
201
345
|
const now = Date.now();
|
|
346
|
+
pruneKillBackoff(now);
|
|
202
347
|
for (const thread of spawnedThreads.filter((t) => t.threadType === "worker" && now - t.createdAt > ttlMs)) {
|
|
203
348
|
try {
|
|
204
|
-
|
|
205
|
-
|
|
349
|
+
// Two-phase teardown: only counts as "cleaned" once the worker is actually
|
|
350
|
+
// torn down (Phase 2). Phase 1 just opens the grace window and returns false.
|
|
351
|
+
if (await cleanupSingleWorker(thread, db, telegram, chatId, threadLifecycle))
|
|
352
|
+
result.cleaned++;
|
|
206
353
|
}
|
|
207
354
|
catch (err) {
|
|
208
355
|
result.errors.push(`Thread ${thread.threadId}: ${errorMessage(err)}`);
|
|
@@ -236,17 +383,11 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
236
383
|
const inMemEntry = spawnedThreads.find((t) => t.threadId === row.thread_id);
|
|
237
384
|
const killPid = pidEntry?.pid ?? inMemEntry?.pid;
|
|
238
385
|
const killStartedAt = pidEntry?.startedAt ?? inMemEntry?.startedAt;
|
|
239
|
-
//
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
if (closed > 0)
|
|
245
|
-
log.info(`[worker-cleanup] Closed ${closed} MCP transport(s) for stale worker thread ${row.thread_id}.`);
|
|
246
|
-
}
|
|
247
|
-
catch (err) {
|
|
248
|
-
log.warn(`[worker-cleanup] closeThreadSessions failed for stale thread ${row.thread_id}: ${errorMessage(err)}`);
|
|
249
|
-
}
|
|
386
|
+
// Kill the process FIRST (best-effort). Do NOT close the MCP transport of
|
|
387
|
+
// a still-live agent: closing a mid-task agent's socket does not make it
|
|
388
|
+
// self-exit — it orphans it (alive, but no MCP connection → wedged). So
|
|
389
|
+
// confirm death via the kill, then tear the transport down only in the
|
|
390
|
+
// confirmed-dead branch below.
|
|
250
391
|
if (killPid)
|
|
251
392
|
await killProcessTree(killPid, row.thread_id, "worker-cleanup-expired", killStartedAt);
|
|
252
393
|
// Invariant: a worker's topic is deleted ONLY when its process is confirmed
|
|
@@ -255,6 +396,16 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
255
396
|
// here would orphan the topic of a still-running worker.
|
|
256
397
|
const processGone = !killPid || !isProcessAlive(killPid, killStartedAt);
|
|
257
398
|
if (processGone) {
|
|
399
|
+
// Confirmed dead — its socket is already gone, so closing the transport
|
|
400
|
+
// is bookkeeping (and marks the close expected). Safe now.
|
|
401
|
+
try {
|
|
402
|
+
const closed = closeThreadSessions(row.thread_id);
|
|
403
|
+
if (closed > 0)
|
|
404
|
+
log.info(`[worker-cleanup] Closed ${closed} MCP transport(s) for stale worker thread ${row.thread_id}.`);
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
log.warn(`[worker-cleanup] closeThreadSessions failed for stale thread ${row.thread_id}: ${errorMessage(err)}`);
|
|
408
|
+
}
|
|
258
409
|
try {
|
|
259
410
|
// For workers, thread_id IS the Telegram topic ID (created via createManagedTopic).
|
|
260
411
|
// Use explicit telegram_topic_id if set, otherwise fall back to thread_id.
|
|
@@ -275,7 +426,9 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
275
426
|
clearUnkillableAttempt(row.thread_id);
|
|
276
427
|
}
|
|
277
428
|
else {
|
|
278
|
-
|
|
429
|
+
// Left alive by an EPERM kill — leave the MCP transport OPEN (closing it
|
|
430
|
+
// would orphan the live agent) and defer topic deletion to the reaper.
|
|
431
|
+
log.warn(`[worker-cleanup] Stale worker thread ${row.thread_id} survived kill — leaving its MCP transport open and deferring topic deletion to reaper (process still alive).`);
|
|
279
432
|
}
|
|
280
433
|
threadLifecycle.archiveThread(db, row.thread_id);
|
|
281
434
|
try {
|
|
@@ -300,30 +453,51 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
300
453
|
catch (err) {
|
|
301
454
|
log.warn(`[worker-cleanup] Failed to query stale DB workers: ${errorMessage(err)}`);
|
|
302
455
|
}
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
456
|
+
// Periodic self-clearing sweep: delete leftover Telegram topics for workers that
|
|
457
|
+
// are already archived but whose topic was never deleted (e.g. archived via a raw
|
|
458
|
+
// status update that bypassed the normal decommission/topic-delete path). Safe to
|
|
459
|
+
// run every cycle: it selects ONLY rows that still carry a topic id, and nulls the
|
|
460
|
+
// id after a successful (or already-gone) delete, so each orphan is handled once
|
|
461
|
+
// and the query converges to empty. The `IS NOT NULL` gate guarantees we delete
|
|
462
|
+
// the EXPLICIT topic id — never the thread_id fallback that once removed a root
|
|
463
|
+
// topic (see topic-deletion-bug).
|
|
464
|
+
try {
|
|
465
|
+
const orphanRows = db
|
|
466
|
+
.prepare(`SELECT thread_id, telegram_topic_id FROM thread_registry
|
|
467
|
+
WHERE type = 'worker' AND status = 'archived' AND telegram_topic_id IS NOT NULL`)
|
|
468
|
+
.all();
|
|
469
|
+
for (const row of orphanRows) {
|
|
470
|
+
// Guard against a concurrent unarchive between snapshot and delete: re-read
|
|
471
|
+
// the row and skip if it is no longer an archived worker still carrying this
|
|
472
|
+
// exact topic id — otherwise we could delete a just-reactivated worker's topic.
|
|
473
|
+
const cur = db
|
|
474
|
+
.prepare(`SELECT status, telegram_topic_id FROM thread_registry WHERE thread_id = ?`)
|
|
475
|
+
.get(row.thread_id);
|
|
476
|
+
if (!cur || cur.status !== "archived" || cur.telegram_topic_id !== row.telegram_topic_id)
|
|
477
|
+
continue;
|
|
478
|
+
try {
|
|
479
|
+
await telegram.deleteForumTopic(chatId, row.telegram_topic_id);
|
|
480
|
+
updateThread(db, row.thread_id, { telegramTopicId: null });
|
|
481
|
+
result.cleaned++;
|
|
482
|
+
log.info(`[worker-cleanup] Deleted orphan topic ${row.telegram_topic_id} for archived worker ${row.thread_id}`);
|
|
483
|
+
}
|
|
484
|
+
catch (err) {
|
|
485
|
+
const msg = errorMessage(err);
|
|
486
|
+
if (topicAlreadyGone(msg)) {
|
|
487
|
+
// Already gone IS the desired end state — clear the id so we stop revisiting it.
|
|
488
|
+
updateThread(db, row.thread_id, { telegramTopicId: null });
|
|
489
|
+
log.info(`[worker-cleanup] Orphan topic for archived worker ${row.thread_id} already gone (${msg}) — cleared.`);
|
|
318
490
|
}
|
|
319
|
-
|
|
320
|
-
//
|
|
491
|
+
else {
|
|
492
|
+
// Genuine transient Telegram failure — leave the id set to retry next cycle.
|
|
493
|
+
log.warn(`[worker-cleanup] deleteForumTopic failed for archived worker ${row.thread_id} — will retry next cycle: ${msg}`);
|
|
321
494
|
}
|
|
322
495
|
}
|
|
323
496
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
}
|
|
497
|
+
}
|
|
498
|
+
catch (err) {
|
|
499
|
+
log.warn(`[worker-cleanup] Failed to sweep orphan topics: ${errorMessage(err)}`);
|
|
500
|
+
}
|
|
327
501
|
// Backstop process reaper: a worker process must be stopped in ANY case once
|
|
328
502
|
// it should no longer be running. The stale-rows path above skips already-
|
|
329
503
|
// archived rows, and the archived-topic sweep never killed the
|
|
@@ -339,6 +513,12 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
339
513
|
const isWorker = entry.threadType === "worker" || row?.type === "worker";
|
|
340
514
|
if (!isWorker)
|
|
341
515
|
continue;
|
|
516
|
+
// Respect the TTL grace window: a worker mid-grace is INTENTIONALLY left
|
|
517
|
+
// alive with its MCP socket open so it can deliver a final result. The
|
|
518
|
+
// two-phase teardown in cleanupSingleWorker owns killing it once grace
|
|
519
|
+
// expires — reaping it here would defeat the grace and drop its result.
|
|
520
|
+
if (isWorkerInGrace(entry.threadId))
|
|
521
|
+
continue;
|
|
342
522
|
// HARD lifetime cap from created_at (see staleRows above): an idle worker
|
|
343
523
|
// refreshing last_active_at must still die once it outlives ttlMs.
|
|
344
524
|
const pastTtl = row?.ts ? now - new Date(row.ts).getTime() > ttlMs : true;
|
|
@@ -354,16 +534,15 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
354
534
|
result.cleaned++;
|
|
355
535
|
continue;
|
|
356
536
|
}
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
await killProcessTree(entry.pid, entry.threadId, "worker-cleanup-reap", entry.startedAt);
|
|
537
|
+
// Attempt to reap the lingering process. Do NOT close its MCP transport
|
|
538
|
+
// first: for a BUSY agent the kill can EPERM-fail (a server cannot reliably
|
|
539
|
+
// kill its own Windows child), and closing the socket of an agent we could
|
|
540
|
+
// not kill does not make a mid-task agent self-exit — it ORPHANS it (alive,
|
|
541
|
+
// but with no MCP connection → wedged forever). So kill first; only once the
|
|
542
|
+
// process is confirmed dead (below) do we tear the transport down.
|
|
543
|
+
// Skip the kill (not the liveness check) while this pid is in backoff —
|
|
544
|
+
// see the circuit-breaker note above.
|
|
545
|
+
const killIssued = await killWorkerWithBackoff(entry.pid, entry.threadId, "worker-cleanup-reap", entry.startedAt, now);
|
|
367
546
|
// killProcessTree resolves even when the kill FAILS (e.g. EPERM on a
|
|
368
547
|
// Windows claude.exe whose wedged bash.exe child makes taskkill /T a
|
|
369
548
|
// no-op). Re-check liveness before claiming success — otherwise we log a
|
|
@@ -371,17 +550,20 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
371
550
|
// hiding a zombie that then reappears and "reaps" forever. If it survived,
|
|
372
551
|
// surface it to the operator (rate-limited) so it gets a manual kill.
|
|
373
552
|
if (isProcessAlive(entry.pid, entry.startedAt)) {
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
|
|
553
|
+
// Still alive after an EPERM kill — LEAVE its MCP transport OPEN so the
|
|
554
|
+
// agent stays connected rather than being orphaned (closing a live agent's
|
|
555
|
+
// socket does not make it self-exit if it is mid-task). An idle unkillable
|
|
556
|
+
// worker simply stays parked in wait, which is harmless; it is escalated
|
|
557
|
+
// to the operator below once it has resisted for the full threshold.
|
|
379
558
|
const name = row?.name ?? `thread ${entry.threadId}`;
|
|
380
559
|
// Anchor the clock at the first survived kill (persisted). Keep retrying
|
|
381
560
|
// silently — most wedged workers thaw and die within minutes. Only once a
|
|
382
561
|
// worker has resisted for the full threshold do we escalate to Telegram.
|
|
383
562
|
const firstAttempt = recordFirstUnkillableAttempt(entry.threadId, now);
|
|
384
563
|
const stuckMs = now - firstAttempt;
|
|
564
|
+
// Only count a failure for a kill we actually issued.
|
|
565
|
+
if (killIssued)
|
|
566
|
+
recordKillFailure(entry.pid, entry.threadId);
|
|
385
567
|
if (stuckMs < UNKILLABLE_ALERT_THRESHOLD_MS) {
|
|
386
568
|
log.debug(`[worker-cleanup] Worker pid=${entry.pid} thread=${entry.threadId} ('${name}') survived kill — stuck ${Math.round(stuckMs / 60000)}m, below alert threshold; retrying silently.`);
|
|
387
569
|
continue;
|
|
@@ -408,8 +590,15 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
408
590
|
continue;
|
|
409
591
|
}
|
|
410
592
|
clearUnkillableAttempt(entry.threadId);
|
|
593
|
+
// Confirmed dead after kill — its socket is already gone, so closing the
|
|
594
|
+
// transport is bookkeeping (and marks the close expected). Do it now, then
|
|
595
|
+
// delete the topic (invariant-safe) and finalize.
|
|
596
|
+
const closedTransports = closeThreadSessions(entry.threadId);
|
|
597
|
+
if (closedTransports > 0) {
|
|
598
|
+
log.info(`[worker-cleanup] Closed ${closedTransports} MCP transport(s) for reaped worker thread ${entry.threadId}.`);
|
|
599
|
+
}
|
|
411
600
|
log.info(`[worker-cleanup] Reaped lingering worker pid=${entry.pid} thread=${entry.threadId} (status=${row?.status ?? "unknown"}, pastTtl=${pastTtl})`);
|
|
412
|
-
//
|
|
601
|
+
// Delete the topic (invariant-safe) and finalize.
|
|
413
602
|
await finalizeDeadWorker(entry.threadId, entry.filePath, db, telegram, chatId, threadLifecycle);
|
|
414
603
|
result.cleaned++;
|
|
415
604
|
}
|
|
@@ -419,7 +608,29 @@ export async function cleanupExpiredWorkers(db, telegram, chatId, threadLifecycl
|
|
|
419
608
|
}
|
|
420
609
|
return result;
|
|
421
610
|
}
|
|
611
|
+
/** Two-phase TTL teardown for a past-TTL worker still tracked in spawnedThreads.
|
|
612
|
+
* Phase 1 (first past-TTL sweep): enter the grace window and return WITHOUT
|
|
613
|
+
* closing the MCP socket or killing the process, so the worker can finish its
|
|
614
|
+
* turn and deliver a final result. Phase 2 (a later sweep, once the grace
|
|
615
|
+
* deadline passes): run the full teardown. Returns true only when Phase 2
|
|
616
|
+
* actually tore the worker down, so the caller only counts real cleanups. */
|
|
422
617
|
async function cleanupSingleWorker(thread, db, telegram, chatId, threadLifecycle) {
|
|
618
|
+
const now = Date.now();
|
|
619
|
+
const deadline = workerGraceDeadlines.get(thread.threadId);
|
|
620
|
+
if (deadline === undefined) {
|
|
621
|
+
// Phase 1: enter grace. Do NOT closeThreadSessions and do NOT killProcessTree
|
|
622
|
+
// — the MCP socket MUST stay open so the worker can still deliver its result.
|
|
623
|
+
// The poll loop surfaces the "wrap up" message to the worker via isWorkerInGrace.
|
|
624
|
+
workerGraceDeadlines.set(thread.threadId, now + WORKER_GRACE_MS);
|
|
625
|
+
log.info(`[worker-cleanup] Worker ${thread.threadId} ('${thread.name}') past TTL — entering grace (min ${Math.round(WORKER_GRACE_MS / 1000)}s) to deliver its final result. Teardown happens on a later sweep, so effective kill of a non-delivering worker is bounded by the sweep cadence (~5–10 min past TTL), not exactly ${Math.round(WORKER_GRACE_MS / 1000)}s.`);
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
if (now <= deadline) {
|
|
629
|
+
// Still inside grace — keep the socket open and let the worker finish.
|
|
630
|
+
return false;
|
|
631
|
+
}
|
|
632
|
+
// Phase 2: grace elapsed — full teardown (decommissionWorker clears the deadline).
|
|
423
633
|
await decommissionWorker(thread, { db, telegram, chatId, threadLifecycle });
|
|
634
|
+
return true;
|
|
424
635
|
}
|
|
425
636
|
//# sourceMappingURL=worker-cleanup.service.js.map
|