visionclaw 0.1.218 → 0.1.219-dev.codex-issue-2805-worker-runtime.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/api-proxy.d.ts +0 -16
- package/dist/agent/api-proxy.d.ts.map +1 -1
- package/dist/agent/api-proxy.js +9 -8
- package/dist/agent/api-proxy.js.map +1 -1
- package/dist/agent/browser-launcher.d.ts +3 -0
- package/dist/agent/browser-launcher.d.ts.map +1 -1
- package/dist/agent/browser-launcher.js +101 -43
- package/dist/agent/browser-launcher.js.map +1 -1
- package/dist/agent/coding-conventions.d.ts +8 -0
- package/dist/agent/coding-conventions.d.ts.map +1 -0
- package/dist/agent/coding-conventions.js +19 -0
- package/dist/agent/coding-conventions.js.map +1 -0
- package/dist/agent/command-handlers.d.ts.map +1 -1
- package/dist/agent/command-handlers.js +4 -16
- package/dist/agent/command-handlers.js.map +1 -1
- package/dist/agent/context-action-restart-fallback.d.ts.map +1 -1
- package/dist/agent/context-action-restart-fallback.js +9 -3
- package/dist/agent/context-action-restart-fallback.js.map +1 -1
- package/dist/agent/context-refresh.d.ts.map +1 -1
- package/dist/agent/context-refresh.js +0 -7
- package/dist/agent/context-refresh.js.map +1 -1
- package/dist/agent/context.d.ts +2 -4
- package/dist/agent/context.d.ts.map +1 -1
- package/dist/agent/context.js +12 -21
- package/dist/agent/context.js.map +1 -1
- package/dist/agent/engine-fallback.d.ts +0 -35
- package/dist/agent/engine-fallback.d.ts.map +1 -1
- package/dist/agent/engine-fallback.js +4 -3
- package/dist/agent/engine-fallback.js.map +1 -1
- package/dist/agent/engine-switch.d.ts +19 -38
- package/dist/agent/engine-switch.d.ts.map +1 -1
- package/dist/agent/engine-switch.js +46 -111
- package/dist/agent/engine-switch.js.map +1 -1
- package/dist/agent/engines/browser-tool-policy.d.ts +12 -1
- package/dist/agent/engines/browser-tool-policy.d.ts.map +1 -1
- package/dist/agent/engines/browser-tool-policy.js +17 -2
- package/dist/agent/engines/browser-tool-policy.js.map +1 -1
- package/dist/agent/engines/claude/executor-shell-groups.d.ts +11 -0
- package/dist/agent/engines/claude/executor-shell-groups.d.ts.map +1 -0
- package/dist/agent/engines/claude/executor-shell-groups.js +22 -0
- package/dist/agent/engines/claude/executor-shell-groups.js.map +1 -0
- package/dist/agent/engines/claude/session.d.ts +13 -4
- package/dist/agent/engines/claude/session.d.ts.map +1 -1
- package/dist/agent/engines/claude/session.js +165 -84
- package/dist/agent/engines/claude/session.js.map +1 -1
- package/dist/agent/engines/pi/shared/usage.d.ts +5 -0
- package/dist/agent/engines/pi/shared/usage.d.ts.map +1 -1
- package/dist/agent/engines/pi/shared/usage.js +22 -3
- package/dist/agent/engines/pi/shared/usage.js.map +1 -1
- package/dist/agent/engines/pi/tools.d.ts +3 -10
- package/dist/agent/engines/pi/tools.d.ts.map +1 -1
- package/dist/agent/engines/pi/tools.js +6 -17
- package/dist/agent/engines/pi/tools.js.map +1 -1
- package/dist/agent/engines/pi-coding/executor-shell.d.ts +4 -0
- package/dist/agent/engines/pi-coding/executor-shell.d.ts.map +1 -0
- package/dist/agent/engines/pi-coding/executor-shell.js +175 -0
- package/dist/agent/engines/pi-coding/executor-shell.js.map +1 -0
- package/dist/agent/engines/pi-coding/session.d.ts +4 -4
- package/dist/agent/engines/pi-coding/session.d.ts.map +1 -1
- package/dist/agent/engines/pi-coding/session.js +118 -66
- package/dist/agent/engines/pi-coding/session.js.map +1 -1
- package/dist/agent/engines/pi-coding/tools.d.ts +2 -5
- package/dist/agent/engines/pi-coding/tools.d.ts.map +1 -1
- package/dist/agent/engines/pi-coding/tools.js +7 -5
- package/dist/agent/engines/pi-coding/tools.js.map +1 -1
- package/dist/agent/engines/session-types.d.ts +8 -8
- package/dist/agent/engines/session-types.d.ts.map +1 -1
- package/dist/agent/external-context/interrupt.d.ts +0 -2
- package/dist/agent/external-context/interrupt.d.ts.map +1 -1
- package/dist/agent/external-context/interrupt.js +0 -1
- package/dist/agent/external-context/interrupt.js.map +1 -1
- package/dist/agent/external-context/registry.d.ts +0 -2
- package/dist/agent/external-context/registry.d.ts.map +1 -1
- package/dist/agent/external-context/registry.js +2 -4
- package/dist/agent/external-context/registry.js.map +1 -1
- package/dist/agent/external-context/runtime.d.ts.map +1 -1
- package/dist/agent/external-context/runtime.js +15 -14
- package/dist/agent/external-context/runtime.js.map +1 -1
- package/dist/agent/external-context/types.d.ts +0 -2
- package/dist/agent/external-context/types.d.ts.map +1 -1
- package/dist/agent/fast-responder.d.ts +4 -20
- package/dist/agent/fast-responder.d.ts.map +1 -1
- package/dist/agent/fast-responder.js +28 -42
- package/dist/agent/fast-responder.js.map +1 -1
- package/dist/agent/interrupt-handler.d.ts +0 -6
- package/dist/agent/interrupt-handler.d.ts.map +1 -1
- package/dist/agent/interrupt-handler.js +5 -47
- package/dist/agent/interrupt-handler.js.map +1 -1
- package/dist/agent/lab-run-system-prompt.d.ts.map +1 -1
- package/dist/agent/lab-run-system-prompt.js +4 -1
- package/dist/agent/lab-run-system-prompt.js.map +1 -1
- package/dist/agent/lifecycle-trace.d.ts.map +1 -1
- package/dist/agent/lifecycle-trace.js +6 -5
- package/dist/agent/lifecycle-trace.js.map +1 -1
- package/dist/agent/loop.d.ts +8 -2
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/agent/loop.js +291 -156
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/memory-protocol.d.ts +2 -2
- package/dist/agent/memory-protocol.d.ts.map +1 -1
- package/dist/agent/memory-protocol.js +3 -15
- package/dist/agent/memory-protocol.js.map +1 -1
- package/dist/agent/memory-store.d.ts +58 -0
- package/dist/agent/memory-store.d.ts.map +1 -0
- package/dist/agent/memory-store.js +152 -0
- package/dist/agent/memory-store.js.map +1 -0
- package/dist/agent/model-provider.d.ts +18 -5
- package/dist/agent/model-provider.d.ts.map +1 -1
- package/dist/agent/model-provider.js +58 -18
- package/dist/agent/model-provider.js.map +1 -1
- package/dist/agent/nightly-maintenance/dreaming-batch.d.ts +184 -0
- package/dist/agent/nightly-maintenance/dreaming-batch.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-batch.js +192 -0
- package/dist/agent/nightly-maintenance/dreaming-batch.js.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-files.d.ts +48 -0
- package/dist/agent/nightly-maintenance/dreaming-files.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-files.js +109 -0
- package/dist/agent/nightly-maintenance/dreaming-files.js.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-ledger.d.ts +62 -0
- package/dist/agent/nightly-maintenance/dreaming-ledger.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-ledger.js +146 -0
- package/dist/agent/nightly-maintenance/dreaming-ledger.js.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-planner.d.ts +86 -0
- package/dist/agent/nightly-maintenance/dreaming-planner.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-planner.js +304 -0
- package/dist/agent/nightly-maintenance/dreaming-planner.js.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-runner.d.ts +93 -0
- package/dist/agent/nightly-maintenance/dreaming-runner.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-runner.js +156 -0
- package/dist/agent/nightly-maintenance/dreaming-runner.js.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-sources.d.ts +115 -0
- package/dist/agent/nightly-maintenance/dreaming-sources.d.ts.map +1 -0
- package/dist/agent/nightly-maintenance/dreaming-sources.js +270 -0
- package/dist/agent/nightly-maintenance/dreaming-sources.js.map +1 -0
- package/dist/agent/nightly-maintenance/index.d.ts +28 -8
- package/dist/agent/nightly-maintenance/index.d.ts.map +1 -1
- package/dist/agent/nightly-maintenance/index.js +42 -15
- package/dist/agent/nightly-maintenance/index.js.map +1 -1
- package/dist/agent/nightly-maintenance/skill-learning-gate.js +1 -1
- package/dist/agent/nightly-maintenance/skill-learning-gate.js.map +1 -1
- package/dist/agent/nightly-maintenance/vocabulary-processor.d.ts +1 -1
- package/dist/agent/nightly-maintenance/vocabulary-processor.d.ts.map +1 -1
- package/dist/agent/owner-channel-resolver.d.ts +2 -0
- package/dist/agent/owner-channel-resolver.d.ts.map +1 -1
- package/dist/agent/owner-channel-resolver.js +4 -0
- package/dist/agent/owner-channel-resolver.js.map +1 -1
- package/dist/agent/product-identity.d.ts +5 -4
- package/dist/agent/product-identity.d.ts.map +1 -1
- package/dist/agent/product-identity.js +7 -6
- package/dist/agent/product-identity.js.map +1 -1
- package/dist/agent/public-activity.js +1 -1
- package/dist/agent/public-activity.js.map +1 -1
- package/dist/agent/runtime-context.d.ts +27 -0
- package/dist/agent/runtime-context.d.ts.map +1 -0
- package/dist/agent/runtime-context.js +60 -0
- package/dist/agent/runtime-context.js.map +1 -0
- package/dist/agent/runtime-disclosure-policy.d.ts +6 -0
- package/dist/agent/runtime-disclosure-policy.d.ts.map +1 -0
- package/dist/agent/runtime-disclosure-policy.js +16 -0
- package/dist/agent/runtime-disclosure-policy.js.map +1 -0
- package/dist/agent/runtime-surface.d.ts +15 -4
- package/dist/agent/runtime-surface.d.ts.map +1 -1
- package/dist/agent/runtime-surface.js +0 -1
- package/dist/agent/runtime-surface.js.map +1 -1
- package/dist/agent/session-manager.d.ts +30 -126
- package/dist/agent/session-manager.d.ts.map +1 -1
- package/dist/agent/session-manager.js +60 -484
- package/dist/agent/session-manager.js.map +1 -1
- package/dist/agent/state.d.ts +42 -2
- package/dist/agent/state.d.ts.map +1 -1
- package/dist/agent/state.js +21 -11
- package/dist/agent/state.js.map +1 -1
- package/dist/agent/stream-handler.d.ts +6 -13
- package/dist/agent/stream-handler.d.ts.map +1 -1
- package/dist/agent/stream-handler.js +8 -62
- package/dist/agent/stream-handler.js.map +1 -1
- package/dist/agent/system-prompt.d.ts +2 -0
- package/dist/agent/system-prompt.d.ts.map +1 -1
- package/dist/agent/system-prompt.js +21 -20
- package/dist/agent/system-prompt.js.map +1 -1
- package/dist/agent/tool-usage.d.ts.map +1 -1
- package/dist/agent/tool-usage.js +0 -1
- package/dist/agent/tool-usage.js.map +1 -1
- package/dist/agent/transcript/transcript-backfill.js +3 -3
- package/dist/agent/transcript/transcript-backfill.js.map +1 -1
- package/dist/agent/transcript/transcript-indexer.d.ts +3 -31
- package/dist/agent/transcript/transcript-indexer.d.ts.map +1 -1
- package/dist/agent/transcript/transcript-indexer.js +58 -57
- package/dist/agent/transcript/transcript-indexer.js.map +1 -1
- package/dist/agent/transcript-parsers/trivial-tools.d.ts.map +1 -1
- package/dist/agent/transcript-parsers/trivial-tools.js +0 -2
- package/dist/agent/transcript-parsers/trivial-tools.js.map +1 -1
- package/dist/agent/transcript-parsers/types.d.ts +1 -1
- package/dist/agent/transcript-summarizer.d.ts +14 -0
- package/dist/agent/transcript-summarizer.d.ts.map +1 -1
- package/dist/agent/transcript-summarizer.js +69 -45
- package/dist/agent/transcript-summarizer.js.map +1 -1
- package/dist/agent/usage/usage-report-queue.d.ts.map +1 -1
- package/dist/agent/usage/usage-report-queue.js +53 -52
- package/dist/agent/usage/usage-report-queue.js.map +1 -1
- package/dist/agent/usage/usage-report-types.d.ts +1 -1
- package/dist/agent/usage/usage-report-types.d.ts.map +1 -1
- package/dist/agent/vocabulary-glossary.d.ts +3 -3
- package/dist/agent/wake-cycle-runner.d.ts +22 -17
- package/dist/agent/wake-cycle-runner.d.ts.map +1 -1
- package/dist/agent/wake-cycle-runner.js +190 -320
- package/dist/agent/wake-cycle-runner.js.map +1 -1
- package/dist/builtin-skills/owner-manual/SKILL.md +2 -2
- package/dist/calendar/occurrence-ownership.d.ts.map +1 -1
- package/dist/calendar/occurrence-ownership.js +6 -5
- package/dist/calendar/occurrence-ownership.js.map +1 -1
- package/dist/calendar/scheduler.d.ts.map +1 -1
- package/dist/calendar/scheduler.js +4 -3
- package/dist/calendar/scheduler.js.map +1 -1
- package/dist/channels/interface.d.ts +8 -2
- package/dist/channels/interface.d.ts.map +1 -1
- package/dist/channels/interface.js +8 -1
- package/dist/channels/interface.js.map +1 -1
- package/dist/channels/manager.d.ts +12 -0
- package/dist/channels/manager.d.ts.map +1 -1
- package/dist/channels/manager.js +97 -8
- package/dist/channels/manager.js.map +1 -1
- package/dist/channels/queue.d.ts.map +1 -1
- package/dist/channels/queue.js +8 -0
- package/dist/channels/queue.js.map +1 -1
- package/dist/channels/telegram.d.ts.map +1 -1
- package/dist/channels/telegram.js +28 -2
- package/dist/channels/telegram.js.map +1 -1
- package/dist/channels/tui.d.ts +14 -1
- package/dist/channels/tui.d.ts.map +1 -1
- package/dist/channels/tui.js +64 -2
- package/dist/channels/tui.js.map +1 -1
- package/dist/channels/unavailable-target.d.ts +20 -0
- package/dist/channels/unavailable-target.d.ts.map +1 -0
- package/dist/channels/unavailable-target.js +82 -0
- package/dist/channels/unavailable-target.js.map +1 -0
- package/dist/cli/index.d.ts +2 -2
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +22 -53
- package/dist/cli/index.js.map +1 -1
- package/dist/config/index.d.ts +13 -7
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +51 -64
- package/dist/config/index.js.map +1 -1
- package/dist/config/types.d.ts +28 -7
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/types.js +22 -20
- package/dist/config/types.js.map +1 -1
- package/dist/conversations/lazy-runtime.d.ts +5 -0
- package/dist/conversations/lazy-runtime.d.ts.map +1 -0
- package/dist/conversations/lazy-runtime.js +12 -0
- package/dist/conversations/lazy-runtime.js.map +1 -0
- package/dist/conversations/runtime.d.ts +17 -0
- package/dist/conversations/runtime.d.ts.map +1 -0
- package/dist/conversations/runtime.js +92 -0
- package/dist/conversations/runtime.js.map +1 -0
- package/dist/conversations/scheduler.d.ts +57 -0
- package/dist/conversations/scheduler.d.ts.map +1 -0
- package/dist/conversations/scheduler.js +175 -0
- package/dist/conversations/scheduler.js.map +1 -0
- package/dist/data-processing/pipeline.d.ts +0 -2
- package/dist/data-processing/pipeline.d.ts.map +1 -1
- package/dist/data-processing/pipeline.js +19 -24
- package/dist/data-processing/pipeline.js.map +1 -1
- package/dist/data-processing/restructure.d.ts +0 -2
- package/dist/data-processing/restructure.d.ts.map +1 -1
- package/dist/data-processing/restructure.js +6 -9
- package/dist/data-processing/restructure.js.map +1 -1
- package/dist/data-processing/types.d.ts +2 -9
- package/dist/data-processing/types.d.ts.map +1 -1
- package/dist/data-processing/types.js +2 -13
- package/dist/data-processing/types.js.map +1 -1
- package/dist/email/email-route-store.d.ts.map +1 -1
- package/dist/email/email-route-store.js +6 -5
- package/dist/email/email-route-store.js.map +1 -1
- package/dist/i18n/messages.d.ts +26 -2
- package/dist/i18n/messages.d.ts.map +1 -1
- package/dist/i18n/messages.js +30 -5
- package/dist/i18n/messages.js.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/logger.d.ts +11 -0
- package/dist/logger.d.ts.map +1 -1
- package/dist/logger.js +49 -1
- package/dist/logger.js.map +1 -1
- package/dist/obs/mini-app/status.d.ts.map +1 -1
- package/dist/obs/mini-app/status.js +0 -1
- package/dist/obs/mini-app/status.js.map +1 -1
- package/dist/onboarding/onboarding-service-client.d.ts +1 -0
- package/dist/onboarding/onboarding-service-client.d.ts.map +1 -1
- package/dist/onboarding/onboarding-service-client.js.map +1 -1
- package/dist/onboarding/remote-set-owner.d.ts.map +1 -1
- package/dist/onboarding/remote-set-owner.js +0 -1
- package/dist/onboarding/remote-set-owner.js.map +1 -1
- package/dist/onboarding/set-owner.d.ts +1 -1
- package/dist/onboarding/set-owner.d.ts.map +1 -1
- package/dist/onboarding/set-owner.js +3 -6
- package/dist/onboarding/set-owner.js.map +1 -1
- package/dist/reconfigure.d.ts.map +1 -1
- package/dist/reconfigure.js +1 -32
- package/dist/reconfigure.js.map +1 -1
- package/dist/reset.d.ts.map +1 -1
- package/dist/reset.js +3 -1
- package/dist/reset.js.map +1 -1
- package/dist/threads/active-threads-store.d.ts.map +1 -1
- package/dist/threads/active-threads-store.js +8 -27
- package/dist/threads/active-threads-store.js.map +1 -1
- package/dist/threads/dreaming-execution.d.ts +10 -0
- package/dist/threads/dreaming-execution.d.ts.map +1 -0
- package/dist/threads/dreaming-execution.js +78 -0
- package/dist/threads/dreaming-execution.js.map +1 -0
- package/dist/threads/execution-authority.d.ts +43 -0
- package/dist/threads/execution-authority.d.ts.map +1 -0
- package/dist/threads/execution-authority.js +75 -0
- package/dist/threads/execution-authority.js.map +1 -0
- package/dist/threads/execution-budget.d.ts +107 -0
- package/dist/threads/execution-budget.d.ts.map +1 -0
- package/dist/threads/execution-budget.js +257 -0
- package/dist/threads/execution-budget.js.map +1 -0
- package/dist/threads/execution-host.d.ts +17 -0
- package/dist/threads/execution-host.d.ts.map +1 -0
- package/dist/threads/execution-host.js +66 -0
- package/dist/threads/execution-host.js.map +1 -0
- package/dist/threads/execution-scheduler.d.ts +162 -0
- package/dist/threads/execution-scheduler.d.ts.map +1 -0
- package/dist/threads/execution-scheduler.js +413 -0
- package/dist/threads/execution-scheduler.js.map +1 -0
- package/dist/threads/execution-snapshot.d.ts +91 -0
- package/dist/threads/execution-snapshot.d.ts.map +1 -0
- package/dist/threads/execution-snapshot.js +146 -0
- package/dist/threads/execution-snapshot.js.map +1 -0
- package/dist/threads/execution-store.d.ts +374 -0
- package/dist/threads/execution-store.d.ts.map +1 -0
- package/dist/threads/execution-store.js +910 -0
- package/dist/threads/execution-store.js.map +1 -0
- package/dist/threads/execution-thread-error.d.ts +9 -0
- package/dist/threads/execution-thread-error.d.ts.map +1 -0
- package/dist/threads/execution-thread-error.js +12 -0
- package/dist/threads/execution-thread-error.js.map +1 -0
- package/dist/threads/executor-exit.d.ts +18 -0
- package/dist/threads/executor-exit.d.ts.map +1 -0
- package/dist/threads/executor-exit.js +21 -0
- package/dist/threads/executor-exit.js.map +1 -0
- package/dist/threads/foreground-task.d.ts +91 -0
- package/dist/threads/foreground-task.d.ts.map +1 -0
- package/dist/threads/foreground-task.js +191 -0
- package/dist/threads/foreground-task.js.map +1 -0
- package/dist/threads/foreman-business.d.ts +9 -0
- package/dist/threads/foreman-business.d.ts.map +1 -0
- package/dist/threads/foreman-business.js +117 -0
- package/dist/threads/foreman-business.js.map +1 -0
- package/dist/threads/foreman-client.d.ts +18 -0
- package/dist/threads/foreman-client.d.ts.map +1 -0
- package/dist/threads/foreman-client.js +51 -0
- package/dist/threads/foreman-client.js.map +1 -0
- package/dist/threads/foreman-delivery.d.ts +5 -0
- package/dist/threads/foreman-delivery.d.ts.map +1 -0
- package/dist/threads/foreman-delivery.js +21 -0
- package/dist/threads/foreman-delivery.js.map +1 -0
- package/dist/threads/foreman-execution.d.ts +32 -0
- package/dist/threads/foreman-execution.d.ts.map +1 -0
- package/dist/threads/foreman-execution.js +90 -0
- package/dist/threads/foreman-execution.js.map +1 -0
- package/dist/threads/foreman-host.d.ts +77 -0
- package/dist/threads/foreman-host.d.ts.map +1 -0
- package/dist/threads/foreman-host.js +891 -0
- package/dist/threads/foreman-host.js.map +1 -0
- package/dist/threads/foreman-protocol.d.ts +242 -0
- package/dist/threads/foreman-protocol.d.ts.map +1 -0
- package/dist/threads/foreman-protocol.js +116 -0
- package/dist/threads/foreman-protocol.js.map +1 -0
- package/dist/threads/foreman-store.d.ts +151 -0
- package/dist/threads/foreman-store.d.ts.map +1 -0
- package/dist/threads/foreman-store.js +299 -0
- package/dist/threads/foreman-store.js.map +1 -0
- package/dist/threads/generic-execution.d.ts +25 -0
- package/dist/threads/generic-execution.d.ts.map +1 -0
- package/dist/threads/generic-execution.js +86 -0
- package/dist/threads/generic-execution.js.map +1 -0
- package/dist/threads/init-thread-dir.d.ts +2 -2
- package/dist/threads/init-thread-dir.d.ts.map +1 -1
- package/dist/threads/init-thread-dir.js +6 -6
- package/dist/threads/init-thread-dir.js.map +1 -1
- package/dist/threads/ipc.d.ts +108 -2
- package/dist/threads/ipc.d.ts.map +1 -1
- package/dist/threads/ipc.js +14 -2
- package/dist/threads/ipc.js.map +1 -1
- package/dist/threads/lab-run-thread.d.ts +3 -5
- package/dist/threads/lab-run-thread.d.ts.map +1 -1
- package/dist/threads/lab-run-thread.js +3 -5
- package/dist/threads/lab-run-thread.js.map +1 -1
- package/dist/threads/lab-runtime-policy.d.ts +26 -21
- package/dist/threads/lab-runtime-policy.d.ts.map +1 -1
- package/dist/threads/lab-runtime-policy.js +31 -126
- package/dist/threads/lab-runtime-policy.js.map +1 -1
- package/dist/threads/process-group.d.ts +41 -0
- package/dist/threads/process-group.d.ts.map +1 -0
- package/dist/threads/process-group.js +161 -0
- package/dist/threads/process-group.js.map +1 -0
- package/dist/threads/relay-adapter.d.ts +8 -0
- package/dist/threads/relay-adapter.d.ts.map +1 -1
- package/dist/threads/relay-adapter.js +26 -1
- package/dist/threads/relay-adapter.js.map +1 -1
- package/dist/threads/supervisor.d.ts +191 -57
- package/dist/threads/supervisor.d.ts.map +1 -1
- package/dist/threads/supervisor.js +766 -427
- package/dist/threads/supervisor.js.map +1 -1
- package/dist/threads/task-workspace.d.ts +8 -0
- package/dist/threads/task-workspace.d.ts.map +1 -0
- package/dist/threads/task-workspace.js +52 -0
- package/dist/threads/task-workspace.js.map +1 -0
- package/dist/threads/worker-entry.d.ts.map +1 -1
- package/dist/threads/worker-entry.js +314 -34
- package/dist/threads/worker-entry.js.map +1 -1
- package/dist/threads/worker-execution-runtime.d.ts +75 -0
- package/dist/threads/worker-execution-runtime.d.ts.map +1 -0
- package/dist/threads/worker-execution-runtime.js +142 -0
- package/dist/threads/worker-execution-runtime.js.map +1 -0
- package/dist/threads/worker-execution.d.ts +35 -0
- package/dist/threads/worker-execution.d.ts.map +1 -0
- package/dist/threads/worker-execution.js +2 -0
- package/dist/threads/worker-execution.js.map +1 -0
- package/dist/threads/worker-runtime-policy.d.ts +42 -0
- package/dist/threads/worker-runtime-policy.d.ts.map +1 -0
- package/dist/threads/worker-runtime-policy.js +98 -0
- package/dist/threads/worker-runtime-policy.js.map +1 -0
- package/dist/tools/adapter-tool.d.ts +12 -0
- package/dist/tools/adapter-tool.d.ts.map +1 -0
- package/dist/tools/adapter-tool.js +2 -0
- package/dist/tools/adapter-tool.js.map +1 -0
- package/dist/tools/data-service.d.ts +2 -2
- package/dist/tools/delegation.d.ts +121 -0
- package/dist/tools/delegation.d.ts.map +1 -0
- package/dist/tools/delegation.js +106 -0
- package/dist/tools/delegation.js.map +1 -0
- package/dist/tools/dream-memory.d.ts +122 -0
- package/dist/tools/dream-memory.d.ts.map +1 -0
- package/dist/tools/dream-memory.js +513 -0
- package/dist/tools/dream-memory.js.map +1 -0
- package/dist/tools/email.d.ts +1 -1
- package/dist/tools/execution-control.d.ts +31 -0
- package/dist/tools/execution-control.d.ts.map +1 -0
- package/dist/tools/execution-control.js +71 -0
- package/dist/tools/execution-control.js.map +1 -0
- package/dist/tools/index.d.ts +5 -7
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +7 -13
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/mcp-manager.d.ts +1 -1
- package/dist/tools/memory.d.ts +1 -1
- package/dist/tools/memory.d.ts.map +1 -1
- package/dist/tools/memory.js +69 -45
- package/dist/tools/memory.js.map +1 -1
- package/dist/tools/notify.d.ts +5 -0
- package/dist/tools/notify.d.ts.map +1 -1
- package/dist/tools/notify.js +45 -10
- package/dist/tools/notify.js.map +1 -1
- package/dist/tools/skill.d.ts +1 -1
- package/dist/tools/totp.d.ts +1 -1
- package/dist/tui/client.d.ts +1 -16
- package/dist/tui/client.d.ts.map +1 -1
- package/dist/tui/client.js +0 -4
- package/dist/tui/client.js.map +1 -1
- package/dist/tui/index.d.ts.map +1 -1
- package/dist/tui/index.js +0 -41
- package/dist/tui/index.js.map +1 -1
- package/dist/tui/protocol.d.ts +2 -1
- package/dist/tui/protocol.d.ts.map +1 -1
- package/dist/tui/protocol.js +2 -1
- package/dist/tui/protocol.js.map +1 -1
- package/dist/tui/renderer.d.ts.map +1 -1
- package/dist/tui/renderer.js +0 -5
- package/dist/tui/renderer.js.map +1 -1
- package/dist/tui/request-store.d.ts +1 -1
- package/dist/tui/request-store.d.ts.map +1 -1
- package/dist/tui/request-store.js +2 -2
- package/dist/tui/request-store.js.map +1 -1
- package/dist/tui/server.d.ts.map +1 -1
- package/dist/tui/server.js +137 -25
- package/dist/tui/server.js.map +1 -1
- package/dist/utils/claude-transcripts.d.ts +6 -0
- package/dist/utils/claude-transcripts.d.ts.map +1 -1
- package/dist/utils/claude-transcripts.js +8 -2
- package/dist/utils/claude-transcripts.js.map +1 -1
- package/dist/utils/restart.d.ts.map +1 -1
- package/dist/utils/restart.js +6 -0
- package/dist/utils/restart.js.map +1 -1
- package/dist-agent/bundle.cjs +2 -2
- package/package.json +1 -1
- package/dist/agent/mailbox.d.ts +0 -50
- package/dist/agent/mailbox.d.ts.map +0 -1
- package/dist/agent/mailbox.js +0 -124
- package/dist/agent/mailbox.js.map +0 -1
- package/dist/agent/nightly-maintenance/memory-dreaming-sweep.d.ts +0 -77
- package/dist/agent/nightly-maintenance/memory-dreaming-sweep.d.ts.map +0 -1
- package/dist/agent/nightly-maintenance/memory-dreaming-sweep.js +0 -158
- package/dist/agent/nightly-maintenance/memory-dreaming-sweep.js.map +0 -1
- package/dist/agent/nightly-maintenance/memory-dreaming.d.ts +0 -116
- package/dist/agent/nightly-maintenance/memory-dreaming.d.ts.map +0 -1
- package/dist/agent/nightly-maintenance/memory-dreaming.js +0 -458
- package/dist/agent/nightly-maintenance/memory-dreaming.js.map +0 -1
- package/dist/builtin-skills/catalog/phone-adb-automation/SKILL.md +0 -412
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_input.sh +0 -132
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_launch.sh +0 -166
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_screenshot.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_security_kbd.py +0 -174
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_setup.sh +0 -274
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_swipe.sh +0 -111
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_tap.sh +0 -87
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_ui_parse.py +0 -176
- package/dist/builtin-skills/catalog/phone-adb-automation/phone_wake_unlock.sh +0 -67
- package/dist/builtin-skills/macos-automation/SKILL.md +0 -132
- package/dist/builtin-skills/transcribe-audio/SKILL.md +0 -122
- package/dist/builtin-skills/visionclaw-manual/SKILL.md +0 -337
- package/dist/obs/rdp/icons/icons/app_windows.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_get.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_send.svg +0 -4
- package/dist/obs/rdp/icons/icons/clip_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard.svg +0 -4
- package/dist/obs/rdp/icons/icons/clipboard_shared.svg +0 -4
- package/dist/obs/rdp/icons/icons/control.svg +0 -4
- package/dist/obs/rdp/icons/icons/desktop.svg +0 -4
- package/dist/obs/rdp/icons/icons/display.svg +0 -4
- package/dist/obs/rdp/icons/icons/launchpad.svg +0 -4
- package/dist/obs/rdp/icons/icons/mission_control.svg +0 -4
- package/dist/obs/rdp/icons/icons/screenshot.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_actual.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_fit.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_in.svg +0 -4
- package/dist/obs/rdp/icons/icons/zoom_out.svg +0 -4
- package/dist/realtime/assets/index.html +0 -1058
- package/dist/realtime/assets/samples/alloy.mp3 +0 -0
- package/dist/realtime/assets/samples/ash.mp3 +0 -0
- package/dist/realtime/assets/samples/ballad.mp3 +0 -0
- package/dist/realtime/assets/samples/cedar.mp3 +0 -0
- package/dist/realtime/assets/samples/coral.mp3 +0 -0
- package/dist/realtime/assets/samples/echo.mp3 +0 -0
- package/dist/realtime/assets/samples/marin.mp3 +0 -0
- package/dist/realtime/assets/samples/sage.mp3 +0 -0
- package/dist/realtime/assets/samples/shimmer.mp3 +0 -0
- package/dist/realtime/assets/samples/verse.mp3 +0 -0
- package/dist/threads/task-contracts.d.ts +0 -194
- package/dist/threads/task-contracts.d.ts.map +0 -1
- package/dist/threads/task-contracts.js +0 -82
- package/dist/threads/task-contracts.js.map +0 -1
- package/dist/threads/task-sidecar.d.ts +0 -143
- package/dist/threads/task-sidecar.d.ts.map +0 -1
- package/dist/threads/task-sidecar.js +0 -309
- package/dist/threads/task-sidecar.js.map +0 -1
- package/dist/threads/task-store.d.ts +0 -172
- package/dist/threads/task-store.d.ts.map +0 -1
- package/dist/threads/task-store.js +0 -594
- package/dist/threads/task-store.js.map +0 -1
- package/dist/threads/thread-task-context.d.ts +0 -40
- package/dist/threads/thread-task-context.d.ts.map +0 -1
- package/dist/threads/thread-task-context.js +0 -159
- package/dist/threads/thread-task-context.js.map +0 -1
- package/dist/threads/thread-task-controller.d.ts +0 -122
- package/dist/threads/thread-task-controller.d.ts.map +0 -1
- package/dist/threads/thread-task-controller.js +0 -276
- package/dist/threads/thread-task-controller.js.map +0 -1
- package/dist/threads/thread-task-runtime.d.ts +0 -320
- package/dist/threads/thread-task-runtime.d.ts.map +0 -1
- package/dist/threads/thread-task-runtime.js +0 -843
- package/dist/threads/thread-task-runtime.js.map +0 -1
- package/dist/tools/switch-session.d.ts +0 -15
- package/dist/tools/switch-session.d.ts.map +0 -1
- package/dist/tools/switch-session.js +0 -88
- package/dist/tools/switch-session.js.map +0 -1
- package/dist/tools/thread-tasks.d.ts +0 -10
- package/dist/tools/thread-tasks.d.ts.map +0 -1
- package/dist/tools/thread-tasks.js +0 -88
- package/dist/tools/thread-tasks.js.map +0 -1
- package/dist-agent/assets/index.html +0 -1058
- package/dist-agent/assets/samples/alloy.mp3 +0 -0
- package/dist-agent/assets/samples/ash.mp3 +0 -0
- package/dist-agent/assets/samples/ballad.mp3 +0 -0
- package/dist-agent/assets/samples/cedar.mp3 +0 -0
- package/dist-agent/assets/samples/coral.mp3 +0 -0
- package/dist-agent/assets/samples/echo.mp3 +0 -0
- package/dist-agent/assets/samples/marin.mp3 +0 -0
- package/dist-agent/assets/samples/sage.mp3 +0 -0
- package/dist-agent/assets/samples/shimmer.mp3 +0 -0
- package/dist-agent/assets/samples/verse.mp3 +0 -0
- package/dist-agent/playwriter-relay.cjs +0 -95
- package/dist-agent/realtime/assets/index.html +0 -1058
- package/dist-agent/realtime/assets/samples/alloy.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/ash.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/ballad.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/cedar.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/coral.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/echo.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/marin.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/sage.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/shimmer.mp3 +0 -0
- package/dist-agent/realtime/assets/samples/verse.mp3 +0 -0
|
@@ -1,20 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* - Spawn worker processes on demand (child_process.fork)
|
|
6
|
-
* - Route messages to workers by threadId
|
|
7
|
-
* - Relay worker replies back to real channels
|
|
8
|
-
* - Enforce concurrency limits (maxConcurrentThreads)
|
|
9
|
-
* - Idle timeout & graceful worker shutdown
|
|
10
|
-
* - Crash detection & auto-restart
|
|
11
|
-
* - Force global commands (/restart, /upgrade, /subscribe, …) to the default thread
|
|
12
|
-
* - Engine switch (/engine): update thread.json + restart worker
|
|
13
|
-
* - Worker admission pause for the main process's nightly maintenance
|
|
14
|
-
* - Graceful supervisor shutdown: notify all workers
|
|
2
|
+
* Host for foreground conversations and the shared background execution pool.
|
|
3
|
+
* Chat queues run through ConversationScheduler. Only admitted executions and
|
|
4
|
+
* LabRuns fork processes; their ledger owns recovery and confirmed stops.
|
|
15
5
|
*/
|
|
16
6
|
import { fork } from "node:child_process";
|
|
17
|
-
import {
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
18
8
|
import fs from "node:fs";
|
|
19
9
|
import path from "node:path";
|
|
20
10
|
import { fileURLToPath } from "node:url";
|
|
@@ -22,17 +12,27 @@ import { PUBLIC_ACTIVITY_HISTORY_LIMIT } from "../agent/public-activity.js";
|
|
|
22
12
|
import { isDeliveryTarget } from "../channels/delivery-target.js";
|
|
23
13
|
import { DEFAULT_THREAD_ID, isEmailChannel, normalizeChannelName } from "../channels/interface.js";
|
|
24
14
|
import { MessageQueue } from "../channels/queue.js";
|
|
25
|
-
import { isThreadTasksEnabled } from "../config/types.js";
|
|
26
|
-
import { taskStorePathForThread, threadHasRunnableTasks } from "./task-store.js";
|
|
27
15
|
import { logger } from "../logger.js";
|
|
28
16
|
import { isOwnerInitiatedMessage } from "../agent/owner-initiated-eligibility.js";
|
|
29
17
|
import { getProfile, getSharedProfileDir, getThreadDir, loadOwnerConfig } from "../config/index.js";
|
|
30
|
-
import { resolveDreamingTargets } from "../agent/nightly-maintenance/memory-dreaming-sweep.js";
|
|
31
18
|
import { resolveFinAppCredentials } from "../finapp-credentials.js";
|
|
32
19
|
import { isWorkerToMainMessage, } from "./ipc.js";
|
|
33
20
|
import { setActiveThreadsSnapshot } from "./active-threads-store.js";
|
|
34
21
|
import { isBundled } from "../utils/version-check.js";
|
|
35
|
-
import { ensureThreadApplication, formatThreadEngineLabel, setThreadEngine } from "./thread-meta.js";
|
|
22
|
+
import { ensureThreadApplication, formatThreadEngineLabel, readThreadMeta, setThreadEngine } from "./thread-meta.js";
|
|
23
|
+
import { getLlmRouting } from "../agent/model-provider.js";
|
|
24
|
+
import { ExecutionStore, isTerminalExecutionState } from "./execution-store.js";
|
|
25
|
+
import { ExecutionScheduler } from "./execution-scheduler.js";
|
|
26
|
+
import { ConversationScheduler } from "../conversations/scheduler.js";
|
|
27
|
+
import { createLazyConversationRuntime } from "../conversations/lazy-runtime.js";
|
|
28
|
+
import { bindRuntimeContext } from "../agent/runtime-context.js";
|
|
29
|
+
import { EXECUTION_VERIFY_KEYS_ENV, createExecutionAuthority, verifyKeysOf, } from "./execution-authority.js";
|
|
30
|
+
import { EXECUTOR_GROUP_REGISTRY_FILE, executorGroupStatus, killExecutorGroupAndWait, reapExitedExecutorGroup, signalExecutorGroup, stopRegisteredGroups, } from "./process-group.js";
|
|
31
|
+
import { ExecutorRefusedError } from "./executor-exit.js";
|
|
32
|
+
import { ExecutionThreadInputError } from "./execution-thread-error.js";
|
|
33
|
+
import { applyExecutionReport } from "./execution-host.js";
|
|
34
|
+
import { genericExecutionThreadId, isGenericExecutionThreadId } from "./generic-execution.js";
|
|
35
|
+
import { DEFAULT_LAB_RUN_BUDGET, EXECUTION_SNAPSHOT_ENV, EXECUTION_SNAPSHOT_VERSION, freezeExecutionModel, readStoredExecutionSnapshot, signExecutionSnapshot, } from "./execution-snapshot.js";
|
|
36
36
|
import { assertMessageApplication, threadApplication } from "./application-context.js";
|
|
37
37
|
import { LAB_RUNTIME_POLICY_ENV, LAB_RUNTIME_POLICY_META_KEY, parseLabRuntimePolicy, signLabRuntimePolicy, verifyLabRuntimePolicyEnvelope, } from "./lab-runtime-policy.js";
|
|
38
38
|
import { isLabRunThreadId } from "./lab-run-thread.js";
|
|
@@ -44,6 +44,8 @@ import { ThreadReplyEscalation, threadDeliveryAlert } from "./reply-escalation.j
|
|
|
44
44
|
import { AgentFeedbackOutbox } from "../agent/feedback-outbox.js";
|
|
45
45
|
import { serializeBrowserRuntimeResolution, serializeDesktopRuntimeResolution, WORKER_BROWSER_RUNTIME_ENV, WORKER_DESKTOP_RUNTIME_ENV, } from "./worker-runtime-resolution.js";
|
|
46
46
|
import { isGlobalCommand } from "./global-commands.js";
|
|
47
|
+
import { getClaudeTranscriptDirForProfileDir } from "../utils/claude-transcripts.js";
|
|
48
|
+
import { isForemanRuntime, supportsTaskRuntime } from "./foreman-protocol.js";
|
|
47
49
|
export { isGlobalCommand } from "./global-commands.js";
|
|
48
50
|
const __filename = fileURLToPath(import.meta.url);
|
|
49
51
|
const __dirname = path.dirname(__filename);
|
|
@@ -78,6 +80,11 @@ export function resolveWorkerEntryPoint() {
|
|
|
78
80
|
// Last resort: whatever entry script started this process.
|
|
79
81
|
return process.argv[1] ?? candidate;
|
|
80
82
|
}
|
|
83
|
+
/** Maintenance executions are owned by the main process's nightly maintenance. */
|
|
84
|
+
const MAINTENANCE_SCOPE = "owner";
|
|
85
|
+
const MAINTENANCE_ENVELOPE = (commandId) => ({
|
|
86
|
+
commandId, caller: { kind: "nightly", ref: "dreaming" }, scope: MAINTENANCE_SCOPE,
|
|
87
|
+
});
|
|
81
88
|
/**
|
|
82
89
|
* Parse a `/engine <name>` command. Returns the requested engine, or null when
|
|
83
90
|
* the text is not an engine command. Recognized names: `claude`, `gpt`, `grok`, and `deepseek`.
|
|
@@ -101,6 +108,8 @@ export function parseEngineCommand(text) {
|
|
|
101
108
|
return null;
|
|
102
109
|
}
|
|
103
110
|
export class ThreadSupervisor {
|
|
111
|
+
conversations;
|
|
112
|
+
foregroundDisabled;
|
|
104
113
|
channelManager;
|
|
105
114
|
messageQueue;
|
|
106
115
|
maxConcurrentThreads;
|
|
@@ -111,13 +120,16 @@ export class ThreadSupervisor {
|
|
|
111
120
|
workerEnv;
|
|
112
121
|
desktopRuntimeResolution;
|
|
113
122
|
browserRuntimeResolution;
|
|
114
|
-
crashRestartDelayMs;
|
|
115
123
|
threadQueueFactory;
|
|
116
124
|
onFeedback;
|
|
117
125
|
onBusyChange;
|
|
118
126
|
agentInstanceId = randomUUID();
|
|
119
|
-
/**
|
|
120
|
-
|
|
127
|
+
/** Host signing authority, in memory only; deliberately unrelated to fin-app credentials. */
|
|
128
|
+
executionAuthority;
|
|
129
|
+
workerStopGraceMs;
|
|
130
|
+
executorGroups;
|
|
131
|
+
/** Executor process groups still being emptied after their leader exited. */
|
|
132
|
+
pendingGroupReaps = new Set();
|
|
121
133
|
latestActivities = new Map();
|
|
122
134
|
replyEscalation = new ThreadReplyEscalation();
|
|
123
135
|
feedbackOutbox = new AgentFeedbackOutbox();
|
|
@@ -125,17 +137,18 @@ export class ThreadSupervisor {
|
|
|
125
137
|
workers = new Map();
|
|
126
138
|
/** Main-process handles for the durable queue.db owned by each thread. */
|
|
127
139
|
threadQueues = new Map();
|
|
128
|
-
/** Crash counts must outlive WorkerState objects, which are replaced on respawn. */
|
|
129
|
-
crashCounts = new Map();
|
|
130
|
-
/** Threads waiting for their bounded crash-backoff timer. */
|
|
131
|
-
restartTimers = new Map();
|
|
132
|
-
/** Startup failures / exhausted crash budgets stay dormant until new work arrives. */
|
|
133
|
-
recoveryExhaustedThreads = new Set();
|
|
134
140
|
/** True once shutdownAll() ran — no new workers may be spawned afterwards. */
|
|
135
141
|
stopped = false;
|
|
136
142
|
maintenancePaused = false;
|
|
137
|
-
|
|
143
|
+
/** The one execution the maintenance owner submitted under its pause. */
|
|
144
|
+
maintenanceExecutionId = null;
|
|
145
|
+
onMaintenanceInput;
|
|
146
|
+
handleForemanCommand;
|
|
147
|
+
handleForemanBusinessRead;
|
|
148
|
+
executionStore;
|
|
149
|
+
executionScheduler;
|
|
138
150
|
constructor(options) {
|
|
151
|
+
this.foregroundDisabled = options.conversationFactory === null;
|
|
139
152
|
this.channelManager = options.channelManager;
|
|
140
153
|
this.messageQueue = options.messageQueue;
|
|
141
154
|
this.maxConcurrentThreads = options.maxConcurrentThreads;
|
|
@@ -146,14 +159,71 @@ export class ThreadSupervisor {
|
|
|
146
159
|
this.workerEnv = options.workerEnv;
|
|
147
160
|
this.desktopRuntimeResolution = options.desktopRuntimeResolution;
|
|
148
161
|
this.browserRuntimeResolution = options.browserRuntimeResolution;
|
|
149
|
-
this.
|
|
162
|
+
this.executionAuthority = options.executionAuthority ?? createExecutionAuthority();
|
|
163
|
+
this.workerStopGraceMs = options.workerStopGraceMs ?? 10_000;
|
|
164
|
+
this.executorGroups = options.executorGroups ?? {
|
|
165
|
+
reap: reapExitedExecutorGroup,
|
|
166
|
+
kill: (pgid) => { signalExecutorGroup(pgid, "SIGKILL"); },
|
|
167
|
+
status: executorGroupStatus,
|
|
168
|
+
killAndWait: killExecutorGroupAndWait,
|
|
169
|
+
stopRegistered: (threadId, timeoutMs) => stopRegisteredGroups(path.join(getThreadDir(threadId), EXECUTOR_GROUP_REGISTRY_FILE), timeoutMs),
|
|
170
|
+
};
|
|
150
171
|
this.onFeedback = options.onFeedback;
|
|
151
172
|
this.onBusyChange = options.onBusyChange;
|
|
173
|
+
this.onMaintenanceInput = options.onMaintenanceInput;
|
|
174
|
+
this.handleForemanCommand = options.handleForemanCommand;
|
|
175
|
+
this.handleForemanBusinessRead = options.handleForemanBusinessRead;
|
|
152
176
|
this.threadQueueFactory = options.threadQueueFactory ?? ((threadId, recoverInFlight) => {
|
|
153
177
|
const threadDir = getThreadDir(threadId);
|
|
154
178
|
fs.mkdirSync(threadDir, { recursive: true });
|
|
155
179
|
return new MessageQueue(path.join(threadDir, "queue.db"), { recoverInFlight });
|
|
156
180
|
});
|
|
181
|
+
// Foreground conversation runtimes never use the background worker transport.
|
|
182
|
+
const conversationCommand = this.handleForemanCommand;
|
|
183
|
+
this.conversations = !this.foregroundDisabled
|
|
184
|
+
? new ConversationScheduler({
|
|
185
|
+
create: options.conversationFactory ?? ((start) => createLazyConversationRuntime({
|
|
186
|
+
...start,
|
|
187
|
+
config: this.config,
|
|
188
|
+
foremanClient: conversationCommand && supportsTaskRuntime()
|
|
189
|
+
? { delegationEnabled: isForemanRuntime(this.config), request: bindRuntimeContext((command) => conversationCommand(start.conversationId, command)) }
|
|
190
|
+
: null,
|
|
191
|
+
desktopRuntimeResolution: this.desktopRuntimeResolution,
|
|
192
|
+
browserRuntimeResolution: this.browserRuntimeResolution,
|
|
193
|
+
})),
|
|
194
|
+
queue: (id) => this.getThreadQueue(id, false),
|
|
195
|
+
emit: (id, event, reply) => this.handleConversationEvent(id, event, reply),
|
|
196
|
+
awareness: (id) => this.getConversationTaskSnapshot(id),
|
|
197
|
+
onBusyChange: this.onBusyChange,
|
|
198
|
+
maxConcurrentTurns: this.config.workers?.foregroundConcurrency ?? 2,
|
|
199
|
+
idleTtlMs: this.idleTimeoutMs,
|
|
200
|
+
})
|
|
201
|
+
: null;
|
|
202
|
+
this.executionStore = options.executionStore === undefined
|
|
203
|
+
? new ExecutionStore()
|
|
204
|
+
: options.executionStore;
|
|
205
|
+
this.executionScheduler = this.executionStore
|
|
206
|
+
? new ExecutionScheduler({
|
|
207
|
+
store: this.executionStore,
|
|
208
|
+
host: {
|
|
209
|
+
availableSlots: (record) => this.admissionSlotsFor(record),
|
|
210
|
+
spawn: (record) => this.spawnExecutionWorker(record),
|
|
211
|
+
stop: (executionId, reason) => {
|
|
212
|
+
this.shutdownWorker(genericExecutionThreadId(executionId), reason);
|
|
213
|
+
},
|
|
214
|
+
timeBudgetMs: (record) => this.timeBudgetOf(record.snapshot, record.executionId),
|
|
215
|
+
},
|
|
216
|
+
delivery: {
|
|
217
|
+
deliverExecutionEvent: options.deliverExecutionEvent ?? (() => Promise.resolve(false)),
|
|
218
|
+
},
|
|
219
|
+
groupStatus: (pgid, processStart) => this.executorGroups.status(pgid, processStart),
|
|
220
|
+
killGroupAndWait: (pgid, processStart, timeoutMs) => this.executorGroups.killAndWait(pgid, processStart, timeoutMs),
|
|
221
|
+
stopRegisteredGroups: (executionId, timeoutMs) => {
|
|
222
|
+
const threadId = isLabRunThreadId(executionId) ? executionId : genericExecutionThreadId(executionId);
|
|
223
|
+
return this.executorGroups.stopRegistered(threadId, timeoutMs);
|
|
224
|
+
},
|
|
225
|
+
})
|
|
226
|
+
: null;
|
|
157
227
|
// Rehydrate queues left by a previous daemon process before any worker is
|
|
158
228
|
// spawned. A worker will wake itself from a non-empty queue on startup.
|
|
159
229
|
if (!options.threadQueueFactory && !options.deferRecovery) {
|
|
@@ -185,37 +255,46 @@ export class ThreadSupervisor {
|
|
|
185
255
|
return entries
|
|
186
256
|
.filter((entry) => entry.isDirectory()
|
|
187
257
|
&& /^[a-z0-9][a-z0-9-]*$/.test(entry.name)
|
|
188
|
-
&& !isLabRunThreadId(entry.name)
|
|
258
|
+
&& !isLabRunThreadId(entry.name)
|
|
259
|
+
&& !isGenericExecutionThreadId(entry.name))
|
|
189
260
|
.map((entry) => entry.name)
|
|
190
261
|
.sort();
|
|
191
262
|
}
|
|
192
|
-
/**
|
|
193
|
-
|
|
194
|
-
return
|
|
263
|
+
/** Persisted business threads whose transcripts Dreaming ingests; never Lab or execution threads. */
|
|
264
|
+
getPersistedThreadIds() {
|
|
265
|
+
return this.listPersistedThreadIds();
|
|
195
266
|
}
|
|
196
|
-
/**
|
|
267
|
+
/**
|
|
268
|
+
* Atomically check all worker work and hold admission for one maintenance
|
|
269
|
+
* step. Internal executions count too: maintenance never starts while any
|
|
270
|
+
* execution runs, is stopping, or waits in the queue.
|
|
271
|
+
*/
|
|
197
272
|
tryPauseForMaintenance() {
|
|
198
|
-
if (this.stopped || this.maintenancePaused
|
|
273
|
+
if (this.stopped || this.maintenancePaused)
|
|
274
|
+
return null;
|
|
275
|
+
if (this.conversations?.hasBusyConversations())
|
|
199
276
|
return null;
|
|
200
277
|
if ([...this.workers.values()].some((worker) => !worker.ready || worker.shuttingDown || this.isWorkerVisiblyBusy(worker)))
|
|
201
278
|
return null;
|
|
202
279
|
if ([...this.threadQueues.values()].some((queue) => queue.hasOutstandingWork()))
|
|
203
280
|
return null;
|
|
281
|
+
if (this.executionScheduler?.hasOutstandingWork())
|
|
282
|
+
return null;
|
|
204
283
|
this.maintenancePaused = true;
|
|
205
284
|
return () => {
|
|
285
|
+
const maintenanceExecution = this.maintenanceExecutionId
|
|
286
|
+
? this.executionStore?.get(this.maintenanceExecutionId) ?? null
|
|
287
|
+
: null;
|
|
288
|
+
if (maintenanceExecution && !isTerminalExecutionState(maintenanceExecution.state)) {
|
|
289
|
+
// The step owner always waits for a confirmed stop; keep the writer
|
|
290
|
+
// fenced rather than admitting others next to a live executor.
|
|
291
|
+
logger.err(`[supervisor] Maintenance released while execution=${maintenanceExecution.executionId} is ${maintenanceExecution.state}`);
|
|
292
|
+
}
|
|
293
|
+
this.maintenanceExecutionId = null;
|
|
206
294
|
this.maintenancePaused = false;
|
|
207
|
-
const pending = this.maintenanceMessages;
|
|
208
|
-
this.maintenanceMessages = [];
|
|
209
295
|
if (this.stopped)
|
|
210
|
-
return;
|
|
211
|
-
|
|
212
|
-
try {
|
|
213
|
-
this.routeMessage(msg, threadId);
|
|
214
|
-
}
|
|
215
|
-
catch (err) {
|
|
216
|
-
logger.warn(`[supervisor] Could not resume queued thread=${threadId}: ${String(err)}`);
|
|
217
|
-
}
|
|
218
|
-
}
|
|
296
|
+
return;
|
|
297
|
+
this.executionScheduler?.requestAdmission();
|
|
219
298
|
this.schedulePendingThreads();
|
|
220
299
|
};
|
|
221
300
|
}
|
|
@@ -226,58 +305,16 @@ export class ThreadSupervisor {
|
|
|
226
305
|
if (queue.hasOutstandingWork()) {
|
|
227
306
|
logger.system(`[supervisor] Recovered pending work for thread=${threadId}`);
|
|
228
307
|
}
|
|
229
|
-
else if (this.hasRunnableTasks(threadId)) {
|
|
230
|
-
logger.system(`[supervisor] Recovered runnable tasks for thread=${threadId}`);
|
|
231
|
-
}
|
|
232
308
|
}
|
|
233
309
|
this.schedulePendingThreads();
|
|
234
310
|
}
|
|
235
|
-
/**
|
|
236
|
-
* Whether a thread without queued messages still has a task that may
|
|
237
|
-
* start (#1996 PR 5). Its worker queues the resume itself once it runs:
|
|
238
|
-
* the supervisor only starts it, it never writes the thread's task store.
|
|
239
|
-
* By design (owner 10-03): a task that is not due yet is not counted;
|
|
240
|
-
* its due or unblock wake comes from the existing calendar / event-trigger
|
|
241
|
-
* messages to the thread. Thread tasks never schedule wakes.
|
|
242
|
-
* TODO: a worker started this way whose first pass queues nothing (gated)
|
|
243
|
-
* has no idle timer until its first turn, so it holds a slot until evicted.
|
|
244
|
-
*/
|
|
245
|
-
hasRunnableTasks(threadId) {
|
|
246
|
-
if (!isThreadTasksEnabled(this.config))
|
|
247
|
-
return false;
|
|
248
|
-
return threadHasRunnableTasks(taskStorePathForThread(threadId), threadId);
|
|
249
|
-
}
|
|
250
|
-
/**
|
|
251
|
-
* Start persisted threads whenever a worker slot is available. Work remains
|
|
252
|
-
* in its queue.db when all slots are busy and is retried after a worker turn
|
|
253
|
-
* or exit, rather than being rejected or routed to the default thread.
|
|
254
|
-
*/
|
|
311
|
+
/** Resume durable foreground messages without allocating a worker process. */
|
|
255
312
|
schedulePendingThreads() {
|
|
256
|
-
if (this.stopped || this.
|
|
313
|
+
if (this.stopped || this.foregroundDisabled)
|
|
257
314
|
return;
|
|
258
|
-
for (const
|
|
259
|
-
if (
|
|
260
|
-
|
|
261
|
-
if (this.restartTimers.has(threadId))
|
|
262
|
-
continue;
|
|
263
|
-
if (this.recoveryExhaustedThreads.has(threadId))
|
|
264
|
-
continue;
|
|
265
|
-
const queue = this.threadQueues.get(threadId);
|
|
266
|
-
if (!queue?.hasOutstandingWork() && !this.hasRunnableTasks(threadId))
|
|
267
|
-
continue;
|
|
268
|
-
if (this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
|
|
269
|
-
const evicted = this.evictLeastRecentlyActiveIdleWorker(threadId);
|
|
270
|
-
if (!evicted && this.getActiveWorkerCount() >= this.maxConcurrentThreads)
|
|
271
|
-
break;
|
|
272
|
-
}
|
|
273
|
-
try {
|
|
274
|
-
this.spawnWorker(threadId);
|
|
275
|
-
}
|
|
276
|
-
catch (error) {
|
|
277
|
-
logger.warn(`[supervisor] Failed to recover thread=${threadId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
278
|
-
this.recoveryExhaustedThreads.add(threadId);
|
|
279
|
-
this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, []));
|
|
280
|
-
}
|
|
315
|
+
for (const [id, queue] of this.threadQueues) {
|
|
316
|
+
if (!isLabRunThreadId(id) && !isGenericExecutionThreadId(id) && queue.hasOutstandingWork())
|
|
317
|
+
this.conversations?.accept(id);
|
|
281
318
|
}
|
|
282
319
|
}
|
|
283
320
|
/**
|
|
@@ -312,6 +349,14 @@ export class ThreadSupervisor {
|
|
|
312
349
|
logger.system(`[supervisor] Global command "${msg.text.trim().split(/\s+/, 1)[0]}" forced to default thread`);
|
|
313
350
|
return false;
|
|
314
351
|
}
|
|
352
|
+
// An execution's worker takes input only through its protocol, so every
|
|
353
|
+
// input is in the ledger. A chat message must not reach it, or start one.
|
|
354
|
+
if (isGenericExecutionThreadId(threadId)) {
|
|
355
|
+
logger.warn(`[supervisor] Refusing a chat message for execution thread=${threadId}`);
|
|
356
|
+
if (!claimRejected)
|
|
357
|
+
return false;
|
|
358
|
+
throw new ExecutionThreadInputError();
|
|
359
|
+
}
|
|
315
360
|
// During shutdown, preserve known app work in its original queue.
|
|
316
361
|
if (this.stopped) {
|
|
317
362
|
// Known application work keeps its original durable queue on shutdown.
|
|
@@ -338,7 +383,7 @@ export class ThreadSupervisor {
|
|
|
338
383
|
}
|
|
339
384
|
// Thread-scoped engine switch: handled by the supervisor (write thread.json
|
|
340
385
|
// + restart worker), NOT forwarded to the worker.
|
|
341
|
-
const engine = parseEngineCommand(msg.text);
|
|
386
|
+
const engine = isLabRunThreadId(threadId) ? null : parseEngineCommand(msg.text);
|
|
342
387
|
if (engine !== null) {
|
|
343
388
|
this.switchThreadEngine(threadId, engine, msg);
|
|
344
389
|
return true;
|
|
@@ -350,74 +395,46 @@ export class ThreadSupervisor {
|
|
|
350
395
|
const labRun = isLabRunThreadId(threadId);
|
|
351
396
|
if (labRun && this.maintenancePaused) {
|
|
352
397
|
// Lab owns its retry/lease protocol; do not acknowledge work we cannot
|
|
353
|
-
// persist. The caller retries admission after this bounded step
|
|
398
|
+
// persist. The caller retries admission after this bounded step, which
|
|
399
|
+
// is preempted so the retry does not wait for a whole Dreaming batch.
|
|
400
|
+
this.onMaintenanceInput?.();
|
|
354
401
|
if (claimRejected)
|
|
355
402
|
throw new Error("Nightly maintenance is holding worker admission; retry this delivery");
|
|
356
403
|
return false;
|
|
357
404
|
}
|
|
358
|
-
|
|
405
|
+
if (labRun && !this.workers.has(threadId) && this.labWriterBlocksAdmission(threadId, msg)) {
|
|
406
|
+
// One writer per Lab workspace: a previous run of this Lab is still
|
|
407
|
+
// stopping. Reject explicitly; the caller retries once it has stopped.
|
|
408
|
+
if (claimRejected)
|
|
409
|
+
throw new Error("A previous run of this Lab is still stopping; retry this delivery");
|
|
410
|
+
return false;
|
|
411
|
+
}
|
|
359
412
|
if (!labRun) {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
this.crashCounts.delete(threadId);
|
|
413
|
+
this.getThreadQueue(threadId, true).enqueue(msg);
|
|
414
|
+
if (this.maintenancePaused && isOwnerInitiatedMessage(msg))
|
|
415
|
+
this.onMaintenanceInput?.();
|
|
416
|
+
if (!this.foregroundDisabled) {
|
|
417
|
+
this.conversations?.accept(threadId, msg);
|
|
366
418
|
}
|
|
367
|
-
}
|
|
368
|
-
if (this.maintenancePaused) {
|
|
369
|
-
this.maintenanceMessages.push({ msg, threadId });
|
|
370
419
|
return true;
|
|
371
420
|
}
|
|
372
421
|
let state = this.workers.get(threadId);
|
|
373
422
|
if (state?.shuttingDown) {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
|
|
378
|
-
return true;
|
|
379
|
-
}
|
|
380
|
-
if (!labRun && this.restartTimers.has(threadId)) {
|
|
381
|
-
logger.debug(`[supervisor] Worker restart already scheduled for thread=${threadId}`);
|
|
382
|
-
return true;
|
|
383
|
-
}
|
|
384
|
-
if (!labRun && this.recoveryExhaustedThreads.has(threadId) && !durableInserted) {
|
|
385
|
-
logger.debug(`[supervisor] Duplicate work remains dormant after recovery exhaustion: thread=${threadId}`);
|
|
386
|
-
return true;
|
|
423
|
+
if (claimRejected)
|
|
424
|
+
throw new Error("The Lab executor is stopping; retry this delivery");
|
|
425
|
+
return false;
|
|
387
426
|
}
|
|
388
427
|
if (!state) {
|
|
389
|
-
// Check concurrency limit before spawning. Idle workers are only a
|
|
390
|
-
// keep-warm optimization: they must not block a new logical thread from
|
|
391
|
-
// making progress. If the process pool is full, evict the least-recently
|
|
392
|
-
// active idle worker first, and reject only when every slot is busy (or
|
|
393
|
-
// still spawning/shutting down).
|
|
394
428
|
if (this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
// LabRun has its own Fin-app lease and is intentionally not stored
|
|
399
|
-
// in the ordinary thread queue. Preserve the pre-existing explicit
|
|
400
|
-
// failure contract instead of claiming work that was never saved.
|
|
401
|
-
logger.warn(`[supervisor] Concurrency limit reached (${this.maxConcurrentThreads}), rejecting LabRun thread=${threadId}`);
|
|
402
|
-
deliverReplyToMessage(this.channelManager, msg, `⚠️ Thread concurrency limit reached (${this.maxConcurrentThreads} active). Please try again later.`).catch(() => { });
|
|
403
|
-
return claimRejected;
|
|
404
|
-
}
|
|
405
|
-
logger.info(`[supervisor] Concurrency limit reached (${this.maxConcurrentThreads}); keeping thread=${threadId} queued`);
|
|
406
|
-
this.publishSyntheticActivity(threadId, "queued", "Waiting for a worker slot", [msg]);
|
|
407
|
-
// The message is already durably accepted in queue.db. Do not leak
|
|
408
|
-
// it into the default thread or ask the provider to retry.
|
|
409
|
-
return true;
|
|
410
|
-
}
|
|
429
|
+
deliverReplyToMessage(this.channelManager, msg, `⚠️ Worker capacity reached (${this.maxConcurrentThreads} active). Please try again later.`)
|
|
430
|
+
.catch(() => { });
|
|
431
|
+
return claimRejected;
|
|
411
432
|
}
|
|
412
433
|
state = this.spawnWorker(threadId, { bootstrapMessage: msg });
|
|
413
434
|
}
|
|
414
|
-
// Worker still spawning: buffer the message; flushed on `ready`.
|
|
415
435
|
if (!state.ready) {
|
|
416
|
-
logger.debug(`[supervisor] Worker spawning, buffering message for thread=${threadId}`);
|
|
417
436
|
const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
|
|
418
437
|
state.pendingMessages.push(msg);
|
|
419
|
-
// Check all pending messages: only publish status if at least one is owner-initiated
|
|
420
|
-
this.publishSyntheticActivity(threadId, "queued", "Starting thread worker", state.pendingMessages);
|
|
421
438
|
this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
|
|
422
439
|
return true;
|
|
423
440
|
}
|
|
@@ -425,27 +442,11 @@ export class ThreadSupervisor {
|
|
|
425
442
|
return true;
|
|
426
443
|
}
|
|
427
444
|
getActiveWorkerCount() {
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
return count;
|
|
434
|
-
}
|
|
435
|
-
evictLeastRecentlyActiveIdleWorker(requestedThreadId) {
|
|
436
|
-
let candidate = null;
|
|
437
|
-
for (const state of this.workers.values()) {
|
|
438
|
-
if (!state.ready || state.busy || state.shuttingDown)
|
|
439
|
-
continue;
|
|
440
|
-
if (!candidate || Date.parse(state.lastActivityAt) < Date.parse(candidate.lastActivityAt)) {
|
|
441
|
-
candidate = state;
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
if (!candidate)
|
|
445
|
-
return false;
|
|
446
|
-
logger.system(`[supervisor] Evicting idle worker thread=${candidate.threadId} to make room for thread=${requestedThreadId}`);
|
|
447
|
-
this.shutdownWorker(candidate.threadId, "idle-evicted");
|
|
448
|
-
return true;
|
|
445
|
+
// Persisted process records include groups whose leader exited but whose
|
|
446
|
+
// stop has not been proved. Neither they nor stopping workers free a slot.
|
|
447
|
+
const untracked = this.executionStore?.liveExecutions()
|
|
448
|
+
.filter((record) => !this.workers.has(record.executionId)).length ?? this.pendingGroupReaps.size;
|
|
449
|
+
return this.workers.size + untracked;
|
|
449
450
|
}
|
|
450
451
|
/**
|
|
451
452
|
* Switch a thread's engine: persist the model/provider into thread.json, then
|
|
@@ -464,6 +465,7 @@ export class ThreadSupervisor {
|
|
|
464
465
|
return;
|
|
465
466
|
}
|
|
466
467
|
// Restart the running worker so it re-reads thread.json on next spawn.
|
|
468
|
+
void this.conversations?.stop(threadId);
|
|
467
469
|
const state = this.workers.get(threadId);
|
|
468
470
|
if (state) {
|
|
469
471
|
this.shutdownWorker(threadId, "engine-switch");
|
|
@@ -472,10 +474,370 @@ export class ThreadSupervisor {
|
|
|
472
474
|
deliverReplyToMessage(this.channelManager, msg, `✅ Thread engine switched to ${engine}. The next message will start a fresh session with a summary of your previous work.`)
|
|
473
475
|
.catch(() => { });
|
|
474
476
|
}
|
|
477
|
+
/** Start admitting and delivering internal executions (#2805 1c). */
|
|
478
|
+
startExecutions() {
|
|
479
|
+
this.executionScheduler?.start();
|
|
480
|
+
}
|
|
481
|
+
/** Foreground turns have their own admission pool; execution slots are unrelated. */
|
|
482
|
+
acquireConversationTurn(signal) {
|
|
483
|
+
return this.conversations?.acquireTurn(signal) ?? Promise.resolve(() => undefined);
|
|
484
|
+
}
|
|
485
|
+
handleConversationEvent(conversationId, event, reply) {
|
|
486
|
+
if (event.type === "reply") {
|
|
487
|
+
if (!isDeliveryTarget(event.target)
|
|
488
|
+
|| (event.ownerEscalation === true && !this.isOwnerEscalationTarget(event.target))) {
|
|
489
|
+
if (event.replyId)
|
|
490
|
+
reply({ type: "reply-result", replyId: event.replyId, ok: false, errorMessage: "Invalid conversation reply target" });
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
void this.channelManager.deliver(event.target, event.message, event.attachments, {
|
|
494
|
+
businessThreadId: event.ownerEscalation ? DEFAULT_THREAD_ID : conversationId,
|
|
495
|
+
...(event.kind ? { kind: event.kind } : {}),
|
|
496
|
+
}).then(() => {
|
|
497
|
+
this.replyEscalation.success(conversationId);
|
|
498
|
+
if (event.replyId)
|
|
499
|
+
reply({ type: "reply-result", replyId: event.replyId, ok: true });
|
|
500
|
+
}).catch((error) => {
|
|
501
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
502
|
+
if (event.replyId)
|
|
503
|
+
reply({ type: "reply-result", replyId: event.replyId, ok: false, errorMessage: message });
|
|
504
|
+
if (this.replyEscalation.failure(conversationId))
|
|
505
|
+
void this.alertOwnerOfThreadDeliveryFailure(conversationId, event.target, message);
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
else if (event.type === "typing") {
|
|
509
|
+
if (isDeliveryTarget(event.target))
|
|
510
|
+
void this.channelManager.sendTyping(event.target).catch(() => { });
|
|
511
|
+
}
|
|
512
|
+
else if (event.type === "status" && event.activity) {
|
|
513
|
+
this.latestActivities.set(conversationId, event.activity);
|
|
514
|
+
this.channelManager.publishThreadActivity(conversationId, event.activity, this.agentInstanceId);
|
|
515
|
+
}
|
|
516
|
+
else if (event.type === "feedback") {
|
|
517
|
+
const persisted = this.feedbackOutbox.enqueue(event.feedback) ?? this.feedbackOutbox.getPersisted(event.feedback.id);
|
|
518
|
+
if (persisted)
|
|
519
|
+
this.onFeedback?.(persisted);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* The execution API for trusted in-process callers. Submissions are
|
|
524
|
+
* accepted only after the command, record and snapshot are durable.
|
|
525
|
+
*/
|
|
526
|
+
executionService() {
|
|
527
|
+
const store = this.executionStore;
|
|
528
|
+
if (!store)
|
|
529
|
+
return null;
|
|
530
|
+
return {
|
|
531
|
+
atomic: (operation) => store.atomic(operation),
|
|
532
|
+
attemptsForTask: (taskId, scope) => store.attemptsForTask(taskId, scope),
|
|
533
|
+
submit: (envelope, input) => {
|
|
534
|
+
const executionId = `exec-${randomUUID()}`;
|
|
535
|
+
const outcome = store.submit(envelope, {
|
|
536
|
+
...input,
|
|
537
|
+
executionId,
|
|
538
|
+
snapshot: this.signExecutionSnapshotFor(executionId, input.budget, envelope.caller),
|
|
539
|
+
});
|
|
540
|
+
if (outcome.kind === "ok" && !outcome.replay) {
|
|
541
|
+
if (this.maintenancePaused)
|
|
542
|
+
this.onMaintenanceInput?.();
|
|
543
|
+
this.executionScheduler?.requestAdmission();
|
|
544
|
+
}
|
|
545
|
+
return outcome;
|
|
546
|
+
},
|
|
547
|
+
sendInput: (envelope, input) => {
|
|
548
|
+
const outcome = store.sendInput(envelope, input);
|
|
549
|
+
if (outcome.kind === "ok" && !outcome.replay) {
|
|
550
|
+
const record = store.get(input.executionId);
|
|
551
|
+
if (record)
|
|
552
|
+
this.notifyExecutionChanged(record);
|
|
553
|
+
this.forwardExecutionInput(input.executionId);
|
|
554
|
+
}
|
|
555
|
+
return outcome;
|
|
556
|
+
},
|
|
557
|
+
cancel: (envelope, input) => {
|
|
558
|
+
const outcome = store.cancel(envelope, input);
|
|
559
|
+
if (outcome.kind === "ok") {
|
|
560
|
+
const record = store.get(input.executionId);
|
|
561
|
+
if (record)
|
|
562
|
+
this.notifyExecutionChanged(record);
|
|
563
|
+
}
|
|
564
|
+
return outcome;
|
|
565
|
+
},
|
|
566
|
+
get: (executionId) => store.get(executionId),
|
|
567
|
+
latestForTask: (taskId, scope) => store.latestForTask(taskId, scope),
|
|
568
|
+
eventsAfter: (executionId, afterSeq) => store.eventsAfter(executionId, afterSeq),
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* The trusted maintenance entry (#2805 stage 2). Only the maintenance owner,
|
|
573
|
+
* while holding the pause it acquired, may submit; that one execution is the
|
|
574
|
+
* only work admitted until the hold is released. The caller and scope are
|
|
575
|
+
* fixed here, never taken from the payload.
|
|
576
|
+
*/
|
|
577
|
+
submitMaintenanceExecution(input) {
|
|
578
|
+
const store = this.executionStore;
|
|
579
|
+
if (!store || !this.executionScheduler)
|
|
580
|
+
throw new Error("Internal executions are unavailable");
|
|
581
|
+
if (!this.maintenancePaused)
|
|
582
|
+
throw new Error("Maintenance does not hold worker admission");
|
|
583
|
+
if (this.maintenanceExecutionId)
|
|
584
|
+
throw new Error("A maintenance execution is already registered");
|
|
585
|
+
const executionId = `exec-${randomUUID()}`;
|
|
586
|
+
const outcome = store.submit(MAINTENANCE_ENVELOPE(`maintenance-${input.taskId}`), {
|
|
587
|
+
taskId: input.taskId,
|
|
588
|
+
payload: input.payload,
|
|
589
|
+
executionId,
|
|
590
|
+
snapshot: this.signExecutionSnapshotFor(executionId, input.budget),
|
|
591
|
+
});
|
|
592
|
+
if (outcome.kind !== "ok")
|
|
593
|
+
throw new Error(`Maintenance execution was not accepted: ${outcome.reason}`);
|
|
594
|
+
this.maintenanceExecutionId = outcome.value.executionId;
|
|
595
|
+
this.executionScheduler.requestAdmission();
|
|
596
|
+
const record = store.get(outcome.value.executionId);
|
|
597
|
+
if (!record)
|
|
598
|
+
throw new Error("Maintenance execution disappeared after submit");
|
|
599
|
+
return record;
|
|
600
|
+
}
|
|
601
|
+
/** Preempt the maintenance execution: cancel it; it ends only after its executor stops. */
|
|
602
|
+
cancelMaintenanceExecution(reason) {
|
|
603
|
+
const store = this.executionStore;
|
|
604
|
+
const executionId = this.maintenanceExecutionId;
|
|
605
|
+
if (!store || !executionId)
|
|
606
|
+
return;
|
|
607
|
+
const outcome = store.cancel(MAINTENANCE_ENVELOPE(`maintenance-cancel-${randomUUID()}`), { executionId, reason });
|
|
608
|
+
const record = store.get(executionId);
|
|
609
|
+
if (outcome.kind === "ok" && record)
|
|
610
|
+
this.notifyExecutionChanged(record);
|
|
611
|
+
}
|
|
612
|
+
/**
|
|
613
|
+
* While maintenance holds admission only its registered execution starts.
|
|
614
|
+
* A maintenance execution starts only under that hold, never on its own.
|
|
615
|
+
*/
|
|
616
|
+
admissionSlotsFor(record) {
|
|
617
|
+
if (this.stopped)
|
|
618
|
+
return 0;
|
|
619
|
+
const maintenance = record.caller.kind === "nightly";
|
|
620
|
+
const admitted = maintenance
|
|
621
|
+
? this.maintenancePaused && record.executionId === this.maintenanceExecutionId
|
|
622
|
+
: !this.maintenancePaused;
|
|
623
|
+
if (!admitted)
|
|
624
|
+
return 0;
|
|
625
|
+
const parent = record.payload.parentTaskId ?? record.taskId;
|
|
626
|
+
const perTask = this.config.workers?.maxConcurrentPerTask ?? this.maxConcurrentThreads;
|
|
627
|
+
const live = this.executionStore?.liveExecutions() ?? [];
|
|
628
|
+
if (live.filter((item) => (item.payload.parentTaskId ?? item.taskId) === parent).length >= perTask)
|
|
629
|
+
return 0;
|
|
630
|
+
return this.maxConcurrentThreads - this.getActiveWorkerCount();
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* After its outcome is applied, remove what a finished maintenance
|
|
634
|
+
* execution holds of memory and conversation text: its worker directory,
|
|
635
|
+
* its transcripts, and the report content in the execution ledger. The
|
|
636
|
+
* Dreaming edit log and backups are the record that is kept.
|
|
637
|
+
*/
|
|
638
|
+
discardMaintenanceExecutionContent(executionId) {
|
|
639
|
+
const record = this.executionStore?.get(executionId);
|
|
640
|
+
if (record?.caller.kind !== "nightly" || !isTerminalExecutionState(record.state))
|
|
641
|
+
return;
|
|
642
|
+
this.executionStore?.redactResultContent(executionId);
|
|
643
|
+
const threadDir = getThreadDir(genericExecutionThreadId(executionId));
|
|
644
|
+
try {
|
|
645
|
+
fs.rmSync(getClaudeTranscriptDirForProfileDir(threadDir), { recursive: true, force: true });
|
|
646
|
+
fs.rmSync(threadDir, { recursive: true, force: true });
|
|
647
|
+
}
|
|
648
|
+
catch (error) {
|
|
649
|
+
logger.warn(`[supervisor] Could not remove files of maintenance execution ${executionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
/** Cancel a maintenance execution left queued by a previous host; nothing else. */
|
|
653
|
+
cancelStaleMaintenanceExecution(executionId) {
|
|
654
|
+
const store = this.executionStore;
|
|
655
|
+
const record = store?.get(executionId) ?? null;
|
|
656
|
+
if (!store || record?.caller.kind !== "nightly" || record.state !== "queued")
|
|
657
|
+
return record;
|
|
658
|
+
store.cancel(MAINTENANCE_ENVELOPE(`maintenance-cancel-${randomUUID()}`), { executionId, reason: "left by a previous host" });
|
|
659
|
+
return store.get(executionId);
|
|
660
|
+
}
|
|
661
|
+
getExecutionRecord(executionId) {
|
|
662
|
+
return this.executionStore?.get(executionId) ?? null;
|
|
663
|
+
}
|
|
664
|
+
latestExecutionForTask(taskId) {
|
|
665
|
+
return this.executionStore?.latestForTask(taskId, MAINTENANCE_SCOPE) ?? null;
|
|
666
|
+
}
|
|
667
|
+
/** Observe every committed execution event (live streams such as TUI SSE). */
|
|
668
|
+
onExecutionEvent(listener) {
|
|
669
|
+
return this.executionStore?.onEvent(listener) ?? (() => undefined);
|
|
670
|
+
}
|
|
671
|
+
/** The time budget frozen in an execution's stored snapshot. */
|
|
672
|
+
timeBudgetOf(rawSnapshot, executionId) {
|
|
673
|
+
try {
|
|
674
|
+
return readStoredExecutionSnapshot(rawSnapshot, executionId).budget.maxDurationMs;
|
|
675
|
+
}
|
|
676
|
+
catch (error) {
|
|
677
|
+
logger.warn(`[supervisor] No host deadline for execution=${executionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
678
|
+
return null;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
/** A command changed an execution: admit queued work and stop cancelled executors. */
|
|
682
|
+
notifyExecutionChanged(record) {
|
|
683
|
+
if (!this.executionScheduler)
|
|
684
|
+
return;
|
|
685
|
+
if (record.state === "cancel_requested")
|
|
686
|
+
this.executionScheduler.requestStop(record.executionId, "cancel_requested");
|
|
687
|
+
if (record.state === "queued")
|
|
688
|
+
this.executionScheduler.requestAdmission();
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* Give a LabRun the same host execution record as other executions
|
|
692
|
+
* (#2805 1d). LabRun keeps its own Fin-App lease and explicit-rejection
|
|
693
|
+
* capacity contract, so it is admitted and started here in one step rather
|
|
694
|
+
* than queued. A crash-free restart of the host finds the record and its
|
|
695
|
+
* executor pid, and can prove that executor stopped before any retry runs.
|
|
696
|
+
*/
|
|
697
|
+
recordLabExecution(state, envelope, snapshot) {
|
|
698
|
+
const store = this.executionStore;
|
|
699
|
+
const pid = state.process.pid;
|
|
700
|
+
if (!store || pid === undefined)
|
|
701
|
+
return;
|
|
702
|
+
const { policy } = verifyLabRuntimePolicyEnvelope(envelope, this.labPolicyVerifyKey(), state.threadId);
|
|
703
|
+
try {
|
|
704
|
+
const record = store.admitAndStart({
|
|
705
|
+
executionId: state.threadId,
|
|
706
|
+
taskId: `lab-run-${policy.runId}`,
|
|
707
|
+
scope: policy.tenantId,
|
|
708
|
+
caller: { kind: "lab", ref: policy.runId },
|
|
709
|
+
payload: { goal: `LabRun ${policy.runId} for Lab ${policy.labId}`, workspaceRef: `lab:${policy.labId}` },
|
|
710
|
+
snapshot: JSON.stringify(snapshot),
|
|
711
|
+
process: { pid, processStart: new Date().toISOString() },
|
|
712
|
+
});
|
|
713
|
+
state.labExecution = { executionId: record.executionId, generation: record.workerInstanceId };
|
|
714
|
+
}
|
|
715
|
+
catch (error) {
|
|
716
|
+
// The Lab lease protocol remains authoritative; a ledger failure is logged, not fatal.
|
|
717
|
+
logger.warn(`[supervisor] Could not record LabRun execution ${state.threadId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* The LabRun executor exited. Its terminal facts already went to Fin-App
|
|
722
|
+
* through lab_run_control; the ledger records that the writer is gone so the
|
|
723
|
+
* shared Lab workspace may admit the next run. Lab never auto-resumes: a
|
|
724
|
+
* retry is a new LabRun.
|
|
725
|
+
*/
|
|
726
|
+
confirmLabExecutionStopped(execution, code) {
|
|
727
|
+
const store = this.executionStore;
|
|
728
|
+
if (!store)
|
|
729
|
+
return;
|
|
730
|
+
const record = store.get(execution.executionId);
|
|
731
|
+
if (record?.state === "running" && record.result === null) {
|
|
732
|
+
store.completeExecution(execution.executionId, execution.generation, code === 0
|
|
733
|
+
? { status: "succeeded", processedInputSeq: record.inputSeq, summary: "LabRun executor finished; terminal facts were reported to Fin-App" }
|
|
734
|
+
: { status: "failed", processedInputSeq: record.inputSeq, error: { code: "executor_exit", message: `LabRun executor exited with code ${String(code)}` } });
|
|
735
|
+
}
|
|
736
|
+
store.confirmStopped(execution.executionId, execution.generation, {
|
|
737
|
+
status: "failed",
|
|
738
|
+
error: { code: "executor_exit", message: "LabRun executor stopped" },
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
/**
|
|
742
|
+
* Fin-App cancelled a LabRun, or its lease is gone. Record the request and
|
|
743
|
+
* stop the executor; the record turns `cancelled` only after the process exits.
|
|
744
|
+
*/
|
|
745
|
+
cancelLabRun(runId, reason) {
|
|
746
|
+
const threadId = `fin-lab-run-${runId}`;
|
|
747
|
+
const record = this.executionStore?.requestHostCancel(threadId, reason);
|
|
748
|
+
const running = this.workers.has(threadId);
|
|
749
|
+
if (running)
|
|
750
|
+
this.shutdownWorker(threadId, `lab-cancel: ${reason}`.slice(0, 200));
|
|
751
|
+
return record !== null || running;
|
|
752
|
+
}
|
|
753
|
+
/** A new LabRun may not start while another run of the same Lab still has a live executor. */
|
|
754
|
+
labWriterBlocksAdmission(threadId, msg) {
|
|
755
|
+
let labId;
|
|
756
|
+
let runId;
|
|
757
|
+
try {
|
|
758
|
+
const envelope = this.authorizeLabRuntimePolicy(threadId, msg.meta?.[LAB_RUNTIME_POLICY_META_KEY]);
|
|
759
|
+
({ labId, runId } = verifyLabRuntimePolicyEnvelope(envelope, this.labPolicyVerifyKey(), threadId).policy);
|
|
760
|
+
}
|
|
761
|
+
catch {
|
|
762
|
+
return false; // Policy errors are reported by the normal spawn path.
|
|
763
|
+
}
|
|
764
|
+
return this.hasLiveLabWriter(labId, runId);
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Whether a Lab workspace may admit a new writer. Both this host's live
|
|
768
|
+
* workers and the ledger count: a LabRun that is running or stopping, or
|
|
769
|
+
* whose executor (including survivors in its process group) is not yet
|
|
770
|
+
* confirmed stopped, is still a writer, also across a host restart.
|
|
771
|
+
*/
|
|
772
|
+
hasLiveLabWriter(labId, exceptRunId) {
|
|
773
|
+
for (const state of this.workers.values()) {
|
|
774
|
+
if (!isLabRunThreadId(state.threadId) || !state.labRuntimePolicy)
|
|
775
|
+
continue;
|
|
776
|
+
const { policy } = verifyLabRuntimePolicyEnvelope(state.labRuntimePolicy, this.labPolicyVerifyKey(), state.threadId);
|
|
777
|
+
if (policy.labId === labId && policy.runId !== exceptRunId)
|
|
778
|
+
return true;
|
|
779
|
+
}
|
|
780
|
+
return this.executionStore?.hasLiveWriter(`lab:${labId}`, exceptRunId === undefined ? undefined : `fin-lab-run-${exceptRunId}`) ?? false;
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Forward accepted input to a running executor; a stopped one reads it on
|
|
784
|
+
* resume. An executor that is stopping gets nothing: it would consume the
|
|
785
|
+
* input and exit before its model could see it.
|
|
786
|
+
*/
|
|
787
|
+
forwardExecutionInput(executionId) {
|
|
788
|
+
const store = this.executionStore;
|
|
789
|
+
const state = this.workers.get(genericExecutionThreadId(executionId));
|
|
790
|
+
if (!store || state?.execution?.executionId !== executionId || !state.ready || state.shuttingDown || !state.process.connected)
|
|
791
|
+
return;
|
|
792
|
+
const record = store.get(executionId);
|
|
793
|
+
if (!record)
|
|
794
|
+
return;
|
|
795
|
+
for (const input of store.pendingInputs(executionId, record.processedInputSeq)) {
|
|
796
|
+
state.process.send({
|
|
797
|
+
type: "execution-input",
|
|
798
|
+
executionId,
|
|
799
|
+
inputSeq: input.inputSeq,
|
|
800
|
+
questionId: input.questionId,
|
|
801
|
+
text: input.text,
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Fork the executor for an admitted generic execution. It shares the
|
|
807
|
+
* ordinary worker pool and its restrictions; its snapshot is frozen here.
|
|
808
|
+
*/
|
|
809
|
+
spawnExecutionWorker(record) {
|
|
810
|
+
const threadId = genericExecutionThreadId(record.executionId);
|
|
811
|
+
// The ledger records what was admitted. Re-sign it with this host's key:
|
|
812
|
+
// the stored signature may be from a host process that no longer exists.
|
|
813
|
+
let snapshot;
|
|
814
|
+
try {
|
|
815
|
+
snapshot = JSON.stringify(signExecutionSnapshot(readStoredExecutionSnapshot(record.snapshot, record.executionId), this.authority().snapshot));
|
|
816
|
+
}
|
|
817
|
+
catch (error) {
|
|
818
|
+
throw new ExecutorRefusedError(`Execution ${record.executionId} has an unusable snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
819
|
+
}
|
|
820
|
+
ensureThreadApplication(threadId, "gru");
|
|
821
|
+
const state = this.spawnWorker(threadId, {
|
|
822
|
+
execution: { executionId: record.executionId, snapshot },
|
|
823
|
+
});
|
|
824
|
+
const pid = state.process.pid;
|
|
825
|
+
if (pid === undefined)
|
|
826
|
+
throw new Error(`Execution worker for ${record.executionId} did not start`);
|
|
827
|
+
state.execution = { executionId: record.executionId, generation: record.workerInstanceId + 1 };
|
|
828
|
+
if (record.caller.kind === "nightly")
|
|
829
|
+
state.privateActivity = true;
|
|
830
|
+
if (record.caller.kind === "general")
|
|
831
|
+
state.foreman = true;
|
|
832
|
+
return { pid, processStart: new Date().toISOString() };
|
|
833
|
+
}
|
|
475
834
|
/**
|
|
476
835
|
* Spawn a worker process for the given threadId.
|
|
477
836
|
*/
|
|
478
837
|
spawnWorker(threadId, options = {}) {
|
|
838
|
+
if (!options.execution && !isLabRunThreadId(threadId)) {
|
|
839
|
+
throw new Error("Only admitted Tasks and LabRuns may start worker processes");
|
|
840
|
+
}
|
|
479
841
|
logger.system(`[supervisor] Spawning worker for thread=${threadId}`);
|
|
480
842
|
const threadMeta = ensureThreadApplication(threadId, options.bootstrapMessage?.app);
|
|
481
843
|
threadApplication(threadId, this.config);
|
|
@@ -492,9 +854,26 @@ export class ThreadSupervisor {
|
|
|
492
854
|
if (labRuntimePolicy) {
|
|
493
855
|
// Resolve the trusted tenant/Lab identity before fork. This catches a
|
|
494
856
|
// symlink or tenant-marker collision even if the worker entry never runs.
|
|
495
|
-
ensureLabWorkspace(verifyLabRuntimePolicyEnvelope(labRuntimePolicy, this.
|
|
857
|
+
ensureLabWorkspace(verifyLabRuntimePolicyEnvelope(labRuntimePolicy, this.labPolicyVerifyKey(), threadId), getSharedProfileDir());
|
|
496
858
|
workerEnv[LAB_RUNTIME_POLICY_ENV] = JSON.stringify(labRuntimePolicy);
|
|
497
|
-
|
|
859
|
+
}
|
|
860
|
+
// Workers verify with public keys only; no signing key leaves this host.
|
|
861
|
+
// (fork skips undefined values, so an inherited copy is never passed on.)
|
|
862
|
+
delete workerEnv.VISIONCLAW_LAB_POLICY_SECRET;
|
|
863
|
+
const internalExecution = labRuntimePolicy !== undefined || options.execution !== undefined;
|
|
864
|
+
workerEnv[EXECUTION_VERIFY_KEYS_ENV] = internalExecution
|
|
865
|
+
? JSON.stringify(verifyKeysOf(this.authority()))
|
|
866
|
+
: undefined;
|
|
867
|
+
// Freeze the execution snapshot once, at admission. Respawns of the same
|
|
868
|
+
// run reuse it, so editable thread.json or routing changes cannot alter it.
|
|
869
|
+
const executionSnapshot = labRuntimePolicy
|
|
870
|
+
? options.trustedExecutionSnapshot ?? this.admitExecutionSnapshot(threadId)
|
|
871
|
+
: undefined;
|
|
872
|
+
if (executionSnapshot)
|
|
873
|
+
workerEnv[EXECUTION_SNAPSHOT_ENV] = JSON.stringify(executionSnapshot);
|
|
874
|
+
if (options.execution) {
|
|
875
|
+
workerEnv[EXECUTION_SNAPSHOT_ENV] = options.execution.snapshot;
|
|
876
|
+
workerEnv.VISIONCLAW_EXECUTION_ID = options.execution.executionId;
|
|
498
877
|
}
|
|
499
878
|
if (this.desktopRuntimeResolution) {
|
|
500
879
|
workerEnv[WORKER_DESKTOP_RUNTIME_ENV] = serializeDesktopRuntimeResolution(this.desktopRuntimeResolution);
|
|
@@ -506,6 +885,9 @@ export class ThreadSupervisor {
|
|
|
506
885
|
stdio: ["inherit", "inherit", "inherit", "ipc"],
|
|
507
886
|
cwd: process.cwd(),
|
|
508
887
|
env: workerEnv,
|
|
888
|
+
// An internal executor leads its own process group, and its commands
|
|
889
|
+
// stay in it, so a stop can reach every process it started.
|
|
890
|
+
...(internalExecution && process.platform !== "win32" ? { detached: true } : {}),
|
|
509
891
|
...(this.workerExecPath ? { execPath: this.workerExecPath } : {}),
|
|
510
892
|
});
|
|
511
893
|
const state = {
|
|
@@ -516,11 +898,14 @@ export class ThreadSupervisor {
|
|
|
516
898
|
busy: false,
|
|
517
899
|
taskDescription: "spawning",
|
|
518
900
|
lastActivityAt: new Date().toISOString(),
|
|
519
|
-
idleTimer: null,
|
|
520
901
|
shuttingDown: false,
|
|
521
902
|
pendingMessages: [],
|
|
903
|
+
internalExecution,
|
|
522
904
|
...(labRuntimePolicy ? { labRuntimePolicy: this.freezeLabRuntimePolicy(labRuntimePolicy) } : {}),
|
|
905
|
+
...(executionSnapshot ? { executionSnapshot: Object.freeze({ ...executionSnapshot }) } : {}),
|
|
523
906
|
};
|
|
907
|
+
if (labRuntimePolicy && executionSnapshot)
|
|
908
|
+
this.recordLabExecution(state, labRuntimePolicy, executionSnapshot);
|
|
524
909
|
this.workers.set(threadId, state);
|
|
525
910
|
child.on("message", (data) => {
|
|
526
911
|
this.handleWorkerMessage(threadId, data);
|
|
@@ -547,6 +932,13 @@ export class ThreadSupervisor {
|
|
|
547
932
|
case "lab-runtime-bootstrap-failed":
|
|
548
933
|
this.handleLabRuntimeBootstrapFailed(state, data);
|
|
549
934
|
break;
|
|
935
|
+
case "lab-runtime-lease-lost":
|
|
936
|
+
if (!isLabRunThreadId(state.threadId) || data.threadId !== state.threadId)
|
|
937
|
+
break;
|
|
938
|
+
logger.warn(`[supervisor] LabRun lease lost: thread=${state.threadId} reason=${data.errorMessage}`);
|
|
939
|
+
this.executionStore?.requestHostCancel(state.threadId, `lease lost: ${data.errorMessage}`);
|
|
940
|
+
this.shutdownWorker(state.threadId, "lab-lease-lost");
|
|
941
|
+
break;
|
|
550
942
|
case "ready":
|
|
551
943
|
this.handleReady(state, data);
|
|
552
944
|
break;
|
|
@@ -554,28 +946,101 @@ export class ThreadSupervisor {
|
|
|
554
946
|
this.handleReply(state, data);
|
|
555
947
|
break;
|
|
556
948
|
case "typing":
|
|
557
|
-
this.handleTyping(state, data);
|
|
558
949
|
break;
|
|
559
950
|
case "status":
|
|
560
951
|
this.handleStatus(state, data);
|
|
561
952
|
break;
|
|
562
953
|
case "feedback":
|
|
563
|
-
this.handleFeedback(state, data);
|
|
564
954
|
break;
|
|
565
955
|
case "turn-complete":
|
|
566
|
-
|
|
956
|
+
break;
|
|
957
|
+
case "execution-bootstrap-ready":
|
|
958
|
+
this.handleExecutionBootstrap(state);
|
|
959
|
+
break;
|
|
960
|
+
case "execution-result":
|
|
961
|
+
case "execution-progress":
|
|
962
|
+
case "execution-question":
|
|
963
|
+
case "execution-input-applied":
|
|
964
|
+
case "execution-tool":
|
|
965
|
+
case "execution-usage":
|
|
966
|
+
if (!state.execution || !this.executionStore || !this.executionScheduler) {
|
|
967
|
+
logger.warn(`[supervisor] Ignoring execution report from non-execution thread=${threadId}`);
|
|
968
|
+
break;
|
|
969
|
+
}
|
|
970
|
+
applyExecutionReport(this.executionStore, this.executionScheduler, state.execution, data);
|
|
971
|
+
break;
|
|
972
|
+
case "foreman-request":
|
|
973
|
+
this.handleForemanRequest(state, data);
|
|
567
974
|
break;
|
|
568
975
|
}
|
|
569
976
|
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
977
|
+
/**
|
|
978
|
+
* A conversational worker's background-task command. The thread is the
|
|
979
|
+
* worker this supervisor started for it, never a value from the message;
|
|
980
|
+
* internal executions (Lab, Dreaming, Foreman workers) cannot delegate.
|
|
981
|
+
*/
|
|
982
|
+
handleForemanRequest(state, msg) {
|
|
983
|
+
const answer = (result) => {
|
|
984
|
+
if (state.shuttingDown || !state.process.connected)
|
|
985
|
+
return;
|
|
986
|
+
try {
|
|
987
|
+
state.process.send({ type: "foreman-response", requestId: msg.requestId, result });
|
|
988
|
+
}
|
|
989
|
+
catch (error) {
|
|
990
|
+
logger.warn(`[supervisor] Could not answer a background-task command from thread=${state.threadId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
991
|
+
}
|
|
992
|
+
};
|
|
993
|
+
if (typeof msg.requestId !== "string")
|
|
575
994
|
return;
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
995
|
+
if (state.internalExecution && msg.command.type === "business_read") {
|
|
996
|
+
const expected = state.execution;
|
|
997
|
+
const record = expected ? this.executionStore?.get(expected.executionId) : null;
|
|
998
|
+
if (!record || state.shuttingDown || record.state !== "running" || record.caller.kind !== "general"
|
|
999
|
+
|| record.workerInstanceId !== expected?.generation || !record.payload.businessRead || !this.handleForemanBusinessRead) {
|
|
1000
|
+
answer({ ok: false, text: "Business reads require a live authorized Foreman execution" });
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
const workspace = record.payload.workspaceRef ?? path.join(getThreadDir(state.threadId), "workspace");
|
|
1004
|
+
this.handleForemanBusinessRead(record.executionId, msg.command, workspace).then((result) => {
|
|
1005
|
+
const current = this.executionStore?.get(record.executionId);
|
|
1006
|
+
if (current?.state === "running" && current.workerInstanceId === expected.generation)
|
|
1007
|
+
answer(result);
|
|
1008
|
+
else
|
|
1009
|
+
answer({ ok: false, text: "The execution stopped before the business read completed" });
|
|
1010
|
+
}).catch((error) => answer({ ok: false, text: `Business read failed: ${error instanceof Error ? error.message : String(error)}` }));
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
answer({ ok: false, text: "Internal executions cannot delegate or control other Tasks" });
|
|
1014
|
+
}
|
|
1015
|
+
/**
|
|
1016
|
+
* The executor is listening: hand it the admitted record from the ledger.
|
|
1017
|
+
* Workers never open the ledger themselves.
|
|
1018
|
+
*/
|
|
1019
|
+
handleExecutionBootstrap(state) {
|
|
1020
|
+
const store = this.executionStore;
|
|
1021
|
+
const execution = state.execution;
|
|
1022
|
+
if (!store || !execution) {
|
|
1023
|
+
logger.warn(`[supervisor] Refusing execution bootstrap for non-execution thread=${state.threadId}`);
|
|
1024
|
+
return;
|
|
1025
|
+
}
|
|
1026
|
+
const record = store.get(execution.executionId);
|
|
1027
|
+
if (record?.workerInstanceId !== execution.generation) {
|
|
1028
|
+
logger.warn(`[supervisor] Execution ${execution.executionId} has no record for generation ${execution.generation}; stopping its executor`);
|
|
1029
|
+
this.shutdownWorker(state.threadId, "no_admitted_record");
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
1032
|
+
state.process.send({
|
|
1033
|
+
type: "execution-start",
|
|
1034
|
+
record,
|
|
1035
|
+
unfinishedTools: store.unfinishedToolCalls(record.executionId),
|
|
1036
|
+
// input_applied is a session handoff cursor, not a durable provider
|
|
1037
|
+
// receipt. A crash between that cursor and the SDK's transcript write
|
|
1038
|
+
// must not erase an accepted answer. Replay the authoritative history.
|
|
1039
|
+
pendingInputs: store.pendingInputs(record.executionId, 0),
|
|
1040
|
+
}, (error) => {
|
|
1041
|
+
if (error)
|
|
1042
|
+
logger.debug(`[supervisor] Execution start IPC unavailable for ${state.threadId}: ${error.message}`);
|
|
1043
|
+
});
|
|
579
1044
|
}
|
|
580
1045
|
handleLabRuntimeBootstrapReady(state) {
|
|
581
1046
|
if (!state.labRuntimePolicy || !isLabRunThreadId(state.threadId)) {
|
|
@@ -602,7 +1067,7 @@ export class ThreadSupervisor {
|
|
|
602
1067
|
}
|
|
603
1068
|
let policy;
|
|
604
1069
|
try {
|
|
605
|
-
policy = verifyLabRuntimePolicyEnvelope(state.labRuntimePolicy, this.
|
|
1070
|
+
policy = verifyLabRuntimePolicyEnvelope(state.labRuntimePolicy, this.labPolicyVerifyKey(), state.threadId).policy;
|
|
606
1071
|
}
|
|
607
1072
|
catch (error) {
|
|
608
1073
|
logger.warn(`[supervisor] Refusing invalid Lab bootstrap failure policy: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -640,6 +1105,9 @@ export class ThreadSupervisor {
|
|
|
640
1105
|
logger.debug(`[supervisor] Worker became ready while shutting down: thread=${state.threadId}`);
|
|
641
1106
|
return;
|
|
642
1107
|
}
|
|
1108
|
+
// Input accepted after the executor's start record was read: hand it over now.
|
|
1109
|
+
if (state.execution)
|
|
1110
|
+
this.forwardExecutionInput(state.execution.executionId);
|
|
643
1111
|
// Flush any messages buffered while the worker was spawning, in order.
|
|
644
1112
|
if (state.pendingMessages.length > 0) {
|
|
645
1113
|
const pending = state.pendingMessages;
|
|
@@ -651,43 +1119,7 @@ export class ThreadSupervisor {
|
|
|
651
1119
|
}
|
|
652
1120
|
}
|
|
653
1121
|
handleReply(state, msg) {
|
|
654
|
-
|
|
655
|
-
const errorMessage = "worker reply contained an invalid target";
|
|
656
|
-
logger.warn(`[supervisor] ${errorMessage}`);
|
|
657
|
-
this.sendReplyResult(state, msg.replyId, false, errorMessage);
|
|
658
|
-
return;
|
|
659
|
-
}
|
|
660
|
-
// A worker's sends belong to its thread, so mail adapters record provider
|
|
661
|
-
// replies against it. An owner escalation must not: the owner's answer is
|
|
662
|
-
// private to the owner's own (default) conversation. The marker is only
|
|
663
|
-
// honored for a real owner target resolved here, in the main process.
|
|
664
|
-
let businessThreadId = state.threadId;
|
|
665
|
-
if (msg.ownerEscalation === true) {
|
|
666
|
-
if (!this.isOwnerEscalationTarget(msg.target)) {
|
|
667
|
-
const errorMessage = "owner escalation target is not a bound owner target";
|
|
668
|
-
logger.warn(`[supervisor] Refused reply from thread=${state.threadId}: ${errorMessage}`);
|
|
669
|
-
this.sendReplyResult(state, msg.replyId, false, errorMessage);
|
|
670
|
-
return;
|
|
671
|
-
}
|
|
672
|
-
businessThreadId = DEFAULT_THREAD_ID;
|
|
673
|
-
}
|
|
674
|
-
logger.debug(`[supervisor] Relaying reply from channel=${msg.target.channel}`);
|
|
675
|
-
this.channelManager.deliver(msg.target, msg.message, msg.attachments, {
|
|
676
|
-
businessThreadId,
|
|
677
|
-
...(msg.kind ? { kind: msg.kind } : {}),
|
|
678
|
-
})
|
|
679
|
-
.then(() => {
|
|
680
|
-
this.replyEscalation.success(state.threadId);
|
|
681
|
-
this.sendReplyResult(state, msg.replyId, true);
|
|
682
|
-
})
|
|
683
|
-
.catch((err) => {
|
|
684
|
-
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
685
|
-
logger.warn(`[supervisor] Failed to relay reply: ${errorMessage}`);
|
|
686
|
-
this.sendReplyResult(state, msg.replyId, false, errorMessage);
|
|
687
|
-
if (this.replyEscalation.failure(state.threadId)) {
|
|
688
|
-
void this.alertOwnerOfThreadDeliveryFailure(state.threadId, msg.target, errorMessage);
|
|
689
|
-
}
|
|
690
|
-
});
|
|
1122
|
+
this.sendReplyResult(state, msg.replyId, false, "Internal worker executions cannot deliver external messages");
|
|
691
1123
|
}
|
|
692
1124
|
/** True when `target` is one of the owner's usable bound targets (no reply context). */
|
|
693
1125
|
isOwnerEscalationTarget(target) {
|
|
@@ -743,17 +1175,8 @@ export class ThreadSupervisor {
|
|
|
743
1175
|
logger.debug(`[supervisor] Could not ack reply ${replyId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
744
1176
|
}
|
|
745
1177
|
}
|
|
746
|
-
handleTyping(_state, msg) {
|
|
747
|
-
if (!isDeliveryTarget(msg.target)) {
|
|
748
|
-
logger.warn("[supervisor] Worker typing contained an invalid target");
|
|
749
|
-
return;
|
|
750
|
-
}
|
|
751
|
-
this.channelManager.sendTyping(msg.target).catch(() => {
|
|
752
|
-
// best-effort
|
|
753
|
-
});
|
|
754
|
-
}
|
|
755
1178
|
handleStatus(state, msg) {
|
|
756
|
-
if (msg.activity) {
|
|
1179
|
+
if (msg.activity && !state.privateActivity) {
|
|
757
1180
|
const current = state.activity;
|
|
758
1181
|
const known = this.latestActivities.get(state.threadId);
|
|
759
1182
|
const wrongWorker = state.workerInstanceId !== undefined
|
|
@@ -774,38 +1197,13 @@ export class ThreadSupervisor {
|
|
|
774
1197
|
// fenced projection still proves whether this worker is busy or idle.
|
|
775
1198
|
const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
|
|
776
1199
|
state.busy = msg.busy;
|
|
777
|
-
state.taskDescription = msg.taskDescription;
|
|
1200
|
+
state.taskDescription = state.privateActivity ? "nightly maintenance" : msg.taskDescription;
|
|
778
1201
|
state.lastActivityAt = msg.lastActivityAt;
|
|
779
1202
|
this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
|
|
780
|
-
if (!msg.busy)
|
|
1203
|
+
if (!msg.busy) {
|
|
781
1204
|
this.schedulePendingThreads();
|
|
782
|
-
|
|
783
|
-
handleTurnComplete(state, _msg) {
|
|
784
|
-
logger.system(`[supervisor] Turn complete from thread=${state.threadId}`);
|
|
785
|
-
// A completed turn proves the worker is healthy — reset its crash budget so
|
|
786
|
-
// sporadic crashes spread over a long lifetime don't exhaust the cap.
|
|
787
|
-
this.crashCounts.delete(state.threadId);
|
|
788
|
-
this.recoveryExhaustedThreads.delete(state.threadId);
|
|
789
|
-
// Billing is NOT handled here: the worker's own WakeCycleRunner already
|
|
790
|
-
// wrote the wake-cycle row into the shared transcript-memory.db (WAL-safe).
|
|
791
|
-
// Memory dreaming is not triggered here either: it rewrites and deletes
|
|
792
|
-
// files in the shared memories/ tree, so it runs on the deep-night sweep
|
|
793
|
-
// only (owned by NightlyMaintenance), not once per wake cycle.
|
|
794
|
-
this.armIdleTimer(state);
|
|
795
|
-
this.schedulePendingThreads();
|
|
796
|
-
}
|
|
797
|
-
armIdleTimer(state) {
|
|
798
|
-
if (state.idleTimer !== null) {
|
|
799
|
-
clearTimeout(state.idleTimer);
|
|
1205
|
+
this.executionScheduler?.requestAdmission();
|
|
800
1206
|
}
|
|
801
|
-
state.idleTimer = setTimeout(() => {
|
|
802
|
-
if (state.busy) {
|
|
803
|
-
logger.debug(`[supervisor] Worker still busy at idle timeout, extending: thread=${state.threadId}`);
|
|
804
|
-
this.armIdleTimer(state);
|
|
805
|
-
return;
|
|
806
|
-
}
|
|
807
|
-
this.shutdownWorker(state.threadId, "idle-timeout");
|
|
808
|
-
}, this.idleTimeoutMs);
|
|
809
1207
|
}
|
|
810
1208
|
handleWorkerExit(threadId, code, signal) {
|
|
811
1209
|
const state = this.workers.get(threadId);
|
|
@@ -814,11 +1212,6 @@ export class ThreadSupervisor {
|
|
|
814
1212
|
const wasReady = state.ready;
|
|
815
1213
|
logger.system(`[supervisor] Worker exited: thread=${threadId} code=${code} signal=${signal} wasReady=${wasReady}`);
|
|
816
1214
|
// Clear idle timer.
|
|
817
|
-
if (state.idleTimer !== null) {
|
|
818
|
-
clearTimeout(state.idleTimer);
|
|
819
|
-
state.idleTimer = null;
|
|
820
|
-
}
|
|
821
|
-
const pendingMessages = state.pendingMessages;
|
|
822
1215
|
if (state.busy && state.activity && !state.shuttingDown) {
|
|
823
1216
|
const now = new Date().toISOString();
|
|
824
1217
|
const failedActivity = {
|
|
@@ -840,113 +1233,51 @@ export class ThreadSupervisor {
|
|
|
840
1233
|
this.workers.delete(threadId);
|
|
841
1234
|
if (wasVisibleBusy)
|
|
842
1235
|
this.onBusyChange?.();
|
|
843
|
-
//
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
}
|
|
850
|
-
// Clean exit, or any exit during a requested shutdown (a graceful stop that
|
|
851
|
-
// ends non-zero is not a crash): don't count towards the crash cap. Respawn
|
|
852
|
-
// only when messages arrived during the shutdown window — but never after
|
|
853
|
-
// the supervisor itself has been stopped.
|
|
854
|
-
if (code === 0 || state.shuttingDown) {
|
|
855
|
-
if (code === 0)
|
|
856
|
-
this.crashCounts.delete(threadId);
|
|
857
|
-
if (pendingMessages.length > 0 && !this.stopped) {
|
|
858
|
-
this.spawnWorkerForPendingMessages(threadId, pendingMessages, "shutdown-respawn", state.labRuntimePolicy);
|
|
859
|
-
}
|
|
860
|
-
this.schedulePendingThreads();
|
|
861
|
-
return;
|
|
862
|
-
}
|
|
863
|
-
// Crash recovery: restart if crashed unexpectedly and under threshold.
|
|
864
|
-
const MAX_CRASH_COUNT = 3;
|
|
865
|
-
const nextCrashCount = wasReady ? (this.crashCounts.get(threadId) ?? 0) + 1 : this.crashCounts.get(threadId) ?? 0;
|
|
866
|
-
if (wasReady) {
|
|
867
|
-
this.crashCounts.set(threadId, nextCrashCount);
|
|
868
|
-
}
|
|
869
|
-
if (wasReady && nextCrashCount <= MAX_CRASH_COUNT && !this.stopped) {
|
|
870
|
-
logger.warn(`[supervisor] Worker crashed, restarting (attempt ${nextCrashCount}/${MAX_CRASH_COUNT}): thread=${threadId}`);
|
|
871
|
-
// Re-spawn after a brief delay.
|
|
872
|
-
const trustedLabRuntimePolicy = state.labRuntimePolicy;
|
|
873
|
-
const timer = setTimeout(() => {
|
|
874
|
-
this.restartTimers.delete(threadId);
|
|
875
|
-
if (this.stopped || this.maintenancePaused || this.workers.has(threadId))
|
|
876
|
-
return;
|
|
877
|
-
try {
|
|
878
|
-
const next = this.spawnWorker(threadId, {
|
|
879
|
-
bootstrapMessage: pendingMessages[0],
|
|
880
|
-
trustedLabRuntimePolicy,
|
|
881
|
-
});
|
|
882
|
-
this.enqueuePendingWork(next, pendingMessages);
|
|
883
|
-
}
|
|
884
|
-
catch (error) {
|
|
885
|
-
this.handleWorkerRespawnFailure(threadId, pendingMessages, "crash-restart", error);
|
|
886
|
-
}
|
|
887
|
-
}, this.crashRestartDelayMs);
|
|
888
|
-
this.restartTimers.set(threadId, timer);
|
|
1236
|
+
// Any exit frees a slot that queued executions may take.
|
|
1237
|
+
this.executionScheduler?.requestAdmission();
|
|
1238
|
+
// An internal executor's stop is confirmed only once its whole process
|
|
1239
|
+
// group is gone, so background jobs it started cannot outlive it.
|
|
1240
|
+
if (state.execution || state.labExecution) {
|
|
1241
|
+
this.confirmExecutorGroupStopped(state, code, signal);
|
|
889
1242
|
this.schedulePendingThreads();
|
|
890
1243
|
return;
|
|
891
1244
|
}
|
|
892
|
-
//
|
|
893
|
-
//
|
|
894
|
-
// Tell the senders of any buffered messages instead of dropping silently.
|
|
895
|
-
if (nextCrashCount > MAX_CRASH_COUNT) {
|
|
896
|
-
logger.err(`[supervisor] Worker crash limit exceeded, not restarting: thread=${threadId}`);
|
|
897
|
-
}
|
|
898
|
-
else {
|
|
899
|
-
logger.err(`[supervisor] Worker failed during startup, not restarting: thread=${threadId}`);
|
|
900
|
-
}
|
|
901
|
-
this.recoveryExhaustedThreads.add(threadId);
|
|
902
|
-
this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, pendingMessages));
|
|
1245
|
+
// A refused Lab bootstrap may exit before registering an execution.
|
|
1246
|
+
// Its service owns retries; there are no conversational process respawns.
|
|
903
1247
|
this.schedulePendingThreads();
|
|
904
1248
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
1249
|
+
/**
|
|
1250
|
+
* The executor's leader exited. Empty its process group (the forked
|
|
1251
|
+
* process leads it), then resolve the execution. A group that cannot be
|
|
1252
|
+
* emptied keeps the execution, and a Lab's workspace, blocked.
|
|
1253
|
+
*/
|
|
1254
|
+
confirmExecutorGroupStopped(state, code, signal) {
|
|
1255
|
+
const pgid = state.process.pid;
|
|
1256
|
+
const hostStopping = this.stopped;
|
|
1257
|
+
const reap = (pgid === undefined || process.platform === "win32"
|
|
1258
|
+
? Promise.resolve(true)
|
|
1259
|
+
: Promise.all([
|
|
1260
|
+
this.executorGroups.reap(pgid, this.workerStopGraceMs),
|
|
1261
|
+
this.executorGroups.stopRegistered(state.threadId, this.workerStopGraceMs),
|
|
1262
|
+
]).then((results) => results.every(Boolean)))
|
|
1263
|
+
.catch(() => false)
|
|
1264
|
+
.then((groupStopped) => {
|
|
1265
|
+
if (state.execution) {
|
|
1266
|
+
this.executionScheduler?.handleWorkerExit(state.execution.executionId, code, signal, { hostStopping, groupStopped });
|
|
1267
|
+
}
|
|
1268
|
+
else if (state.labExecution) {
|
|
1269
|
+
if (groupStopped) {
|
|
1270
|
+
this.confirmLabExecutionStopped(state.labExecution, code);
|
|
1271
|
+
}
|
|
1272
|
+
else {
|
|
1273
|
+
logger.err(`[supervisor] LabRun executor group survived its stop: thread=${state.threadId}; its Lab stays blocked`);
|
|
1274
|
+
if (!hostStopping)
|
|
1275
|
+
this.executionScheduler?.blockUntilStopped(state.labExecution.executionId);
|
|
914
1276
|
}
|
|
915
|
-
this.schedulePendingThreads();
|
|
916
|
-
return;
|
|
917
1277
|
}
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
bootstrapMessage: pending[0],
|
|
922
|
-
trustedLabRuntimePolicy,
|
|
923
|
-
});
|
|
924
|
-
this.enqueuePendingWork(next, pending);
|
|
925
|
-
}
|
|
926
|
-
catch (error) {
|
|
927
|
-
this.handleWorkerRespawnFailure(threadId, pending, reason, error);
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
handleWorkerRespawnFailure(threadId, pending, reason, error) {
|
|
931
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
932
|
-
logger.err(`[supervisor] Failed to respawn worker thread=${threadId} after ${reason}: ${message}`);
|
|
933
|
-
this.publishSyntheticActivity(threadId, "failed", "Worker restart failed");
|
|
934
|
-
this.recoveryExhaustedThreads.add(threadId);
|
|
935
|
-
this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, pending));
|
|
936
|
-
}
|
|
937
|
-
outstandingMessages(threadId, pending) {
|
|
938
|
-
const combined = [
|
|
939
|
-
...pending,
|
|
940
|
-
...(this.threadQueues.get(threadId)?.outstandingSnapshot() ?? []),
|
|
941
|
-
];
|
|
942
|
-
const seen = new Set();
|
|
943
|
-
return combined.filter((message) => {
|
|
944
|
-
const key = `${message.channel}:${message.sender}:${message.id}`;
|
|
945
|
-
if (seen.has(key))
|
|
946
|
-
return false;
|
|
947
|
-
seen.add(key);
|
|
948
|
-
return true;
|
|
949
|
-
});
|
|
1278
|
+
})
|
|
1279
|
+
.finally(() => { this.pendingGroupReaps.delete(reap); });
|
|
1280
|
+
this.pendingGroupReaps.add(reap);
|
|
950
1281
|
}
|
|
951
1282
|
freezeLabRuntimePolicy(envelope) {
|
|
952
1283
|
return Object.freeze({
|
|
@@ -954,43 +1285,6 @@ export class ThreadSupervisor {
|
|
|
954
1285
|
signature: envelope.signature,
|
|
955
1286
|
});
|
|
956
1287
|
}
|
|
957
|
-
publishSyntheticActivity(threadId, state, label, messages) {
|
|
958
|
-
if (!threadId.startsWith("fin-") || threadId.startsWith("fin-lab-"))
|
|
959
|
-
return;
|
|
960
|
-
// Suppress owner-facing status for passive-only wakes (email/external-wake).
|
|
961
|
-
// Only publish when at least one message is owner-initiated (Telegram/Discord/etc.).
|
|
962
|
-
if (messages && messages.length > 0 && !messages.some(isOwnerInitiatedMessage)) {
|
|
963
|
-
logger.debug(`[supervisor] Suppressing synthetic activity for thread=${threadId}: all messages are passive ingress`);
|
|
964
|
-
return;
|
|
965
|
-
}
|
|
966
|
-
const now = new Date().toISOString();
|
|
967
|
-
const previous = this.latestActivities.get(threadId);
|
|
968
|
-
const activity = {
|
|
969
|
-
runId: randomUUID(),
|
|
970
|
-
sequence: 1,
|
|
971
|
-
state,
|
|
972
|
-
currentStep: label,
|
|
973
|
-
todos: [],
|
|
974
|
-
updatedAt: now,
|
|
975
|
-
...(state === "queued" ? { heartbeatAt: now } : { completedAt: now }),
|
|
976
|
-
activities: [{ state, label, at: now }],
|
|
977
|
-
workerInstanceId: previous?.workerInstanceId ?? randomUUID(),
|
|
978
|
-
};
|
|
979
|
-
this.latestActivities.set(threadId, activity);
|
|
980
|
-
this.channelManager.publishThreadActivity(threadId, activity, this.agentInstanceId);
|
|
981
|
-
}
|
|
982
|
-
/** Best-effort failure notice to the sender(s) of messages we could not process. */
|
|
983
|
-
notifyPendingSendersOfFailure(threadId, pending) {
|
|
984
|
-
const notified = new Set();
|
|
985
|
-
for (const m of pending) {
|
|
986
|
-
const key = `${m.channel}:${m.sender}`;
|
|
987
|
-
if (notified.has(key))
|
|
988
|
-
continue;
|
|
989
|
-
notified.add(key);
|
|
990
|
-
deliverReplyToMessage(this.channelManager, m, `⚠️ Thread "${threadId}" failed to start and your message could not be processed. Please try again later or contact the owner.`)
|
|
991
|
-
.catch(() => { });
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
1288
|
/**
|
|
995
1289
|
* Deliver a message to the worker via IPC.
|
|
996
1290
|
*/
|
|
@@ -1036,10 +1330,42 @@ export class ThreadSupervisor {
|
|
|
1036
1330
|
...(snapshot.length > 0 ? { activeThreadsSnapshot: snapshot } : {}),
|
|
1037
1331
|
};
|
|
1038
1332
|
state.process.send(ipcMsg);
|
|
1039
|
-
this.armIdleTimer(state);
|
|
1040
1333
|
}
|
|
1041
|
-
|
|
1042
|
-
return this.
|
|
1334
|
+
authority() {
|
|
1335
|
+
return this.executionAuthority;
|
|
1336
|
+
}
|
|
1337
|
+
labPolicyVerifyKey() {
|
|
1338
|
+
return this.authority().labPolicy.publicKey;
|
|
1339
|
+
}
|
|
1340
|
+
/** Freeze and sign a new generic execution's snapshot with this host's authority. */
|
|
1341
|
+
signExecutionSnapshotFor(executionId, budget = DEFAULT_LAB_RUN_BUDGET, caller) {
|
|
1342
|
+
const meta = caller?.kind === "general" ? readThreadMeta(caller.ref) : null;
|
|
1343
|
+
const selection = meta?.engineOverride
|
|
1344
|
+
? { engineOverride: meta.engineOverride, pinnedModel: meta.pinnedModel }
|
|
1345
|
+
: { engineOverride: this.config.engineOverride, pinnedModel: this.config.pinnedModel };
|
|
1346
|
+
return JSON.stringify(signExecutionSnapshot({
|
|
1347
|
+
version: EXECUTION_SNAPSHOT_VERSION,
|
|
1348
|
+
executionId,
|
|
1349
|
+
model: freezeExecutionModel(selection, getLlmRouting(), this.config.provider),
|
|
1350
|
+
budget,
|
|
1351
|
+
}, this.authority().snapshot));
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* Resolve and sign a new execution's snapshot: the thread's engine choice
|
|
1355
|
+
* (else the profile's), the matching route from the current routing
|
|
1356
|
+
* snapshot, and the default task budget. Failover state is not consulted.
|
|
1357
|
+
*/
|
|
1358
|
+
admitExecutionSnapshot(threadId) {
|
|
1359
|
+
const meta = readThreadMeta(threadId);
|
|
1360
|
+
const selection = meta?.engineOverride
|
|
1361
|
+
? { engineOverride: meta.engineOverride, pinnedModel: meta.pinnedModel }
|
|
1362
|
+
: { engineOverride: this.config.engineOverride, pinnedModel: this.config.pinnedModel };
|
|
1363
|
+
return signExecutionSnapshot({
|
|
1364
|
+
version: EXECUTION_SNAPSHOT_VERSION,
|
|
1365
|
+
executionId: threadId,
|
|
1366
|
+
model: freezeExecutionModel(selection, getLlmRouting(), this.config.provider),
|
|
1367
|
+
budget: DEFAULT_LAB_RUN_BUDGET,
|
|
1368
|
+
}, this.authority().snapshot);
|
|
1043
1369
|
}
|
|
1044
1370
|
authorizeLabRuntimePolicy(threadId, raw) {
|
|
1045
1371
|
if (raw && typeof raw === "object") {
|
|
@@ -1049,8 +1375,8 @@ export class ThreadSupervisor {
|
|
|
1049
1375
|
}
|
|
1050
1376
|
// A preserved envelope is trusted only after full HMAC + thread/run
|
|
1051
1377
|
// verification. Re-signing canonicalizes it and prevents object aliasing.
|
|
1052
|
-
const verified = verifyLabRuntimePolicyEnvelope({ payload: envelope.payload, signature: envelope.signature }, this.
|
|
1053
|
-
return signLabRuntimePolicy(verified.policy, this.
|
|
1378
|
+
const verified = verifyLabRuntimePolicyEnvelope({ payload: envelope.payload, signature: envelope.signature }, this.labPolicyVerifyKey(), threadId);
|
|
1379
|
+
return signLabRuntimePolicy(verified.policy, this.authority().labPolicy);
|
|
1054
1380
|
}
|
|
1055
1381
|
if (typeof raw !== "string")
|
|
1056
1382
|
throw new Error(`Lab worker ${threadId} has no runtime policy payload`);
|
|
@@ -1061,7 +1387,7 @@ export class ThreadSupervisor {
|
|
|
1061
1387
|
catch {
|
|
1062
1388
|
throw new Error("Lab runtime policy payload is invalid");
|
|
1063
1389
|
}
|
|
1064
|
-
return signLabRuntimePolicy(parseLabRuntimePolicy(decoded, threadId), this.
|
|
1390
|
+
return signLabRuntimePolicy(parseLabRuntimePolicy(decoded, threadId), this.authority().labPolicy);
|
|
1065
1391
|
}
|
|
1066
1392
|
/**
|
|
1067
1393
|
* Gracefully shutdown a worker.
|
|
@@ -1074,10 +1400,6 @@ export class ThreadSupervisor {
|
|
|
1074
1400
|
return;
|
|
1075
1401
|
state.shuttingDown = true;
|
|
1076
1402
|
logger.system(`[supervisor] Shutting down worker: thread=${threadId} reason=${reason}`);
|
|
1077
|
-
if (state.idleTimer !== null) {
|
|
1078
|
-
clearTimeout(state.idleTimer);
|
|
1079
|
-
state.idleTimer = null;
|
|
1080
|
-
}
|
|
1081
1403
|
const shutdownMsg = {
|
|
1082
1404
|
type: "shutdown",
|
|
1083
1405
|
reason,
|
|
@@ -1088,13 +1410,20 @@ export class ThreadSupervisor {
|
|
|
1088
1410
|
if (error)
|
|
1089
1411
|
logger.debug(`[supervisor] Shutdown IPC unavailable for ${threadId}: ${error.message}`);
|
|
1090
1412
|
});
|
|
1091
|
-
// Wait a grace period, then force-kill if still alive.
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1413
|
+
// Wait a grace period, then force-kill if still alive. An internal
|
|
1414
|
+
// executor is killed with its whole process group.
|
|
1415
|
+
const grace = setTimeout(() => {
|
|
1416
|
+
// Its exit handler already removed it: nothing left to kill.
|
|
1417
|
+
if (this.workers.get(threadId) !== state)
|
|
1418
|
+
return;
|
|
1419
|
+
logger.warn(`[supervisor] Force-killing worker: thread=${threadId}`);
|
|
1420
|
+
const pgid = state.process.pid;
|
|
1421
|
+
if (state.internalExecution && pgid !== undefined && process.platform !== "win32")
|
|
1422
|
+
this.executorGroups.kill(pgid);
|
|
1423
|
+
else
|
|
1095
1424
|
state.process.kill("SIGKILL");
|
|
1096
|
-
|
|
1097
|
-
|
|
1425
|
+
}, this.workerStopGraceMs);
|
|
1426
|
+
grace.unref();
|
|
1098
1427
|
}
|
|
1099
1428
|
/**
|
|
1100
1429
|
* Shutdown all workers gracefully and wait for them to exit.
|
|
@@ -1104,14 +1433,15 @@ export class ThreadSupervisor {
|
|
|
1104
1433
|
async shutdownAll() {
|
|
1105
1434
|
logger.system(`[supervisor] Shutting down all workers (${this.workers.size} active)`);
|
|
1106
1435
|
this.stopped = true;
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
this.restartTimers.clear();
|
|
1436
|
+
this.executionScheduler?.stop();
|
|
1437
|
+
const foregroundShutdown = this.conversations?.shutdown();
|
|
1110
1438
|
// Initiate graceful shutdown for all workers
|
|
1111
1439
|
for (const [threadId] of this.workers) {
|
|
1112
1440
|
this.shutdownWorker(threadId, "supervisor-shutdown");
|
|
1113
1441
|
}
|
|
1114
|
-
// Wait for all workers to exit (each has its own 10s force-kill timer)
|
|
1442
|
+
// Wait for all workers to exit (each has its own 10s force-kill timer).
|
|
1443
|
+
// The supervisor's own exit handler runs first (it was registered at
|
|
1444
|
+
// fork), so an executor's group reap is already pending when this resolves.
|
|
1115
1445
|
const exitPromises = [];
|
|
1116
1446
|
for (const [threadId, state] of this.workers) {
|
|
1117
1447
|
const exitPromise = new Promise((resolve) => {
|
|
@@ -1126,11 +1456,14 @@ export class ThreadSupervisor {
|
|
|
1126
1456
|
});
|
|
1127
1457
|
exitPromises.push(exitPromise);
|
|
1128
1458
|
}
|
|
1129
|
-
await Promise.all(exitPromises);
|
|
1459
|
+
await Promise.all([foregroundShutdown, ...exitPromises]);
|
|
1460
|
+
// Executor groups are confirmed empty (or left for the next host) before the ledger closes.
|
|
1461
|
+
await Promise.all([...this.pendingGroupReaps]);
|
|
1130
1462
|
for (const queue of this.threadQueues.values()) {
|
|
1131
1463
|
queue.close();
|
|
1132
1464
|
}
|
|
1133
1465
|
this.threadQueues.clear();
|
|
1466
|
+
this.executionStore?.close();
|
|
1134
1467
|
logger.system(`[supervisor] All workers shut down`);
|
|
1135
1468
|
}
|
|
1136
1469
|
/** Matches {@link getThreadsOverview}: in-flight turn or buffered work. */
|
|
@@ -1142,23 +1475,18 @@ export class ThreadSupervisor {
|
|
|
1142
1475
|
this.onBusyChange?.();
|
|
1143
1476
|
}
|
|
1144
1477
|
}
|
|
1145
|
-
enqueuePendingWork(state, pending) {
|
|
1146
|
-
if (pending.length === 0)
|
|
1147
|
-
return;
|
|
1148
|
-
const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
|
|
1149
|
-
state.pendingMessages.push(...pending);
|
|
1150
|
-
this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
|
|
1151
|
-
}
|
|
1152
1478
|
/**
|
|
1153
1479
|
* Get a snapshot of all active (ready) worker threads for cross-thread
|
|
1154
1480
|
* awareness. Does not include the main "default" thread.
|
|
1155
1481
|
*/
|
|
1156
1482
|
getActiveThreadsSnapshot() {
|
|
1157
|
-
const result = [];
|
|
1483
|
+
const result = this.conversations?.overview() ?? [];
|
|
1158
1484
|
for (const [threadId, state] of this.workers) {
|
|
1159
1485
|
if (state.ready && !state.shuttingDown) {
|
|
1160
1486
|
result.push({
|
|
1161
1487
|
threadId,
|
|
1488
|
+
kind: state.internalExecution ? "execution" : "conversation",
|
|
1489
|
+
...(state.execution ? { taskId: this.executionStore?.get(state.execution.executionId)?.taskId } : {}),
|
|
1162
1490
|
busy: state.busy,
|
|
1163
1491
|
taskDescription: state.taskDescription,
|
|
1164
1492
|
lastActivityAt: state.lastActivityAt,
|
|
@@ -1176,7 +1504,14 @@ export class ThreadSupervisor {
|
|
|
1176
1504
|
* contains worker threads.
|
|
1177
1505
|
*/
|
|
1178
1506
|
refreshMainProcessAwareness() {
|
|
1179
|
-
setActiveThreadsSnapshot(this.
|
|
1507
|
+
setActiveThreadsSnapshot(this.getConversationTaskSnapshot(DEFAULT_THREAD_ID));
|
|
1508
|
+
}
|
|
1509
|
+
/** A chat must never receive another chat's task descriptions or inputs. */
|
|
1510
|
+
getConversationTaskSnapshot(conversationId) {
|
|
1511
|
+
return this.getActiveThreadsSnapshot().filter((item) => {
|
|
1512
|
+
const record = this.executionStore?.get(item.threadId);
|
|
1513
|
+
return record?.caller.kind === "general" && record.caller.ref === conversationId;
|
|
1514
|
+
});
|
|
1180
1515
|
}
|
|
1181
1516
|
/**
|
|
1182
1517
|
* Obs-facing overview of all worker threads (including still-spawning ones),
|
|
@@ -1184,10 +1519,14 @@ export class ThreadSupervisor {
|
|
|
1184
1519
|
* which is the lightweight cross-thread awareness feed (ready workers only).
|
|
1185
1520
|
*/
|
|
1186
1521
|
getThreadsOverview() {
|
|
1187
|
-
const result = []
|
|
1522
|
+
const result = (this.conversations?.overview() ?? []).map((item) => ({
|
|
1523
|
+
...item, engine: formatThreadEngineLabel(item.threadId, this.config),
|
|
1524
|
+
}));
|
|
1188
1525
|
for (const [threadId, state] of this.workers) {
|
|
1189
1526
|
result.push({
|
|
1190
1527
|
threadId,
|
|
1528
|
+
kind: state.internalExecution ? "execution" : "conversation",
|
|
1529
|
+
...(state.execution ? { taskId: this.executionStore?.get(state.execution.executionId)?.taskId } : {}),
|
|
1191
1530
|
// A spawning worker with buffered work is already active even before
|
|
1192
1531
|
// its first busy status IPC arrives.
|
|
1193
1532
|
busy: this.isWorkerVisiblyBusy(state),
|