visionclaw 0.1.219-dev.codex-issue-2805-worker-runtime.2 → 0.1.219-dev.codex-fix-2941-activity-foreground.1

package/dist/threads/supervisor.js

@@ -1,7 +1,17 @@
 /**
- * Host for foreground conversations and the shared background execution pool.
- * Chat queues run through ConversationScheduler. Only admitted executions and
- * LabRuns fork processes; their ledger owns recovery and confirmed stops.
+ * ThreadSupervisor: orchestrates worker processes for non-default business threads.
+ *
+ * Responsibilities:
+ *  - Spawn worker processes on demand (child_process.fork)
+ *  - Route messages to workers by threadId
+ *  - Relay worker replies back to real channels
+ *  - Enforce concurrency limits (maxConcurrentThreads)
+ *  - Idle timeout & graceful worker shutdown
+ *  - Crash detection & auto-restart
+ *  - Force global commands (/restart, /upgrade, /subscribe, …) to the default thread
+ *  - Engine switch (/engine): update thread.json + restart worker
+ *  - Worker admission pause for the main process's nightly maintenance
+ *  - Graceful supervisor shutdown: notify all workers
  */
 import { fork } from "node:child_process";
 import { randomUUID } from "node:crypto";
@@ -12,6 +22,8 @@ import { PUBLIC_ACTIVITY_HISTORY_LIMIT } from "../agent/public-activity.js";
 import { isDeliveryTarget } from "../channels/delivery-target.js";
 import { DEFAULT_THREAD_ID, isEmailChannel, normalizeChannelName } from "../channels/interface.js";
 import { MessageQueue } from "../channels/queue.js";
+import { isThreadTasksEnabled } from "../config/types.js";
+import { taskStorePathForThread, threadHasRunnableTasks } from "./task-store.js";
 import { logger } from "../logger.js";
 import { isOwnerInitiatedMessage } from "../agent/owner-initiated-eligibility.js";
 import { getProfile, getSharedProfileDir, getThreadDir, loadOwnerConfig } from "../config/index.js";
@@ -23,9 +35,6 @@ import { ensureThreadApplication, formatThreadEngineLabel, readThreadMeta, setTh
 import { getLlmRouting } from "../agent/model-provider.js";
 import { ExecutionStore, isTerminalExecutionState } from "./execution-store.js";
 import { ExecutionScheduler } from "./execution-scheduler.js";
-import { ConversationScheduler } from "../conversations/scheduler.js";
-import { createLazyConversationRuntime } from "../conversations/lazy-runtime.js";
-import { bindRuntimeContext } from "../agent/runtime-context.js";
 import { EXECUTION_VERIFY_KEYS_ENV, createExecutionAuthority, verifyKeysOf, } from "./execution-authority.js";
 import { EXECUTOR_GROUP_REGISTRY_FILE, executorGroupStatus, killExecutorGroupAndWait, reapExitedExecutorGroup, signalExecutorGroup, stopRegisteredGroups, } from "./process-group.js";
 import { ExecutorRefusedError } from "./executor-exit.js";
@@ -45,7 +54,7 @@ import { AgentFeedbackOutbox } from "../agent/feedback-outbox.js";
 import { serializeBrowserRuntimeResolution, serializeDesktopRuntimeResolution, WORKER_BROWSER_RUNTIME_ENV, WORKER_DESKTOP_RUNTIME_ENV, } from "./worker-runtime-resolution.js";
 import { isGlobalCommand } from "./global-commands.js";
 import { getClaudeTranscriptDirForProfileDir } from "../utils/claude-transcripts.js";
-import { isForemanRuntime, supportsTaskRuntime } from "./foreman-protocol.js";
+import { MAX_PARALLEL_FOREMAN_TASKS } from "./foreman-protocol.js";
 export { isGlobalCommand } from "./global-commands.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -108,8 +117,6 @@ export function parseEngineCommand(text) {
     return null;
 }
 export class ThreadSupervisor {
-    conversations;
-    foregroundDisabled;
     channelManager;
     messageQueue;
     maxConcurrentThreads;
@@ -120,6 +127,7 @@ export class ThreadSupervisor {
     workerEnv;
     desktopRuntimeResolution;
     browserRuntimeResolution;
+    crashRestartDelayMs;
     threadQueueFactory;
     onFeedback;
     onBusyChange;
@@ -137,18 +145,23 @@ export class ThreadSupervisor {
     workers = new Map();
     /** Main-process handles for the durable queue.db owned by each thread. */
     threadQueues = new Map();
+    /** Crash counts must outlive WorkerState objects, which are replaced on respawn. */
+    crashCounts = new Map();
+    /** Threads waiting for their bounded crash-backoff timer. */
+    restartTimers = new Map();
+    /** Startup failures / exhausted crash budgets stay dormant until new work arrives. */
+    recoveryExhaustedThreads = new Set();
     /** True once shutdownAll() ran — no new workers may be spawned afterwards. */
     stopped = false;
     maintenancePaused = false;
+    maintenanceMessages = [];
     /** The one execution the maintenance owner submitted under its pause. */
     maintenanceExecutionId = null;
     onMaintenanceInput;
     handleForemanCommand;
-    handleForemanBusinessRead;
     executionStore;
     executionScheduler;
     constructor(options) {
-        this.foregroundDisabled = options.conversationFactory === null;
         this.channelManager = options.channelManager;
         this.messageQueue = options.messageQueue;
         this.maxConcurrentThreads = options.maxConcurrentThreads;
@@ -159,6 +172,7 @@ export class ThreadSupervisor {
         this.workerEnv = options.workerEnv;
         this.desktopRuntimeResolution = options.desktopRuntimeResolution;
         this.browserRuntimeResolution = options.browserRuntimeResolution;
+        this.crashRestartDelayMs = options.crashRestartDelayMs ?? 2000;
         this.executionAuthority = options.executionAuthority ?? createExecutionAuthority();
         this.workerStopGraceMs = options.workerStopGraceMs ?? 10_000;
         this.executorGroups = options.executorGroups ?? {
@@ -172,33 +186,11 @@ export class ThreadSupervisor {
         this.onBusyChange = options.onBusyChange;
         this.onMaintenanceInput = options.onMaintenanceInput;
         this.handleForemanCommand = options.handleForemanCommand;
-        this.handleForemanBusinessRead = options.handleForemanBusinessRead;
         this.threadQueueFactory = options.threadQueueFactory ?? ((threadId, recoverInFlight) => {
             const threadDir = getThreadDir(threadId);
             fs.mkdirSync(threadDir, { recursive: true });
             return new MessageQueue(path.join(threadDir, "queue.db"), { recoverInFlight });
         });
-        // Foreground conversation runtimes never use the background worker transport.
-        const conversationCommand = this.handleForemanCommand;
-        this.conversations = !this.foregroundDisabled
-            ? new ConversationScheduler({
-                create: options.conversationFactory ?? ((start) => createLazyConversationRuntime({
-                    ...start,
-                    config: this.config,
-                    foremanClient: conversationCommand && supportsTaskRuntime()
-                        ? { delegationEnabled: isForemanRuntime(this.config), request: bindRuntimeContext((command) => conversationCommand(start.conversationId, command)) }
-                        : null,
-                    desktopRuntimeResolution: this.desktopRuntimeResolution,
-                    browserRuntimeResolution: this.browserRuntimeResolution,
-                })),
-                queue: (id) => this.getThreadQueue(id, false),
-                emit: (id, event, reply) => this.handleConversationEvent(id, event, reply),
-                awareness: (id) => this.getConversationTaskSnapshot(id),
-                onBusyChange: this.onBusyChange,
-                maxConcurrentTurns: this.config.workers?.foregroundConcurrency ?? 2,
-                idleTtlMs: this.idleTimeoutMs,
-            })
-            : null;
         this.executionStore = options.executionStore === undefined
             ? new ExecutionStore()
             : options.executionStore;
@@ -270,9 +262,7 @@ export class ThreadSupervisor {
      * execution runs, is stopping, or waits in the queue.
      */
     tryPauseForMaintenance() {
-        if (this.stopped || this.maintenancePaused)
-            return null;
-        if (this.conversations?.hasBusyConversations())
+        if (this.stopped || this.maintenancePaused || this.restartTimers.size > 0)
             return null;
         if ([...this.workers.values()].some((worker) => !worker.ready || worker.shuttingDown || this.isWorkerVisiblyBusy(worker)))
             return null;
@@ -292,9 +282,20 @@ export class ThreadSupervisor {
             }
             this.maintenanceExecutionId = null;
             this.maintenancePaused = false;
+            const pending = this.maintenanceMessages;
+            this.maintenanceMessages = [];
             if (this.stopped)
-                return;
+                return; // Accepted ordinary messages are already durable.
+            // Executions queued behind the hold are admitted now, not on the next submit.
             this.executionScheduler?.requestAdmission();
+            for (const { msg, threadId } of pending) {
+                try {
+                    this.routeMessage(msg, threadId);
+                }
+                catch (err) {
+                    logger.warn(`[supervisor] Could not resume queued thread=${threadId}: ${String(err)}`);
+                }
+            }
             this.schedulePendingThreads();
         };
     }
@@ -305,16 +306,58 @@ export class ThreadSupervisor {
             if (queue.hasOutstandingWork()) {
                 logger.system(`[supervisor] Recovered pending work for thread=${threadId}`);
             }
+            else if (this.hasRunnableTasks(threadId)) {
+                logger.system(`[supervisor] Recovered runnable tasks for thread=${threadId}`);
+            }
         }
         this.schedulePendingThreads();
     }
-    /** Resume durable foreground messages without allocating a worker process. */
+    /**
+     * Whether a thread without queued messages still has a task that may
+     * start (#1996 PR 5). Its worker queues the resume itself once it runs:
+     * the supervisor only starts it, it never writes the thread's task store.
+     * By design (owner 10-03): a task that is not due yet is not counted;
+     * its due or unblock wake comes from the existing calendar / event-trigger
+     * messages to the thread. Thread tasks never schedule wakes.
+     * TODO: a worker started this way whose first pass queues nothing (gated)
+     * has no idle timer until its first turn, so it holds a slot until evicted.
+     */
+    hasRunnableTasks(threadId) {
+        if (!isThreadTasksEnabled(this.config))
+            return false;
+        return threadHasRunnableTasks(taskStorePathForThread(threadId), threadId);
+    }
+    /**
+     * Start persisted threads whenever a worker slot is available. Work remains
+     * in its queue.db when all slots are busy and is retried after a worker turn
+     * or exit, rather than being rejected or routed to the default thread.
+     */
     schedulePendingThreads() {
-        if (this.stopped || this.foregroundDisabled)
+        if (this.stopped || this.maintenancePaused)
             return;
-        for (const [id, queue] of this.threadQueues) {
-            if (!isLabRunThreadId(id) && !isGenericExecutionThreadId(id) && queue.hasOutstandingWork())
-                this.conversations?.accept(id);
+        for (const threadId of this.threadQueues.keys()) {
+            if (this.workers.has(threadId))
+                continue;
+            if (this.restartTimers.has(threadId))
+                continue;
+            if (this.recoveryExhaustedThreads.has(threadId))
+                continue;
+            const queue = this.threadQueues.get(threadId);
+            if (!queue?.hasOutstandingWork() && !this.hasRunnableTasks(threadId))
+                continue;
+            if (this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
+                const evicted = this.evictLeastRecentlyActiveIdleWorker(threadId);
+                if (!evicted && this.getActiveWorkerCount() >= this.maxConcurrentThreads)
+                    break;
+            }
+            try {
+                this.spawnWorker(threadId);
+            }
+            catch (error) {
+                logger.warn(`[supervisor] Failed to recover thread=${threadId}: ${error instanceof Error ? error.message : String(error)}`);
+                this.recoveryExhaustedThreads.add(threadId);
+                this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, []));
+            }
         }
     }
     /**
@@ -383,7 +426,7 @@ export class ThreadSupervisor {
         }
         // Thread-scoped engine switch: handled by the supervisor (write thread.json
         // + restart worker), NOT forwarded to the worker.
-        const engine = isLabRunThreadId(threadId) ? null : parseEngineCommand(msg.text);
+        const engine = parseEngineCommand(msg.text);
         if (engine !== null) {
             this.switchThreadEngine(threadId, engine, msg);
             return true;
@@ -409,32 +452,71 @@ export class ThreadSupervisor {
                 throw new Error("A previous run of this Lab is still stopping; retry this delivery");
             return false;
         }
+        let durableInserted = false;
         if (!labRun) {
-            this.getThreadQueue(threadId, true).enqueue(msg);
-            if (this.maintenancePaused && isOwnerInitiatedMessage(msg))
-                this.onMaintenanceInput?.();
-            if (!this.foregroundDisabled) {
-                this.conversations?.accept(threadId, msg);
+            durableInserted = this.getThreadQueue(threadId, true).enqueue(msg);
+            // A new provider/request id is an explicit retry after an exhausted
+            // startup/crash budget. Re-enable this thread without letting unrelated
+            // scheduler ticks revive it forever.
+            if (durableInserted && this.recoveryExhaustedThreads.delete(threadId)) {
+                this.crashCounts.delete(threadId);
             }
+        }
+        if (this.maintenancePaused) {
+            this.maintenanceMessages.push({ msg, threadId });
+            if (isOwnerInitiatedMessage(msg))
+                this.onMaintenanceInput?.();
             return true;
         }
         let state = this.workers.get(threadId);
         if (state?.shuttingDown) {
-            if (claimRejected)
-                throw new Error("The Lab executor is stopping; retry this delivery");
-            return false;
+            logger.debug(`[supervisor] Worker shutting down, buffering message for thread=${threadId}`);
+            const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
+            state.pendingMessages.push(msg);
+            this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
+            return true;
+        }
+        if (!labRun && this.restartTimers.has(threadId)) {
+            logger.debug(`[supervisor] Worker restart already scheduled for thread=${threadId}`);
+            return true;
+        }
+        if (!labRun && this.recoveryExhaustedThreads.has(threadId) && !durableInserted) {
+            logger.debug(`[supervisor] Duplicate work remains dormant after recovery exhaustion: thread=${threadId}`);
+            return true;
         }
         if (!state) {
+            // Check concurrency limit before spawning. Idle workers are only a
+            // keep-warm optimization: they must not block a new logical thread from
+            // making progress. If the process pool is full, evict the least-recently
+            // active idle worker first, and reject only when every slot is busy (or
+            // still spawning/shutting down).
             if (this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
-                deliverReplyToMessage(this.channelManager, msg, `⚠️ Worker capacity reached (${this.maxConcurrentThreads} active). Please try again later.`)
-                    .catch(() => { });
-                return claimRejected;
+                const evicted = this.evictLeastRecentlyActiveIdleWorker(threadId);
+                if (!evicted && this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
+                    if (labRun) {
+                        // LabRun has its own Fin-app lease and is intentionally not stored
+                        // in the ordinary thread queue. Preserve the pre-existing explicit
+                        // failure contract instead of claiming work that was never saved.
+                        logger.warn(`[supervisor] Concurrency limit reached (${this.maxConcurrentThreads}), rejecting LabRun thread=${threadId}`);
+                        deliverReplyToMessage(this.channelManager, msg, `⚠️ Thread concurrency limit reached (${this.maxConcurrentThreads} active). Please try again later.`).catch(() => { });
+                        return claimRejected;
+                    }
+                    logger.info(`[supervisor] Concurrency limit reached (${this.maxConcurrentThreads}); keeping thread=${threadId} queued`);
+                    this.publishSyntheticActivity(threadId, "queued", "Waiting for a worker slot", [msg]);
+                    // The message is already durably accepted in queue.db. Do not leak
+                    // it into the default thread or ask the provider to retry.
+                    return true;
+                }
             }
             state = this.spawnWorker(threadId, { bootstrapMessage: msg });
         }
+        // Worker still spawning: buffer the message; flushed on `ready`.
         if (!state.ready) {
+            logger.debug(`[supervisor] Worker spawning, buffering message for thread=${threadId}`);
             const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
             state.pendingMessages.push(msg);
+            // Check all pending messages: only publish status if at least one is owner-initiated
+            this.publishSyntheticActivity(threadId, "queued", "Starting thread worker", state.pendingMessages);
             this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
             return true;
         }
@@ -442,11 +524,28 @@ export class ThreadSupervisor {
         return true;
     }
     getActiveWorkerCount() {
-        // Persisted process records include groups whose leader exited but whose
-        // stop has not been proved. Neither they nor stopping workers free a slot.
-        const untracked = this.executionStore?.liveExecutions()
-            .filter((record) => !this.workers.has(record.executionId)).length ?? this.pendingGroupReaps.size;
-        return this.workers.size + untracked;
+        let count = 0;
+        for (const state of this.workers.values()) {
+            if (!state.shuttingDown)
+                count += 1;
+        }
+        return count;
+    }
+    evictLeastRecentlyActiveIdleWorker(requestedThreadId) {
+        let candidate = null;
+        for (const state of this.workers.values()) {
+            // An internal executor between turns is not idle keep-warm capacity.
+            if (!state.ready || state.busy || state.shuttingDown || state.internalExecution)
+                continue;
+            if (!candidate || Date.parse(state.lastActivityAt) < Date.parse(candidate.lastActivityAt)) {
+                candidate = state;
+            }
+        }
+        if (!candidate)
+            return false;
+        logger.system(`[supervisor] Evicting idle worker thread=${candidate.threadId} to make room for thread=${requestedThreadId}`);
+        this.shutdownWorker(candidate.threadId, "idle-evicted");
+        return true;
     }
     /**
      * Switch a thread's engine: persist the model/provider into thread.json, then
@@ -465,7 +564,6 @@ export class ThreadSupervisor {
             return;
         }
         // Restart the running worker so it re-reads thread.json on next spawn.
-        void this.conversations?.stop(threadId);
         const state = this.workers.get(threadId);
         if (state) {
             this.shutdownWorker(threadId, "engine-switch");
@@ -478,47 +576,6 @@ export class ThreadSupervisor {
     startExecutions() {
         this.executionScheduler?.start();
     }
-    /** Foreground turns have their own admission pool; execution slots are unrelated. */
-    acquireConversationTurn(signal) {
-        return this.conversations?.acquireTurn(signal) ?? Promise.resolve(() => undefined);
-    }
-    handleConversationEvent(conversationId, event, reply) {
-        if (event.type === "reply") {
-            if (!isDeliveryTarget(event.target)
-                || (event.ownerEscalation === true && !this.isOwnerEscalationTarget(event.target))) {
-                if (event.replyId)
-                    reply({ type: "reply-result", replyId: event.replyId, ok: false, errorMessage: "Invalid conversation reply target" });
-                return;
-            }
-            void this.channelManager.deliver(event.target, event.message, event.attachments, {
-                businessThreadId: event.ownerEscalation ? DEFAULT_THREAD_ID : conversationId,
-                ...(event.kind ? { kind: event.kind } : {}),
-            }).then(() => {
-                this.replyEscalation.success(conversationId);
-                if (event.replyId)
-                    reply({ type: "reply-result", replyId: event.replyId, ok: true });
-            }).catch((error) => {
-                const message = error instanceof Error ? error.message : String(error);
-                if (event.replyId)
-                    reply({ type: "reply-result", replyId: event.replyId, ok: false, errorMessage: message });
-                if (this.replyEscalation.failure(conversationId))
-                    void this.alertOwnerOfThreadDeliveryFailure(conversationId, event.target, message);
-            });
-        }
-        else if (event.type === "typing") {
-            if (isDeliveryTarget(event.target))
-                void this.channelManager.sendTyping(event.target).catch(() => { });
-        }
-        else if (event.type === "status" && event.activity) {
-            this.latestActivities.set(conversationId, event.activity);
-            this.channelManager.publishThreadActivity(conversationId, event.activity, this.agentInstanceId);
-        }
-        else if (event.type === "feedback") {
-            const persisted = this.feedbackOutbox.enqueue(event.feedback) ?? this.feedbackOutbox.getPersisted(event.feedback.id);
-            if (persisted)
-                this.onFeedback?.(persisted);
-        }
-    }
     /**
      * The execution API for trusted in-process callers. Submissions are
      * accepted only after the command, record and snapshot are durable.
@@ -528,14 +585,12 @@ export class ThreadSupervisor {
         if (!store)
             return null;
         return {
-            atomic: (operation) => store.atomic(operation),
-            attemptsForTask: (taskId, scope) => store.attemptsForTask(taskId, scope),
             submit: (envelope, input) => {
                 const executionId = `exec-${randomUUID()}`;
                 const outcome = store.submit(envelope, {
                     ...input,
                     executionId,
-                    snapshot: this.signExecutionSnapshotFor(executionId, input.budget, envelope.caller),
+                    snapshot: this.signExecutionSnapshotFor(executionId),
                 });
                 if (outcome.kind === "ok" && !outcome.replay) {
                     if (this.maintenancePaused)
@@ -622,11 +677,21 @@ export class ThreadSupervisor {
             : !this.maintenancePaused;
         if (!admitted)
             return 0;
-        const parent = record.payload.parentTaskId ?? record.taskId;
-        const perTask = this.config.workers?.maxConcurrentPerTask ?? this.maxConcurrentThreads;
-        const live = this.executionStore?.liveExecutions() ?? [];
-        if (live.filter((item) => (item.payload.parentTaskId ?? item.taskId) === parent).length >= perTask)
-            return 0;
+        // Foreman tasks share the overall pool but never take more than their cap.
+        // A stopping executor still counts: its slot frees only once it exits.
+        if (record.caller.kind === "general") {
+            let foremanWorkers = 0;
+            for (const state of this.workers.values())
+                if (state.foreman)
+                    foremanWorkers += 1;
+            if (foremanWorkers >= MAX_PARALLEL_FOREMAN_TASKS)
+                return 0;
+        }
+        const free = this.maxConcurrentThreads - this.getActiveWorkerCount();
+        // An idle keep-warm worker yields its slot rather than holding admission
+        // until its idle timeout.
+        if (free > 0 || !this.evictLeastRecentlyActiveIdleWorker(record.executionId))
+            return free;
         return this.maxConcurrentThreads - this.getActiveWorkerCount();
     }
     /**
@@ -835,9 +900,6 @@ export class ThreadSupervisor {
      * Spawn a worker process for the given threadId.
      */
     spawnWorker(threadId, options = {}) {
-        if (!options.execution && !isLabRunThreadId(threadId)) {
-            throw new Error("Only admitted Tasks and LabRuns may start worker processes");
-        }
         logger.system(`[supervisor] Spawning worker for thread=${threadId}`);
         const threadMeta = ensureThreadApplication(threadId, options.bootstrapMessage?.app);
         threadApplication(threadId, this.config);
@@ -898,6 +960,7 @@ export class ThreadSupervisor {
             busy: false,
             taskDescription: "spawning",
             lastActivityAt: new Date().toISOString(),
+            idleTimer: null,
             shuttingDown: false,
             pendingMessages: [],
             internalExecution,
@@ -946,13 +1009,16 @@ export class ThreadSupervisor {
                 this.handleReply(state, data);
                 break;
             case "typing":
+                this.handleTyping(state, data);
                 break;
             case "status":
                 this.handleStatus(state, data);
                 break;
             case "feedback":
+                this.handleFeedback(state, data);
                 break;
             case "turn-complete":
+                this.handleTurnComplete(state, data);
                 break;
             case "execution-bootstrap-ready":
                 this.handleExecutionBootstrap(state);
@@ -992,25 +1058,15 @@ export class ThreadSupervisor {
         };
         if (typeof msg.requestId !== "string")
             return;
-        if (state.internalExecution && msg.command.type === "business_read") {
-            const expected = state.execution;
-            const record = expected ? this.executionStore?.get(expected.executionId) : null;
-            if (!record || state.shuttingDown || record.state !== "running" || record.caller.kind !== "general"
-                || record.workerInstanceId !== expected?.generation || !record.payload.businessRead || !this.handleForemanBusinessRead) {
-                answer({ ok: false, text: "Business reads require a live authorized Foreman execution" });
-                return;
-            }
-            const workspace = record.payload.workspaceRef ?? path.join(getThreadDir(state.threadId), "workspace");
-            this.handleForemanBusinessRead(record.executionId, msg.command, workspace).then((result) => {
-                const current = this.executionStore?.get(record.executionId);
-                if (current?.state === "running" && current.workerInstanceId === expected.generation)
-                    answer(result);
-                else
-                    answer({ ok: false, text: "The execution stopped before the business read completed" });
-            }).catch((error) => answer({ ok: false, text: `Business read failed: ${error instanceof Error ? error.message : String(error)}` }));
+        if (state.internalExecution || !this.handleForemanCommand) {
+            answer({ ok: false, text: "Error: background tasks are not available in this conversation." });
             return;
         }
-        answer({ ok: false, text: "Internal executions cannot delegate or control other Tasks" });
+        this.handleForemanCommand(state.threadId, msg.command)
+            .then(answer)
+            .catch((error) => {
+            answer({ ok: false, text: `Error: ${error instanceof Error ? error.message : String(error)}` });
+        });
     }
     /**
      * The executor is listening: hand it the admitted record from the ledger.
@@ -1033,15 +1089,22 @@ export class ThreadSupervisor {
             type: "execution-start",
             record,
             unfinishedTools: store.unfinishedToolCalls(record.executionId),
-            // input_applied is a session handoff cursor, not a durable provider
-            // receipt. A crash between that cursor and the SDK's transcript write
-            // must not erase an accepted answer. Replay the authoritative history.
-            pendingInputs: store.pendingInputs(record.executionId, 0),
+            pendingInputs: store.pendingInputs(record.executionId, record.processedInputSeq),
         }, (error) => {
             if (error)
                 logger.debug(`[supervisor] Execution start IPC unavailable for ${state.threadId}: ${error.message}`);
         });
     }
+    handleFeedback(state, msg) {
+        logger.debug(`[supervisor] Feedback received from thread=${state.threadId}: ${msg.feedback.id}`);
+        const persisted = this.feedbackOutbox.enqueue(msg.feedback)
+            ?? this.feedbackOutbox.getPersisted(msg.feedback.id);
+        if (!persisted)
+            return;
+        this.onFeedback?.(persisted);
+        // Main HeartbeatManager will discover this durable item on its next tick.
+        logger.debug(`[supervisor] Persisted feedback from thread=${state.threadId}: ${persisted.id}`);
+    }
     handleLabRuntimeBootstrapReady(state) {
         if (!state.labRuntimePolicy || !isLabRunThreadId(state.threadId)) {
             logger.warn(`[supervisor] Refusing Lab credential bootstrap for non-Lab thread=${state.threadId}`);
@@ -1119,7 +1182,47 @@ export class ThreadSupervisor {
         }
     }
     handleReply(state, msg) {
-        this.sendReplyResult(state, msg.replyId, false, "Internal worker executions cannot deliver external messages");
+        if (state.internalExecution) {
+            this.sendReplyResult(state, msg.replyId, false, "Internal worker executions cannot deliver external messages");
+            return;
+        }
+        if (!isDeliveryTarget(msg.target)) {
+            const errorMessage = "worker reply contained an invalid target";
+            logger.warn(`[supervisor] ${errorMessage}`);
+            this.sendReplyResult(state, msg.replyId, false, errorMessage);
+            return;
+        }
+        // A worker's sends belong to its thread, so mail adapters record provider
+        // replies against it. An owner escalation must not: the owner's answer is
+        // private to the owner's own (default) conversation. The marker is only
+        // honored for a real owner target resolved here, in the main process.
+        let businessThreadId = state.threadId;
+        if (msg.ownerEscalation === true) {
+            if (!this.isOwnerEscalationTarget(msg.target)) {
+                const errorMessage = "owner escalation target is not a bound owner target";
+                logger.warn(`[supervisor] Refused reply from thread=${state.threadId}: ${errorMessage}`);
+                this.sendReplyResult(state, msg.replyId, false, errorMessage);
+                return;
+            }
+            businessThreadId = DEFAULT_THREAD_ID;
+        }
+        logger.debug(`[supervisor] Relaying reply from channel=${msg.target.channel}`);
+        this.channelManager.deliver(msg.target, msg.message, msg.attachments, {
+            businessThreadId,
+            ...(msg.kind ? { kind: msg.kind } : {}),
+        })
+            .then(() => {
+            this.replyEscalation.success(state.threadId);
+            this.sendReplyResult(state, msg.replyId, true);
+        })
+            .catch((err) => {
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            logger.warn(`[supervisor] Failed to relay reply: ${errorMessage}`);
+            this.sendReplyResult(state, msg.replyId, false, errorMessage);
+            if (this.replyEscalation.failure(state.threadId)) {
+                void this.alertOwnerOfThreadDeliveryFailure(state.threadId, msg.target, errorMessage);
+            }
+        });
     }
     /** True when `target` is one of the owner's usable bound targets (no reply context). */
     isOwnerEscalationTarget(target) {
@@ -1175,6 +1278,17 @@ export class ThreadSupervisor {
             logger.debug(`[supervisor] Could not ack reply ${replyId}: ${err instanceof Error ? err.message : String(err)}`);
         }
     }
+    handleTyping(state, msg) {
+        if (state.internalExecution)
+            return;
+        if (!isDeliveryTarget(msg.target)) {
+            logger.warn("[supervisor] Worker typing contained an invalid target");
+            return;
+        }
+        this.channelManager.sendTyping(msg.target).catch(() => {
+            // best-effort
+        });
+    }
     handleStatus(state, msg) {
         if (msg.activity && !state.privateActivity) {
             const current = state.activity;
@@ -1205,6 +1319,34 @@ export class ThreadSupervisor {
             this.executionScheduler?.requestAdmission();
         }
     }
+    handleTurnComplete(state, _msg) {
+        logger.system(`[supervisor] Turn complete from thread=${state.threadId}`);
+        // A completed turn proves the worker is healthy — reset its crash budget so
+        // sporadic crashes spread over a long lifetime don't exhaust the cap.
+        this.crashCounts.delete(state.threadId);
+        this.recoveryExhaustedThreads.delete(state.threadId);
+        // Billing is NOT handled here: the worker's own WakeCycleRunner already
+        // wrote the wake-cycle row into the shared transcript-memory.db (WAL-safe).
+        // Memory dreaming is not triggered here either: it rewrites and deletes
+        // files in the shared memories/ tree, so it runs on the deep-night sweep
+        // only (owned by NightlyMaintenance), not once per wake cycle.
+        this.armIdleTimer(state);
+        this.schedulePendingThreads();
+        this.executionScheduler?.requestAdmission();
+    }
+    armIdleTimer(state) {
+        if (state.idleTimer !== null) {
+            clearTimeout(state.idleTimer);
+        }
+        state.idleTimer = setTimeout(() => {
+            if (state.busy) {
+                logger.debug(`[supervisor] Worker still busy at idle timeout, extending: thread=${state.threadId}`);
+                this.armIdleTimer(state);
+                return;
+            }
+            this.shutdownWorker(state.threadId, "idle-timeout");
+        }, this.idleTimeoutMs);
+    }
     handleWorkerExit(threadId, code, signal) {
         const state = this.workers.get(threadId);
         if (!state)
@@ -1212,6 +1354,11 @@ export class ThreadSupervisor {
         const wasReady = state.ready;
         logger.system(`[supervisor] Worker exited: thread=${threadId} code=${code} signal=${signal} wasReady=${wasReady}`);
         // Clear idle timer.
+        if (state.idleTimer !== null) {
+            clearTimeout(state.idleTimer);
+            state.idleTimer = null;
+        }
+        const pendingMessages = state.pendingMessages;
         if (state.busy && state.activity && !state.shuttingDown) {
             const now = new Date().toISOString();
             const failedActivity = {
@@ -1242,8 +1389,68 @@ export class ThreadSupervisor {
             this.schedulePendingThreads();
             return;
         }
-        // A refused Lab bootstrap may exit before registering an execution.
-        // Its service owns retries; there are no conversational process respawns.
+        // LabRun workers are single-shot executors. A retry is a new durable run
+        // with a new `fin-lab-run-<runId>` thread; never revive the old session.
+        if (isLabRunThreadId(threadId)) {
+            this.crashCounts.delete(threadId);
+            this.schedulePendingThreads();
+            return;
+        }
+        // Clean exit, or any exit during a requested shutdown (a graceful stop that
+        // ends non-zero is not a crash): don't count towards the crash cap. Respawn
+        // only when messages arrived during the shutdown window — but never after
+        // the supervisor itself has been stopped.
+        if (code === 0 || state.shuttingDown) {
+            if (code === 0)
+                this.crashCounts.delete(threadId);
+            if (pendingMessages.length > 0 && !this.stopped) {
+                this.spawnWorkerForPendingMessages(threadId, pendingMessages, "shutdown-respawn", state.labRuntimePolicy, state.executionSnapshot);
+            }
+            this.schedulePendingThreads();
+            return;
+        }
+        // Crash recovery: restart if crashed unexpectedly and under threshold.
+        const MAX_CRASH_COUNT = 3;
+        const nextCrashCount = wasReady ? (this.crashCounts.get(threadId) ?? 0) + 1 : this.crashCounts.get(threadId) ?? 0;
+        if (wasReady) {
+            this.crashCounts.set(threadId, nextCrashCount);
+        }
+        if (wasReady && nextCrashCount <= MAX_CRASH_COUNT && !this.stopped) {
+            logger.warn(`[supervisor] Worker crashed, restarting (attempt ${nextCrashCount}/${MAX_CRASH_COUNT}): thread=${threadId}`);
+            // Re-spawn after a brief delay.
+            const trustedLabRuntimePolicy = state.labRuntimePolicy;
+            const trustedExecutionSnapshot = state.executionSnapshot;
+            const timer = setTimeout(() => {
+                this.restartTimers.delete(threadId);
+                if (this.stopped || this.maintenancePaused || this.workers.has(threadId))
+                    return;
+                try {
+                    const next = this.spawnWorker(threadId, {
+                        bootstrapMessage: pendingMessages[0],
+                        trustedLabRuntimePolicy,
+                        trustedExecutionSnapshot,
+                    });
+                    this.enqueuePendingWork(next, pendingMessages);
+                }
+                catch (error) {
+                    this.handleWorkerRespawnFailure(threadId, pendingMessages, "crash-restart", error);
+                }
+            }, this.crashRestartDelayMs);
+            this.restartTimers.set(threadId, timer);
+            this.schedulePendingThreads();
+            return;
+        }
+        // Not restarting: either the worker died before ever becoming ready
+        // (startup failure — restarting would loop) or the crash cap is exhausted.
+        // Tell the senders of any buffered messages instead of dropping silently.
+        if (nextCrashCount > MAX_CRASH_COUNT) {
+            logger.err(`[supervisor] Worker crash limit exceeded, not restarting: thread=${threadId}`);
+        }
+        else {
+            logger.err(`[supervisor] Worker failed during startup, not restarting: thread=${threadId}`);
+        }
+        this.recoveryExhaustedThreads.add(threadId);
+        this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, pendingMessages));
         this.schedulePendingThreads();
     }
     /**
@@ -1266,6 +1473,7 @@ export class ThreadSupervisor {
                 this.executionScheduler?.handleWorkerExit(state.execution.executionId, code, signal, { hostStopping, groupStopped });
             }
             else if (state.labExecution) {
+                this.crashCounts.delete(state.threadId);
                 if (groupStopped) {
                     this.confirmLabExecutionStopped(state.labExecution, code);
                 }
@@ -1279,12 +1487,96 @@ export class ThreadSupervisor {
             .finally(() => { this.pendingGroupReaps.delete(reap); });
         this.pendingGroupReaps.add(reap);
     }
+    spawnWorkerForPendingMessages(threadId, pending, reason, trustedLabRuntimePolicy, trustedExecutionSnapshot) {
+        if (pending.length === 0)
+            return;
+        if (this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
+            const evicted = this.evictLeastRecentlyActiveIdleWorker(threadId);
+            if (!evicted && this.getActiveWorkerCount() >= this.maxConcurrentThreads) {
+                logger.warn(`[supervisor] Cannot respawn worker for thread=${threadId} after ${reason}: concurrency limit reached`);
+                if (isLabRunThreadId(threadId)) {
+                    this.notifyPendingSendersOfFailure(threadId, pending);
+                }
+                this.schedulePendingThreads();
+                return;
+            }
+        }
+        try {
+            const next = this.spawnWorker(threadId, {
+                bootstrapMessage: pending[0],
+                trustedLabRuntimePolicy,
+                trustedExecutionSnapshot,
+            });
+            this.enqueuePendingWork(next, pending);
+        }
+        catch (error) {
+            this.handleWorkerRespawnFailure(threadId, pending, reason, error);
+        }
+    }
+    handleWorkerRespawnFailure(threadId, pending, reason, error) {
+        const message = error instanceof Error ? error.message : String(error);
+        logger.err(`[supervisor] Failed to respawn worker thread=${threadId} after ${reason}: ${message}`);
+        this.publishSyntheticActivity(threadId, "failed", "Worker restart failed");
+        this.recoveryExhaustedThreads.add(threadId);
+        this.notifyPendingSendersOfFailure(threadId, this.outstandingMessages(threadId, pending));
+    }
+    outstandingMessages(threadId, pending) {
+        const combined = [
+            ...pending,
+            ...(this.threadQueues.get(threadId)?.outstandingSnapshot() ?? []),
+        ];
+        const seen = new Set();
+        return combined.filter((message) => {
+            const key = `${message.channel}:${message.sender}:${message.id}`;
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+    }
     freezeLabRuntimePolicy(envelope) {
         return Object.freeze({
             payload: envelope.payload,
             signature: envelope.signature,
         });
     }
+    publishSyntheticActivity(threadId, state, label, messages) {
+        if (!threadId.startsWith("fin-") || threadId.startsWith("fin-lab-"))
+            return;
+        // Suppress owner-facing status for passive-only wakes (email/external-wake).
+        // Only publish when at least one message is owner-initiated (Telegram/Discord/etc.).
+        if (messages && messages.length > 0 && !messages.some(isOwnerInitiatedMessage)) {
+            logger.debug(`[supervisor] Suppressing synthetic activity for thread=${threadId}: all messages are passive ingress`);
+            return;
+        }
+        const now = new Date().toISOString();
+        const previous = this.latestActivities.get(threadId);
+        const activity = {
+            runId: randomUUID(),
+            sequence: 1,
+            state,
+            currentStep: label,
+            todos: [],
+            updatedAt: now,
+            ...(state === "queued" ? { heartbeatAt: now } : { completedAt: now }),
+            activities: [{ state, label, at: now }],
+            workerInstanceId: previous?.workerInstanceId ?? randomUUID(),
+        };
+        this.latestActivities.set(threadId, activity);
+        this.channelManager.publishThreadActivity(threadId, activity, this.agentInstanceId);
+    }
+    /** Best-effort failure notice to the sender(s) of messages we could not process. */
+    notifyPendingSendersOfFailure(threadId, pending) {
+        const notified = new Set();
+        for (const m of pending) {
+            const key = `${m.channel}:${m.sender}`;
+            if (notified.has(key))
+                continue;
+            notified.add(key);
+            deliverReplyToMessage(this.channelManager, m, `⚠️ Thread "${threadId}" failed to start and your message could not be processed. Please try again later or contact the owner.`)
+                .catch(() => { });
+        }
+    }
     /**
      * Deliver a message to the worker via IPC.
      */
@@ -1330,6 +1622,7 @@ export class ThreadSupervisor {
             ...(snapshot.length > 0 ? { activeThreadsSnapshot: snapshot } : {}),
         };
         state.process.send(ipcMsg);
+        this.armIdleTimer(state);
     }
     authority() {
         return this.executionAuthority;
@@ -1338,15 +1631,11 @@ export class ThreadSupervisor {
         return this.authority().labPolicy.publicKey;
     }
     /** Freeze and sign a new generic execution's snapshot with this host's authority. */
-    signExecutionSnapshotFor(executionId, budget = DEFAULT_LAB_RUN_BUDGET, caller) {
-        const meta = caller?.kind === "general" ? readThreadMeta(caller.ref) : null;
-        const selection = meta?.engineOverride
-            ? { engineOverride: meta.engineOverride, pinnedModel: meta.pinnedModel }
-            : { engineOverride: this.config.engineOverride, pinnedModel: this.config.pinnedModel };
+    signExecutionSnapshotFor(executionId, budget = DEFAULT_LAB_RUN_BUDGET) {
         return JSON.stringify(signExecutionSnapshot({
             version: EXECUTION_SNAPSHOT_VERSION,
             executionId,
-            model: freezeExecutionModel(selection, getLlmRouting(), this.config.provider),
+            model: freezeExecutionModel({ engineOverride: this.config.engineOverride, pinnedModel: this.config.pinnedModel }, getLlmRouting(), this.config.provider),
             budget,
         }, this.authority().snapshot));
     }
@@ -1400,6 +1689,10 @@ export class ThreadSupervisor {
             return;
         state.shuttingDown = true;
         logger.system(`[supervisor] Shutting down worker: thread=${threadId} reason=${reason}`);
+        if (state.idleTimer !== null) {
+            clearTimeout(state.idleTimer);
+            state.idleTimer = null;
+        }
         const shutdownMsg = {
             type: "shutdown",
             reason,
@@ -1434,7 +1727,9 @@ export class ThreadSupervisor {
         logger.system(`[supervisor] Shutting down all workers (${this.workers.size} active)`);
         this.stopped = true;
         this.executionScheduler?.stop();
-        const foregroundShutdown = this.conversations?.shutdown();
+        for (const timer of this.restartTimers.values())
+            clearTimeout(timer);
+        this.restartTimers.clear();
         // Initiate graceful shutdown for all workers
         for (const [threadId] of this.workers) {
             this.shutdownWorker(threadId, "supervisor-shutdown");
@@ -1456,7 +1751,7 @@ export class ThreadSupervisor {
             });
             exitPromises.push(exitPromise);
         }
-        await Promise.all([foregroundShutdown, ...exitPromises]);
+        await Promise.all(exitPromises);
         // Executor groups are confirmed empty (or left for the next host) before the ledger closes.
         await Promise.all([...this.pendingGroupReaps]);
         for (const queue of this.threadQueues.values()) {
@@ -1475,18 +1770,23 @@ export class ThreadSupervisor {
             this.onBusyChange?.();
         }
     }
+    enqueuePendingWork(state, pending) {
+        if (pending.length === 0)
+            return;
+        const wasVisibleBusy = this.isWorkerVisiblyBusy(state);
+        state.pendingMessages.push(...pending);
+        this.emitBusyChangeIfNeeded(wasVisibleBusy, state);
+    }
     /**
      * Get a snapshot of all active (ready) worker threads for cross-thread
      * awareness. Does not include the main "default" thread.
      */
     getActiveThreadsSnapshot() {
-        const result = this.conversations?.overview() ?? [];
+        const result = [];
         for (const [threadId, state] of this.workers) {
             if (state.ready && !state.shuttingDown) {
                 result.push({
                     threadId,
-                    kind: state.internalExecution ? "execution" : "conversation",
-                    ...(state.execution ? { taskId: this.executionStore?.get(state.execution.executionId)?.taskId } : {}),
                     busy: state.busy,
                     taskDescription: state.taskDescription,
                     lastActivityAt: state.lastActivityAt,
@@ -1504,14 +1804,7 @@ export class ThreadSupervisor {
      * contains worker threads.
      */
     refreshMainProcessAwareness() {
-        setActiveThreadsSnapshot(this.getConversationTaskSnapshot(DEFAULT_THREAD_ID));
-    }
-    /** A chat must never receive another chat's task descriptions or inputs. */
-    getConversationTaskSnapshot(conversationId) {
-        return this.getActiveThreadsSnapshot().filter((item) => {
-            const record = this.executionStore?.get(item.threadId);
-            return record?.caller.kind === "general" && record.caller.ref === conversationId;
-        });
+        setActiveThreadsSnapshot(this.getActiveThreadsSnapshot());
     }
     /**
      * Obs-facing overview of all worker threads (including still-spawning ones),
@@ -1519,14 +1812,10 @@ export class ThreadSupervisor {
      * which is the lightweight cross-thread awareness feed (ready workers only).
      */
     getThreadsOverview() {
-        const result = (this.conversations?.overview() ?? []).map((item) => ({
-            ...item, engine: formatThreadEngineLabel(item.threadId, this.config),
-        }));
+        const result = [];
         for (const [threadId, state] of this.workers) {
             result.push({
                 threadId,
-                kind: state.internalExecution ? "execution" : "conversation",
-                ...(state.execution ? { taskId: this.executionStore?.get(state.execution.executionId)?.taskId } : {}),
                 // A spawning worker with buffered work is already active even before
                 // its first busy status IPC arrives.
                 busy: this.isWorkerVisiblyBusy(state),