visionclaw 0.1.219-dev.codex-issue-2805-worker-runtime.3 → 0.1.219-dev.codex-fix-2941-activity-foreground.1

package/dist/agent/wake-cycle-runner.js

@@ -8,7 +8,6 @@
 import { isEmailChannel, presentsAssistantText } from "../channels/interface.js";
 import { isCalendarInterruptMessage } from "./owner-initiated-eligibility.js";
 import { emailAddressesMatch } from "../utils/email.js";
-import { EXTERNAL_WAKE_KIND } from "../channels/queue.js";
 import { nextRecoveryPhase } from "./lifecycle-transitions.js";
 import { COMPACTION_FAILURE_CONTINUATION, CompactionFailureError } from "./compaction-failure.js";
 import { ToolOutputRepairError } from "./transcript-output-repair.js";
@@ -33,6 +32,8 @@ import path from "path";
 import { randomUUID } from "node:crypto";
 import { runContextRefresh } from "./context-refresh.js";
 import { loadToolAbortRecoveryChain, saveToolAbortRecoveryChain } from "./recovery-chain.js";
+import { buildThreadTaskContext, executionEvents, isTaskRelevantMessage, recordExecutionEvent, } from "../threads/thread-task-runtime.js";
+import { isThreadTaskContextBlock } from "../threads/thread-task-context.js";
 import { getWorkerExecutionRuntime } from "../threads/worker-execution-runtime.js";
 const AUTH_PATTERNS = /\b(401|402|403|authentication|billing|unauthorized|forbidden|api.?key)\b/i;
 // Fatal environment/installation problems that won't fix themselves on retry.
@@ -276,15 +277,15 @@ export function isSharedCredentialFailure(err) {
     }
     return classifyError(err) === "provider_failed" && !isEngineScopedFailure(err);
 }
-function formatOwnerErrorNotice(category, consecutiveErrors, errMsg) {
+function formatOwnerErrorNotice(category, consecutiveErrors, sleepMin, errMsg) {
     if (category === "fatal") {
-        return "Agent encountered an internal runtime error. This wake has stopped; the next message or heartbeat will retry.";
+        return `Agent encountered an internal runtime error. Sleeping ${sleepMin}m before retrying.`;
     }
     if (category === "local") {
         // Name the cause. The old text blamed the model provider for an error the
         // provider never saw, which is what made the original alert storm
         // impossible to diagnose from the notification alone.
-        return `Agent hit a local session-coordination error (not a provider failure): ${errMsg ?? "unknown cause"}. This wake has stopped; the next message or heartbeat will retry.`;
+        return `Agent hit a local session-coordination error (not a provider failure): ${errMsg ?? "unknown cause"}. Sleeping ${sleepMin}m before retrying.`;
     }
     if (category === "request_invalid") {
         return `Agent stopped this wake because the provider rejected a malformed request (${errMsg ?? "unknown cause"}).`;
@@ -295,7 +296,7 @@ function formatOwnerErrorNotice(category, consecutiveErrors, errMsg) {
     if (category === "empty_turn") {
         return "The assistant returned another empty turn without confirming completion. The task may be incomplete. I stopped automatic continuation to avoid repeating work; existing results should be checked before continuing.";
     }
-    return `Agent encountered an internal model/provider error ${consecutiveErrors} times in a row. This wake has stopped; the next message or heartbeat will retry.`;
+    return `Agent encountered an internal model/provider error ${consecutiveErrors} times in a row. Sleeping ${sleepMin}m before retrying.`;
 }
 function isPromptTooLong(err) {
     const terminalReason = getTerminalReason(err);
@@ -363,6 +364,7 @@ export const MAX_EMPTY_TURN_RETRIES = 1;
 const ERROR_RETRY_COMPACT_USED_PCT = 0.7;
 const BASE_DELAY_MS = 5_000;
 const MAX_DELAY_MS = 5 * 60_000;
+const FATAL_SLEEP_MS = 5 * 60_000;
 /** Max times to retry the LLM call on stream silence timeout before aborting. */
 export const MAX_SILENCE_RETRIES = 3;
 /** Bounded retries for transient stream/network disconnects in any wake cycle. */
@@ -377,6 +379,22 @@ export const MAX_OUTPUT_RECOVERY_ATTEMPTS = 2;
 const SILENCE_TIMEOUT_SCHEDULE_MS = [300_000, 600_000, 900_000];
 /** Decreasing effort levels per attempt to reduce thinking overhead on retries. */
 const EFFORT_SCHEDULE = ["xhigh", "high", "medium"];
+/**
+ * A wake retried on a reset session re-sends its first message, whose thread
+ * task block (#1996) was built before the reset's reconcile. Swap it for a
+ * fresh one so the new session sees exactly one, current copy. Without a
+ * task runtime this returns the blocks unchanged.
+ */
+export function withFreshThreadTaskBlock(blocks) {
+    const fresh = buildThreadTaskContext();
+    const index = blocks.findIndex((block) => block.type === "text" && isThreadTaskContextBlock(block.text));
+    if (index === -1)
+        return fresh ? [...blocks, { type: "text", text: fresh }] : blocks;
+    const rest = blocks.filter((_, i) => i !== index);
+    if (fresh)
+        rest.splice(index, 0, { type: "text", text: fresh });
+    return rest;
+}
 export class WakeCycleRunner {
     externalContextRegistry;
     deps;
@@ -388,12 +406,6 @@ export class WakeCycleRunner {
     localCoordinationErrors = 0;
     /** Empty Claude assistant turns in this wake (not counted as network failures). */
     emptyTurnRetries = 0;
-    /**
-     * Set once a call's silence ceiling is spent. Silence retries live inside
-     * the call, so the outer error handler must not spend another backoff on
-     * the same call. Cleared with the next wake.
-     */
-    silenceRecoveryExhausted = false;
     /** Recovering only while a spendable bucket is in use. Reset when the wake ends. */
     recoveryPhase = "Running";
     /**
@@ -494,7 +506,6 @@ export class WakeCycleRunner {
         const { config, ownerConfig, session, activityTracker, interruptHandler, messageQueue } = this.deps;
         this.malformedRequestRefreshAttempted = false;
         this.emptyTurnRetries = 0;
-        this.silenceRecoveryExhausted = false;
         this.recoveryPhase = "Running";
         // --- Capture transcript start lines for post-cycle indexing ---
         // Recovery can rotate the session to a new transcript, so anchor by
@@ -502,6 +513,7 @@ export class WakeCycleRunner {
         // entry.
         const wakeCycleId = generateWakeCycleId();
         /** Numbers model turns within this wake for their task-event keys. */
+        let responseSeq = 0;
         let awaitingUser = false;
         /** The model ended the wake with an accepted finish (not a timeout or exhausted recovery). */
         let wakeFinished = false;
@@ -539,7 +551,10 @@ export class WakeCycleRunner {
             // failover. If the session cannot be rotated, stay on the current engine
             // and try again next wake rather than replaying its history elsewhere.
             if (recovered) {
-                if (!await this.rotateSessionForEngineSwitch(session, String(activeBefore), recovered.recovered, "recovery")) {
+                if (await this.rotateSessionForEngineSwitch(session, String(activeBefore), recovered.recovered, "recovery")) {
+                    contentBlocks = withFreshThreadTaskBlock(contentBlocks);
+                }
+                else {
                     setFallbackEngine(fallbackBefore);
                 }
             }
@@ -577,7 +592,7 @@ export class WakeCycleRunner {
                     // Retry this inner business turn only for transient stream/socket failures.
                     while (transientStreamRetries <= MAX_TRANSIENT_STREAM_RETRIES) {
                         try {
-                            silenceLoop: for (let silenceAttempt = 1; silenceAttempt <= MAX_SILENCE_RETRIES; silenceAttempt++) {
+                            for (let silenceAttempt = 1; silenceAttempt <= MAX_SILENCE_RETRIES; silenceAttempt++) {
                                 const scheduledSilenceMs = SILENCE_TIMEOUT_SCHEDULE_MS[silenceAttempt - 1] ?? SILENCE_TIMEOUT_SCHEDULE_MS[SILENCE_TIMEOUT_SCHEDULE_MS.length - 1];
                                 // Pass the scheduled bound. Pi auto-compaction starts after the
                                 // stream is already open (system/compaction_start); processAgentStream
@@ -600,6 +615,9 @@ export class WakeCycleRunner {
                                         lastReplyText = streamResult.lastAssistantText;
                                         lastReplyPresented = streamResult.lastAssistantTextPresented === true;
                                     }
+                                    // One event per model turn with its complete prose; the task
+                                    // sidecar folds it into progress. No-op without a task runtime.
+                                    recordExecutionEvent(executionEvents.response(`${wakeCycleId}:${responseSeq++}`, streamResult.assistantText));
                                     // A successful SDK envelope can still contain a legitimate empty
                                     // model turn. Apply the same task-state policy as SDK diagnostics.
                                     if (!this.deps.singleShot && streamResult.emptyTurn && !streamResult.finishCalled && !streamResult.toolTimeout) {
@@ -640,37 +658,22 @@ export class WakeCycleRunner {
                                                 `skipping the remaining ${MAX_SILENCE_RETRIES - silenceAttempt} silence retries`);
                                         }
                                     }
-                                    // silenceAttempt counts the tries already made in this call. A dead
-                                    // engine does not get the remaining ones: that is Stopping, not
-                                    // another Recovering retry.
-                                    const silencePhase = nextRecoveryPhase({
-                                        from: this.recoveryPhase,
-                                        bucket: "silence",
-                                        spent: engineAnswersProbe ? silenceAttempt : MAX_SILENCE_RETRIES,
-                                    });
-                                    if (silencePhase === "Recovering")
-                                        this.recoveryPhase = "Recovering";
-                                    if (silencePhase === "Recovering" && engineAnswersProbe) {
+                                    if (silenceAttempt < MAX_SILENCE_RETRIES && engineAnswersProbe) {
                                         const nextTimeoutMs = SILENCE_TIMEOUT_SCHEDULE_MS[silenceAttempt] ?? SILENCE_TIMEOUT_SCHEDULE_MS[SILENCE_TIMEOUT_SCHEDULE_MS.length - 1];
                                         const nextEffort = tierEffort
                                             ? getLlmRetryEffort(tierEffort, silenceAttempt)
                                             : effortSchedule[silenceAttempt];
                                         logger.system(`Stream silence timeout after ${silenceTimeoutMs / 1000}s (attempt ${silenceAttempt}/${MAX_SILENCE_RETRIES}) — engine still answering, retrying LLM call with ${nextTimeoutMs / 1000}s timeout and effort=${nextEffort}...`);
-                                        // Re-send the same messageContent on the next attempt of this call.
-                                        continue silenceLoop;
+                                        // Re-send the same messageContent on next iteration
+                                        continue;
                                     }
                                     // Retries exhausted, or cut short because the engine failed its
-                                    // health probe. Both are Stopping for this call. A hot engine
-                                    // switch or a successful compaction below continues this wake, so
-                                    // the outer handler is only blocked once this call is actually
-                                    // abandoned.
+                                    // health probe.
                                     logger.err(engineAnswersProbe
                                         ? `Stream silence timeout: all ${MAX_SILENCE_RETRIES} attempts exhausted — aborting`
                                         : `Stream silence timeout on attempt ${silenceAttempt}/${MAX_SILENCE_RETRIES} with a failed health probe — aborting`);
-                                    if (this.deps.singleShot) {
-                                        this.silenceRecoveryExhausted = true;
+                                    if (this.deps.singleShot)
                                         throw streamErr;
-                                    }
                                     const locale = resolveLocale(ownerConfig.language);
                                     // Report what actually happened: a probe-driven short circuit may
                                     // have spent only one of the three attempts, and telling the owner
@@ -687,6 +690,7 @@ export class WakeCycleRunner {
                                     const degraded = await this.degradeActiveEngine(engineFallback, "silence_exhausted", session);
                                     if (degraded === "hot") {
                                         consecutiveErrors = 0;
+                                        contentBlocks = withFreshThreadTaskBlock(contentBlocks);
                                         continue wakeLoop;
                                     }
                                     if (degraded === "restarting") {
@@ -705,6 +709,9 @@ export class WakeCycleRunner {
                                         try {
                                             await activeSession.requestCompaction();
                                             consecutiveErrors = 0;
+                                            // The compacted session must see the tasks as the compaction
+                                            // reconcile just left them, not the wake-start block (#1996).
+                                            contentBlocks = withFreshThreadTaskBlock(contentBlocks);
                                             continue wakeLoop;
                                         }
                                         catch (compactErr) {
@@ -721,7 +728,6 @@ export class WakeCycleRunner {
                                         ...taskContinuationRouting(this.deps.agentState.currentTaskReplyTarget),
                                     });
                                     logger.system("Enqueued interrupted-task recovery message for next wake cycle");
-                                    this.silenceRecoveryExhausted = true;
                                     throw streamErr; // propagate so the outer handler records the error
                                 }
                             }
@@ -729,18 +735,7 @@ export class WakeCycleRunner {
                         }
                         catch (streamErr) {
                             // TODO: guard retry against simultaneous stop requests and settle the failed input generator before retrying; both are rare stream/stop races.
-                            // transientStreamRetries counts the disconnects already retried in
-                            // this turn. Another one is Recovering; the ceiling throws, and
-                            // the outer handler records it instead of retrying this turn.
-                            const networkPhase = isTransientStreamFailure(streamErr)
-                                ? nextRecoveryPhase({
-                                    from: this.recoveryPhase,
-                                    bucket: "network",
-                                    spent: transientStreamRetries,
-                                })
-                                : null;
-                            if (networkPhase === "Recovering") {
-                                this.recoveryPhase = "Recovering";
+                            if (isTransientStreamFailure(streamErr) && transientStreamRetries < MAX_TRANSIENT_STREAM_RETRIES) {
                                 transientStreamRetries++;
                                 logger.system(`Transient stream failure (retry ${transientStreamRetries}/${MAX_TRANSIENT_STREAM_RETRIES}) — retrying the current business turn...`);
                                 continue;
@@ -818,16 +813,7 @@ export class WakeCycleRunner {
                     if (!this.deps.singleShot &&
                         endedWithoutLifecycleAction &&
                         runningSubagents.length === 0) {
-                        // prematureEndContinuations counts the continuations already sent in
-                        // this wake. One more is Recovering; the ceiling is Stopping, and
-                        // this wake ends instead of starting another continuation.
-                        const prematurePhase = nextRecoveryPhase({
-                            from: this.recoveryPhase,
-                            bucket: "premature_result",
-                            spent: prematureEndContinuations,
-                        });
-                        if (prematurePhase === "Recovering") {
-                            this.recoveryPhase = "Recovering";
+                        if (prematureEndContinuations < MAX_PREMATURE_END_CONTINUATIONS) {
                             prematureEndContinuations++;
                             logger.warn(`Turn ended without finish — continuing (${prematureEndContinuations}/${MAX_PREMATURE_END_CONTINUATIONS})`);
                             messageContent = [{
@@ -894,18 +880,9 @@ export class WakeCycleRunner {
                         }
                     }
                     else {
-                        // The persisted count is this chain's own budget. It survives
-                        // wakes and restarts, so a new wake must not refill it.
-                        const toolAbortPhase = nextRecoveryPhase({
-                            from: this.recoveryPhase,
-                            bucket: "tool_abort",
-                            spent: this.toolAbortRecoveries,
-                        });
                         this.toolAbortRecoveryChainId ??= randomUUID();
                         this.toolAbortRecoveries += 1;
-                        const retryable = toolAbortPhase === "Recovering";
-                        if (retryable)
-                            this.recoveryPhase = "Recovering";
+                        const retryable = this.toolAbortRecoveries <= MAX_TOOL_ABORT_RECOVERIES;
                         if (!retryable)
                             this.toolAbortRecoveryExhausted = true;
                         this.persistToolAbortRecoveryChain();
@@ -966,7 +943,7 @@ export class WakeCycleRunner {
                 // Resumes, external wakes and calendars may carry a delivery target
                 // without requesting a reply. Mid-turn work uses explicit notify_user
                 // for its result; a quick acknowledgment does not enable this fallback.
-                const replyMessages = messages.filter((message) => message.channel !== "system" && message.meta?.kind !== EXTERNAL_WAKE_KIND && !isCalendarInterruptMessage(message));
+                const replyMessages = messages.filter((message) => isTaskRelevantMessage(message) && !isCalendarInterruptMessage(message));
                 const fallbackEligible = !this.deps.singleShot &&
                     !streamResult.toolTimeout &&
                     !skipBecauseNotified &&
@@ -1110,14 +1087,21 @@ export class WakeCycleRunner {
                     if (errorResult.kind === "continue_with_message") {
                         consecutiveErrors = errorResult.consecutiveErrors;
                         contentBlocks = [{ type: "text", text: errorResult.message }];
+                        // A rebuilt session starts without the thread's tasks (#1996).
+                        if (errorResult.newSession)
+                            contentBlocks = withFreshThreadTaskBlock(contentBlocks);
                         continue;
                     }
                     if (errorResult.kind === "continue_with_reset") {
                         consecutiveErrors = 0;
+                        contentBlocks = withFreshThreadTaskBlock(contentBlocks);
                         continue;
                     }
                     if (errorResult.kind === "continue_preserve_count") {
                         consecutiveErrors = errorResult.consecutiveErrors;
+                        // Invalid-json recovery may have compacted (#1996): resend the
+                        // current task block. A no-op when nothing changed.
+                        contentBlocks = withFreshThreadTaskBlock(contentBlocks);
                         continue;
                     }
                     consecutiveErrors = errorResult.consecutiveErrors;
@@ -1197,12 +1181,6 @@ export class WakeCycleRunner {
             logger.system(err.message);
             return { kind: "done", consecutiveErrors };
         }
-        if (this.silenceRecoveryExhausted) {
-            // The call already spent its own silence ceiling and moved to Stopping.
-            // Another backoff here would replay that call instead of ending it.
-            logger.system("Stream silence ceiling already spent — ending this wake cycle");
-            return { kind: "done", consecutiveErrors: consecutiveErrors + 1 };
-        }
         const errMsg = err instanceof Error ? err.message : String(err);
         const category = classifyError(err);
         // A local coordination error never reached a provider, so it must not
@@ -1470,15 +1448,16 @@ export class WakeCycleRunner {
                 return { kind: "continue_preserve_count", consecutiveErrors };
             }
             // Retries exhausted: tell the owner what actually broke, naming the
-            // local cause instead of blaming the model provider. The wake ends here.
-            // Sleeping would keep the foreground slot busy until the timer fired.
+            // local cause instead of blaming the model provider.
+            const sleepMin = FATAL_SLEEP_MS / 60_000;
             logger.warn(`Notifying owner about persistent local coordination error: ${errMsg}`);
             try {
-                await notifyOwner(formatOwnerErrorNotice(category, consecutiveErrors, errMsg), config);
+                await notifyOwner(formatOwnerErrorNotice(category, consecutiveErrors, sleepMin, errMsg), config);
             }
             catch (notifyErr) {
                 logger.warn(`Failed to send error notification to owner: ${notifyErr instanceof Error ? notifyErr.message : String(notifyErr)}`);
             }
+            await new Promise((r) => setTimeout(r, FATAL_SLEEP_MS));
             return { kind: "done", consecutiveErrors };
         }
         // --- Known malformed Responses replay ---
@@ -1517,7 +1496,7 @@ export class WakeCycleRunner {
             }
             logger.err(`Malformed Responses replay recovery failed — ending wake: ${errMsg}`);
             try {
-                await notifyOwner(formatOwnerErrorNotice(category, consecutiveErrors, errMsg), config);
+                await notifyOwner(formatOwnerErrorNotice(category, consecutiveErrors, 0, errMsg), config);
             }
             catch (notifyErr) {
                 logger.warn(`Failed to send error notification to owner: ${notifyErr instanceof Error ? notifyErr.message : String(notifyErr)}`);
@@ -1546,7 +1525,7 @@ export class WakeCycleRunner {
             if (phase === "Stopping") {
                 logger.warn(`Empty-turn continuation did not confirm task completion: ${errMsg}`);
                 try {
-                    await notifyOwner(formatOwnerErrorNotice(category, this.emptyTurnRetries, errMsg), config);
+                    await notifyOwner(formatOwnerErrorNotice(category, this.emptyTurnRetries, 0, errMsg), config);
                 }
                 catch (notifyErr) {
                     logger.warn(`Failed to send empty-turn notification to owner: ${notifyErr instanceof Error ? notifyErr.message : String(notifyErr)}`);
@@ -1615,22 +1594,10 @@ export class WakeCycleRunner {
                 logger.system("No alternate engine available for transient network failures; keeping consecutiveErrors and the cooldown");
             }
         }
-        // consecutiveErrors counts the failures already seen in this wake. The
-        // notify threshold is the unknown bucket, not the network disconnect
-        // ceiling. Reaching it is Stopping: notify, then end the wake. The next
-        // attempt waits for a queued message or the proactive timer, not a sleep
-        // that holds this slot.
-        const unknownPhase = category === "fatal"
-            ? "Stopping"
-            : nextRecoveryPhase({
-                from: this.recoveryPhase,
-                bucket: "unknown",
-                spent: consecutiveErrors,
-            });
-        if (unknownPhase === "Recovering")
-            this.recoveryPhase = "Recovering";
-        if (unknownPhase === "Stopping") {
-            const notice = formatOwnerErrorNotice(category, consecutiveErrors);
+        if (category === "fatal" ||
+            consecutiveErrors >= MAX_RETRIES_BEFORE_NOTIFY) {
+            const sleepMin = FATAL_SLEEP_MS / 60_000;
+            const notice = formatOwnerErrorNotice(category, consecutiveErrors, sleepMin);
             logger.warn(`Notifying owner about repeated agent error: ${errMsg}`);
             try {
                 await notifyOwner(notice, config);
@@ -1639,6 +1606,7 @@ export class WakeCycleRunner {
                 logger.warn(`Failed to send error notification to owner: ${notifyErr instanceof Error ? notifyErr.message : String(notifyErr)}`);
             }
             consecutiveErrors = 0;
+            await new Promise((r) => setTimeout(r, FATAL_SLEEP_MS));
         }
         else {
             try {
@@ -1717,15 +1685,8 @@ export class WakeCycleRunner {
         // Two recovery turns total: repair, then one refresh. The second branch is
         // gated by the ceiling, so raising MAX_OUTPUT_RECOVERY_ATTEMPTS is what
         // allows another turn. Advancing before I/O keeps a failed repair from
-        // resetting the allowance. The two kinds keep separate counts.
-        const outputPhase = nextRecoveryPhase({
-            from: this.recoveryPhase,
-            bucket: recoveryKind === "policy_refused" ? "policy_refusal" : "oversized",
-            spent: attempts[recoveryKind],
-        });
-        if (outputPhase === "Recovering")
-            this.recoveryPhase = "Recovering";
-        if (attempts[recoveryKind] === 0 && outputPhase === "Recovering") {
+        // resetting the allowance.
+        if (attempts[recoveryKind] === 0) {
             attempts[recoveryKind]++;
             if (!compactionAlreadyFailed) {
                 try {
@@ -1744,7 +1705,7 @@ export class WakeCycleRunner {
                 logger.system("SDK auto-compaction already failed — skipping same-session transcript repair and rotating context");
             }
         }
-        if (attempts[recoveryKind] < MAX_OUTPUT_RECOVERY_ATTEMPTS && outputPhase === "Recovering") {
+        if (attempts[recoveryKind] < MAX_OUTPUT_RECOVERY_ATTEMPTS) {
             attempts[recoveryKind]++;
             const previousId = session.activeSession.getSessionId();
             try {