specrails-desktop 2.36.0 → 2.42.2

package/server/dist/loop-run-manager.js

@@ -1,27 +1,28 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LoopRunManager = void 0;
 exports.truncate = truncate;
 exports.resolveRunVars = resolveRunVars;
 exports.extractChangeId = extractChangeId;
 exports.recoverOrphanLoopStepAccounting = recoverOrphanLoopStepAccounting;
-const tree_kill_1 = __importDefault(require("tree-kill"));
+const multi_repo_execution_store_1 = require("./multi-repo-execution-store");
+const win_spawn_1 = require("./util/win-spawn");
 const db_1 = require("./db");
 const interactive_job_session_1 = require("./interactive-job-session");
 const loop_graph_1 = require("./loop-graph");
 const loop_command_catalog_1 = require("./loop-command-catalog");
 const loop_constants_1 = require("./loop-constants");
+const loop_step_idle_1 = require("./loop-step-idle");
 const loop_decider_1 = require("./loop-decider");
 const loop_runs_store_1 = require("./loop-runs-store");
 const ai_invocations_1 = require("./ai-invocations");
+const provider_limit_1 = require("./provider-limit");
 const ids_1 = require("./ids");
 const providers_1 = require("./providers");
 const runtime_1 = require("./providers/runtime");
 const result_event_1 = require("./result-event");
 const rails_store_1 = require("./rails-store");
+const verification_sentinel_1 = require("./verification-sentinel");
 const IMPLEMENT_CMD_TOKEN_RE = /\{\{\s*cmd:(?:implement|batch)\s*\}\}/;
 const IMPLEMENT_CMD_TEXT_RE = /(?:^|\s)(?:\/(?:specrails:|sr:)?(?:implement|batch-implement)|\$(?:implement|batch-implement))\b/;
 const LOOP_BLOCKED_LINE_RE = /(?:^|\n)\s*LOOP_BLOCKED:\s*(.+?)(?:\n|$)/i;
@@ -53,6 +54,44 @@ function aiStepBlockedReason(text) {
     }
     return null;
 }
+/** The interactive step's timeout failure reason (matched by isResumeMiss). */
+const AI_STEP_TIMEOUT_ERROR = 'AI step timed out';
+/** Whole-step zero-work judgment. The interactive path reports it
+ *  authoritatively (`zeroWork` set at session settle); the one-shot path is
+ *  derived from the result's accumulated signals — one-shot `text` accumulates
+ *  ONLY from assistant text-delta events, so non-empty text ⇒ assistant events
+ *  were seen. A hard-failed one-shot (non-zero exit) is a crash, not zero-work. */
+function aiStepZeroWork(res) {
+    if (res.zeroWork !== undefined)
+        return res.zeroWork;
+    if (res.failed)
+        return false;
+    return (0, interactive_job_session_1.isZeroWorkSettle)({
+        numTurns: res.numTurns ?? 0,
+        tokensIn: res.tokensIn ?? 0,
+        tokensOut: res.tokensOut ?? 0,
+        tokensCacheRead: res.tokensCacheRead ?? 0,
+        tokensCacheCreate: res.tokensCacheCreate ?? 0,
+        sawAssistantEvent: res.text.trim().length > 0,
+        resultText: res.text.trim() ? res.text : null,
+    });
+}
+/** A RESUMED step whose settle is zero-work for a reason other than an
+ *  unresolvable command or the step timeout: the CLI answered with session
+ *  bookkeeping (no model call — a transcript the resume could not load, a
+ *  session torn down mid-flight) and the prompt never ran. Worth exactly one
+ *  fresh retry; an `Unknown command:` would fail again identically. */
+function isResumeMiss(res) {
+    if (res.stalled)
+        return false;
+    if (!aiStepZeroWork(res))
+        return false;
+    if (interactive_job_session_1.UNKNOWN_COMMAND_RE.test(res.text.trim()))
+        return false;
+    if (res.errorText === AI_STEP_TIMEOUT_ERROR)
+        return false;
+    return true;
+}
 const HISTORY_MAX_CHARS = 1500;
 /** Caps for the `loop_step` detail fields (`template` / `command`) — the
  *  payload persists per event, so a runaway prompt must not bloat the events
@@ -85,7 +124,7 @@ const RUN_TOKEN_RE = /\{\{\s*run\.(\w+)\s*\}\}/g;
 /** First ACTIVE `openspec/changes/<id>` path mentioned in a step's output.
  *  Archive paths (`openspec/changes/archive/...`) are historical destinations,
  *  not runnable change ids for later `{{run.changeId}}` commands. */
-const CHANGE_ID_RE = /openspec\/changes\/(?!archive(?:\/|$))([A-Za-z0-9._-]+)(?=\/|\s|$)/g;
+const CHANGE_ID_RE = /openspec[\\/]+changes[\\/]+(?!archive(?:[\\/]|$))([A-Za-z0-9._-]+)(?=[\\/]|\s|$)/g;
 /** Replace `{{run.<name>}}` with the captured value; uncaptured → '' (never a
  *  leaked literal token). Applied AFTER `{{cmd:*}}` and `{{spec.*}}`. */
 function resolveRunVars(text, vars) {
@@ -318,6 +357,7 @@ class LoopRunManager {
     broadcast;
     executors;
     now;
+    options;
     _disposed = false;
     _cancelled = new Set();
     /** The currently-spawned child per run, so cancel/stop can actually KILL a
@@ -331,11 +371,15 @@ class LoopRunManager {
     _interactiveSteps = new Map();
     _activeStepRecovery = new Map();
     _pausedHumanDecisions = new Map();
-    constructor(db, broadcast, executors, now = () => Date.now()) {
+    constructor(db, broadcast, executors, now = () => Date.now(), 
+    /** `idleTimeoutMs`: override the env-resolved loop-step idle watchdog
+     *  (tests). Undefined ⇒ resolveLoopStepIdleTimeoutMs() per run. */
+    options = {}) {
         this.db = db;
         this.broadcast = broadcast;
         this.executors = executors;
         this.now = now;
+        this.options = options;
     }
     /** WebSocket delivery is advisory. A disconnected/misbehaving listener must
      * never change launch, accounting, traversal, or terminal outcomes. */
@@ -365,7 +409,7 @@ class LoopRunManager {
         const child = this._activeChild.get(runId);
         if (child?.pid) {
             try {
-                (0, tree_kill_1.default)(child.pid, 'SIGKILL', () => { });
+                (0, win_spawn_1.treeKillSafe)(child.pid, 'SIGKILL', () => { });
             }
             catch { /* already gone */ }
         }
@@ -473,7 +517,7 @@ class LoopRunManager {
         for (const child of this._activeChild.values()) {
             if (child?.pid) {
                 try {
-                    (0, tree_kill_1.default)(child.pid, 'SIGKILL', () => { });
+                    (0, win_spawn_1.treeKillSafe)(child.pid, 'SIGKILL', () => { });
                 }
                 catch { /* gone */ }
             }
@@ -489,6 +533,24 @@ class LoopRunManager {
         this._pausedHumanDecisions.clear();
     }
     async run(req) {
+        // Later fresh steps must verify the admitted spec, even if its caller or
+        // backlog changes while the implementation is running.
+        req = { ...req, graph: structuredClone(req.graph) };
+        if (req.spec)
+            req = { ...req, spec: structuredClone(req.spec) };
+        if (req.executionManifest)
+            req = { ...req, executionManifest: structuredClone(req.executionManifest) };
+        (0, loop_graph_1.assertLoopShellRepositoryScope)(req.graph, req.executionManifest?.selectedRepositoryIds ?? (req.repositoryId ? [req.repositoryId] : []));
+        if (req.executionManifest) {
+            for (const node of req.graph.nodes) {
+                if (node.type !== 'shell')
+                    continue;
+                const repositoryId = node.data?.repositoryId;
+                if (typeof repositoryId !== 'string' || !req.executionManifest.repositories.some((repo) => repo.repositoryId === repositoryId)) {
+                    throw new Error(`Shell step ${node.id} requires an explicit selected repositoryId for this execution`);
+                }
+            }
+        }
         if (this._disposed)
             throw new Error('LoopRunManager is shut down');
         const adapter = (0, providers_1.getAdapter)(req.provider);
@@ -512,6 +574,14 @@ class LoopRunManager {
         const aiStepTimeoutMs = req.graph.config.aiStepTimeoutMinutes != null
             ? req.graph.config.aiStepTimeoutMinutes * 60_000
             : undefined;
+        // Inactivity bound for every AI step (loop-step-idle): an UNTIMED step is
+        // still bounded by silence, so a wedged provider never leaves the run
+        // `running` forever. 0 ⇒ disabled.
+        const idleTimeoutMs = this.options.idleTimeoutMs ?? (0, loop_step_idle_1.resolveLoopStepIdleTimeoutMs)({
+            ...process.env,
+            // Preserve installations configured with the previous watchdog name.
+            SPECRAILS_LOOP_STEP_IDLE_TIMEOUT_MS: process.env.SPECRAILS_LOOP_STEP_IDLE_TIMEOUT_MS ?? process.env.SPECRAILS_LOOP_INACTIVITY_MS,
+        });
         // Optional cost cap (USD). Like maxIterations/timeout, it's a BETWEEN-STEPS
         // guard: per-step cost is only known when that step's process exits, so the
         // loop is stopped before the NEXT step once the accumulated total crosses the
@@ -549,6 +619,8 @@ class LoopRunManager {
                 iterationLimit: maxIterations,
                 startedAt: launchStartedAt,
             });
+            if (req.executionManifest)
+                this.db.prepare('UPDATE loop_runs SET execution_manifest = ? WHERE id = ?').run(JSON.stringify(req.executionManifest), runId);
             (0, db_1.createJob)(this.db, {
                 id: runId,
                 command: jobCommand,
@@ -598,8 +670,9 @@ class LoopRunManager {
         // emission (see LoopStepEventPayload).
         const emitStep = (kind, title, nodeId, iteration, detail) => {
             stepNum += 1;
-            openStep = { index: stepNum, nodeId, startMs: this.now() };
-            const payload = { index: stepNum, kind, title, nodeId, iteration, ...(detail ?? {}) };
+            const startedAtMs = this.now();
+            openStep = { index: stepNum, nodeId, startMs: startedAtMs };
+            const payload = { index: stepNum, kind, title, nodeId, iteration, startedAtMs, ...(detail ?? {}) };
             emitRunEvent('loop_step', payload);
             logLine(`\n━━━━━━ Step ${stepNum} · ${title} ━━━━━━`);
         };
@@ -619,6 +692,8 @@ class LoopRunManager {
                 exitCode: input.exitCode ?? null,
                 durationMs: input.durationMs ?? Math.max(0, this.now() - openStep.startMs),
                 ...(input.decision ? { decision: input.decision } : {}),
+                ...(input.reason ? { reason: input.reason } : {}),
+                ...(input.idleMs != null ? { idleMs: input.idleMs } : {}),
             };
             openStep = null;
             emitRunEvent('loop_step_end', payload);
@@ -691,6 +766,8 @@ class LoopRunManager {
         const byId = (0, loop_graph_1.nodesById)(req.graph);
         const start = (0, loop_graph_1.findStartNode)(req.graph);
         let outcome = 'failed';
+        let stallReason;
+        let providerLimitHit;
         let iteration = 0;
         let totalCost = 0;
         let totalTokens = 0;
@@ -720,6 +797,10 @@ class LoopRunManager {
         // RESUMED steps already carry prior context in their session, so re-appending
         // the history there is redundant tokens; this flag keeps it off for them.
         let justContinued = false;
+        // Transport/test failures are engine facts. A model's STOP cannot turn a
+        // failed pass (or a failed final archive command) into a successful run.
+        // A continue starts a fresh repair pass which can establish success again.
+        let passFailed = false;
         // Fail-open honesty: flips true the first time a cost-bearing step yields no
         // priced cost while a cap is set (non-Claude estimate / unknown model / a
         // failed step) → the cap can under-count, so we warn once in the run log.
@@ -930,7 +1011,7 @@ class LoopRunManager {
                     outcome = 'stopped';
                     break;
                 }
-                if (this.now() > deadline) {
+                if (this.now() >= deadline) {
                     outcome = 'failed';
                     break;
                 }
@@ -962,17 +1043,23 @@ class LoopRunManager {
                         nodeId = succs[0]?.id;
                         break;
                     case 'end':
-                        outcome = node.data?.outcome === 'failure' ? 'failed' : 'success';
+                        outcome = node.data?.outcome === 'failure' || passFailed ? 'failed' : 'success';
                         settled = true;
                         break;
                     case 'ai-step': {
+                        // Some roles must inspect the previous step without inheriting its
+                        // provider conversation (notably Revision's mutator → reviewer
+                        // boundary). The node still receives durable disk state, its fully
+                        // rendered prompt, and bounded prior-step history.
+                        if (node.data?.freshSession === true)
+                            aiSessionId = undefined;
                         // Expand magic commands FIRST — `{{cmd:implement}}` becomes the
                         // NATIVE per-provider invocation (claude `/specrails:implement #<id>
                         // --yes`, codex `$implement #<id> --yes`) — then resolve `{{spec.*}}`
                         // data tokens and finally `{{const:*}}` library constants.
                         const rawTemplate = String(node.data?.prompt ?? '');
                         const expanded = (0, loop_constants_1.resolveConstants)(resolveRunVars((0, loop_graph_1.interpolateSpec)((0, loop_command_catalog_1.expandCommands)(rawTemplate, { provider: nodeProvider, ticketIds: req.spec?.ticketIds, specId: req.spec?.id }), req.spec), runVars), constMap);
-                        const base = withReviewContinuationContext(expanded, rawTemplate, req.spec);
+                        const base = [(0, multi_repo_execution_store_1.executionManifestPrompt)(req.executionManifest), withReviewContinuationContext(expanded, rawTemplate, req.spec)].filter(Boolean).join('\n\n');
                         // Inject the cross-iteration history only when there's no live session
                         // to carry it (a fresh pass) OR right after a Decider 'continue' (so the
                         // step sees the verdict). A mid-body resumed step already has it.
@@ -984,69 +1071,171 @@ class LoopRunManager {
                         // were noise once the step explorer landed; the log now opens on real
                         // output). `template` is omitted when rendering changed nothing (a
                         // plain free-text prompt would just duplicate `command`).
-                        emitStep('ai-step', `🤖 ${nodeLabel || 'AI Step'} (${nodeProvider}/${nodeModel}${nodeEffort ? `, effort: ${nodeEffort}` : ''})`, node.id, iteration + 1, {
+                        const stepTitle = `🤖 ${nodeLabel || 'AI Step'} (${nodeProvider}/${nodeModel}${nodeEffort ? `, effort: ${nodeEffort}` : ''})`;
+                        const stepDetail = {
                             ...(rawTemplate && rawTemplate !== base ? { template: capText(rawTemplate, STEP_TEMPLATE_CAP) } : {}),
                             ...(base ? { command: capText(base, STEP_COMMAND_CAP) } : {}),
-                        });
+                        };
+                        emitStep('ai-step', stepTitle, node.id, iteration + 1, stepDetail);
                         // BUG-32: capture the REAL start instant BEFORE the await — the row is
                         // bucketed/ordered by started_at, which must be the turn-start, not the
                         // finish time (this.now() evaluated after the await would be the finish).
-                        const aiStepStart = new Date(this.now()).toISOString();
-                        const aiRecoveryKey = stageAiAccounting('ai', node.id, `loop:${runId}`, aiStepStart);
+                        let aiStepStart = new Date(this.now()).toISOString();
+                        let aiRecoveryKey = stageAiAccounting('ai', node.id, `loop:${runId}`, aiStepStart);
                         // Interactive upgrade (default for persistent-stdin providers): the
                         // executors return a resident-session plan when the kill-switch is on
                         // and the provider is capable (claude); null falls through to the
                         // byte-identical one-shot spawn. The plan mirrors the one-shot's
                         // cwd/env/argv derivation; the engine owns the session lifecycle.
-                        const interactivePlan = this.executors.planInteractiveAiStep?.({
-                            provider: nodeProvider,
-                            model: nodeModel,
-                            effort: nodeEffort,
-                            cwd: req.cwd,
-                            repoDir: req.repoDir,
-                            sessionId: aiSessionId,
-                            aiStepTimeoutMs,
-                        }) ?? null;
-                        const res = interactivePlan
-                            ? await this._runInteractiveAiStep({
-                                runId,
-                                projectId: req.projectId,
-                                plan: interactivePlan,
-                                prompt,
-                                fallbackModel: nodeModel,
-                                nextEventSeq: takeSeq,
-                                recoveryStepKey: aiRecoveryKey,
-                            })
-                            : await this.executors.runAiStep({ prompt, sessionId: aiSessionId, provider: nodeProvider, model: nodeModel, effort: nodeEffort, cwd: req.cwd, repoDir: req.repoDir, onLine: logLine, onRawLine, onSpawn: (c) => this._activeChild.set(runId, c), aiStepTimeoutMs });
+                        // One attempt of this step. `sessionId` is the ONLY thing that
+                        // differs between the resumed attempt and its fresh retry below.
+                        const runAttempt = async (sessionId) => {
+                            const remainingMs = deadline - this.now();
+                            const effectiveTimeoutMs = Number.isFinite(remainingMs)
+                                ? Math.max(1, Math.min(aiStepTimeoutMs === 0 ? Infinity : (aiStepTimeoutMs ?? 15 * 60_000), remainingMs))
+                                : aiStepTimeoutMs;
+                            const coreRun = { runId, spec: req.spec, goal: req.spec ? undefined : JSON.stringify({ loop: req.loopName, graph: req.graph }), repositoryId: req.repositoryId, verificationStep: node.data?.requireVerificationPass === true || /\{\{\s*cmd:(?:verify|revision-verify|opsx:verify)\s*\}\}/.test(rawTemplate) };
+                            const interactivePlan = this.executors.planInteractiveAiStep?.({
+                                coreRun,
+                                provider: nodeProvider,
+                                model: nodeModel,
+                                effort: nodeEffort,
+                                profileName: req.profileName,
+                                cwd: req.cwd,
+                                repoDir: req.repoDir, executionManifest: req.executionManifest,
+                                sessionId,
+                                aiStepTimeoutMs: effectiveTimeoutMs,
+                                idleTimeoutMs,
+                            }) ?? null;
+                            return interactivePlan
+                                ? this._runInteractiveAiStep({
+                                    runId,
+                                    projectId: req.projectId,
+                                    plan: interactivePlan,
+                                    prompt: [interactivePlan.promptPrefix, prompt].filter(Boolean).join('\n\n'),
+                                    fallbackModel: nodeModel,
+                                    nextEventSeq: takeSeq,
+                                    recoveryStepKey: aiRecoveryKey,
+                                })
+                                : this.executors.runAiStep({ coreRun, prompt, sessionId, provider: nodeProvider, model: nodeModel, effort: nodeEffort, profileName: req.profileName, cwd: req.cwd, repoDir: req.repoDir, executionManifest: req.executionManifest, onLine: logLine, onRawLine, onSpawn: (c) => this._activeChild.set(runId, c), aiStepTimeoutMs: effectiveTimeoutMs, idleTimeoutMs });
+                        };
+                        let res = await runAttempt(aiSessionId);
                         if (this._disposed)
                             return neverAfterDispose();
                         this._activeChild.delete(runId);
+                        // Idle stall (loop-step-idle): the provider produced no stream
+                        // activity for the idle budget and the watchdog tore the step down.
+                        // Close this attempt as `stalled`, then retry the SAME step ONCE by
+                        // resuming its session (the worktree + transcript carry the partial
+                        // work). A second stall fails the step and settles the run `stalled`
+                        // instead of leaving it `running` forever. A user cancel is not a
+                        // stall — the loop head settles it 'stopped' below.
+                        if (res.stalled && !this._cancelled.has(runId)) {
+                            const stalledSession = res.sessionId ?? aiSessionId;
+                            const idleMin = Math.max(1, Math.round(idleTimeoutMs / 60_000));
+                            emitStepEnd({ status: 'stalled', reason: 'idle_timeout', idleMs: idleTimeoutMs, durationMs: res.durationMs });
+                            // Each provider process owns independent accounting and recovery
+                            // frontiers. Settle the first attempt before staging its retry.
+                            record(`loop:${runId}`, { ...res, failed: true }, aiStepStart, aiRecoveryKey);
+                            this._activeStepRecovery.delete(runId);
+                            if (this.now() >= deadline || (maxCostUsd !== undefined && totalCost >= maxCostUsd)) {
+                                outcome = this.now() >= deadline ? 'failed' : 'max-cost';
+                                logLine('■ Step stalled; the run deadline or cost cap prevents another attempt.', 'stderr');
+                                settled = true;
+                                break;
+                            }
+                            logLine(`↻ Step stalled — no provider output for ${idleMin} min. Retrying this step once${stalledSession ? ` by resuming session ${stalledSession}` : ''}.`, 'stderr');
+                            emitStep('ai-step', stepTitle, node.id, iteration + 1, { ...stepDetail, attempt: 2 });
+                            aiStepStart = new Date(this.now()).toISOString();
+                            aiRecoveryKey = stageAiAccounting('ai', node.id, `loop:${runId}`, aiStepStart);
+                            res = await runAttempt(stalledSession);
+                            if (this._disposed)
+                                return neverAfterDispose();
+                            this._activeChild.delete(runId);
+                            if (res.stalled && !this._cancelled.has(runId)) {
+                                emitStepEnd({ status: 'stalled', reason: 'idle_timeout', idleMs: idleTimeoutMs, durationMs: res.durationMs });
+                                logLine(`■ Step stalled twice — no provider output for ${idleMin} min on the resumed attempt either; stopping the run.`, 'stderr');
+                                history.push(`AI Step: ${truncate(res.text)}`);
+                                record(`loop:${runId}`, { ...res, failed: true }, aiStepStart, aiRecoveryKey);
+                                this._activeStepRecovery.delete(runId);
+                                stallReason = 'idle_timeout';
+                                outcome = 'stalled';
+                                settled = true;
+                                break;
+                            }
+                        }
+                        // Resume miss (loop run 5c958db2): a step that `--resume`d the
+                        // previous step's session and settled with ZERO work — no model
+                        // call, no `Unknown command:` text — was answered by the CLI's
+                        // session bookkeeping (a transcript the resume could not load, a
+                        // session torn down mid-flight), never by the model. The prompt did
+                        // not run, so retry it ONCE in a fresh session instead of failing
+                        // the step and spinning the loop through verify/fix on nothing.
+                        // Unknown commands, timeouts and cancels are NOT resume misses.
+                        if (aiSessionId && !this._cancelled.has(runId) && isResumeMiss(res)) {
+                            logLine(`↻ Resuming session ${aiSessionId} produced no work — retrying this step in a fresh session.`, 'stderr');
+                            aiSessionId = undefined;
+                            res = await runAttempt(undefined);
+                            if (this._disposed)
+                                return neverAfterDispose();
+                            this._activeChild.delete(runId);
+                        }
                         // Zero-work strictness (run 01f41203): a settle that consumed NO
                         // model work — the claude CLI's synthetic `Unknown command:` result
                         // frame (num_turns 0, no assistant events, zero usage tokens) —
                         // means the step's command never actually ran. A step that didn't
                         // run is FAILED, never 'ok'. The interactive path evaluates the
                         // predicate at session settle (res.zeroWork set); for the one-shot
-                        // path the engine derives it here from the result's accumulated
-                        // signals (text accumulates ONLY from assistant text-delta events,
-                        // so non-empty text ⇒ assistant events were seen).
-                        const oneShotZeroWork = res.zeroWork === undefined && !res.failed && (0, interactive_job_session_1.isZeroWorkSettle)({
-                            numTurns: res.numTurns ?? 0,
-                            tokensIn: res.tokensIn ?? 0,
-                            tokensOut: res.tokensOut ?? 0,
-                            tokensCacheRead: res.tokensCacheRead ?? 0,
-                            tokensCacheCreate: res.tokensCacheCreate ?? 0,
-                            sawAssistantEvent: res.text.trim().length > 0,
-                            resultText: res.text.trim() ? res.text : null,
-                        });
-                        const zeroWork = res.zeroWork === true || oneShotZeroWork;
+                        // path the engine derives it from the result's accumulated signals
+                        // (aiStepZeroWork).
+                        const zeroWork = aiStepZeroWork(res);
+                        const oneShotZeroWork = res.zeroWork === undefined && zeroWork;
+                        // Provider usage/rate limit (run 52124009): the CLI answers with the
+                        // limit notice as the reply. Iterating on it burns cycles and hides
+                        // the real cause — stop the run at once with the provider's own
+                        // words and the reset hint, and tell every surface.
+                        const providerLimit = !this._cancelled.has(runId)
+                            ? ((0, provider_limit_1.classifyProviderLimit)(res.errorText) ?? (0, provider_limit_1.classifyProviderLimit)(res.text))
+                            : null;
+                        if (providerLimit) {
+                            const described = (0, provider_limit_1.describeProviderLimit)(providerLimit);
+                            emitStepEnd({ status: 'failed', reason: 'provider_limit', durationMs: res.durationMs });
+                            logLine(`■ Provider limit reached (${req.provider}): ${described}. Stopping the loop now — relaunch or Recover & retry after the reset.`, 'stderr');
+                            history.push(`AI Step: ${truncate(res.text)}`);
+                            record(`loop:${runId}`, { ...res, failed: true }, aiStepStart, aiRecoveryKey);
+                            this._activeStepRecovery.delete(runId);
+                            this._emit({
+                                type: 'loop.provider_limit',
+                                projectId: req.projectId,
+                                runId,
+                                provider: req.provider,
+                                kind: providerLimit.kind,
+                                message: providerLimit.message,
+                                resetsAt: providerLimit.resetsAt,
+                                timestamp: new Date().toISOString(),
+                            });
+                            stallReason = 'provider_limit';
+                            providerLimitHit = providerLimit;
+                            outcome = 'stalled';
+                            settled = true;
+                            break;
+                        }
                         const blockedReason = aiStepBlockedReason(res.text);
-                        const stepFailed = res.failed === true || zeroWork || blockedReason !== null;
+                        const requiresVerification = node.data?.requireVerificationPass === true
+                            || /\{\{cmd:(?:verify|revision-verify|opsx:verify)\}\}/.test(rawTemplate);
+                        const verificationFailed = requiresVerification
+                            && (0, verification_sentinel_1.parseVerificationSentinel)(res.text).verdict !== 'pass';
+                        const stepFailed = res.failed === true || zeroWork || blockedReason !== null || verificationFailed;
+                        passFailed ||= res.failed === true || zeroWork || (verificationFailed && !blockedReason);
                         const stepErrorText = res.errorText ?? (zeroWork
                             ? `zero work performed — the command never ran${res.text.trim() ? `: ${res.text.trim()}` : ''}`
                             : blockedReason
                                 ? `blocked — ${blockedReason}`
-                                : undefined);
+                                : verificationFailed
+                                    ? 'verification did not finish with VERIFICATION: PASS'
+                                    : undefined);
+                        if (verificationFailed && !blockedReason)
+                            logLine(`✖ ${stepErrorText}`, 'stderr');
                         // Make the failure reason land visibly INSIDE the step's log
                         // segment (the interactive session already surfaced its own note at
                         // settle; the Template/Command flat-log lines are gone, so without
@@ -1067,7 +1256,9 @@ class LoopRunManager {
                         // would otherwise make the next step `--resume` a dead session.
                         if (res.sessionId && (!stepFailed || (blockedReason !== null && !res.failed && !zeroWork)))
                             aiSessionId = res.sessionId;
-                        history.push(`AI Step: ${truncate(res.text)}`);
+                        else if (stepFailed)
+                            aiSessionId = undefined;
+                        history.push(`AI Step${stepFailed ? ` FAILED (${stepErrorText ?? 'provider invocation failed'})` : ''}: ${truncate(res.text)}`);
                         record(`loop:${runId}`, { ...res, failed: stepFailed }, aiStepStart, aiRecoveryKey);
                         this._activeStepRecovery.delete(runId);
                         if (blockedReason) {
@@ -1101,7 +1292,11 @@ class LoopRunManager {
                         // (its `Unknown command:` text is a CLI synthetic, not model output),
                         // so a persistently-unresolvable command aborts the run identically
                         // instead of grinding to the cap.
-                        if (stepFailed && (zeroWork || !res.text.trim())) {
+                        if (this._cancelled.has(runId)) {
+                            // A user cancel tears the step down with no output — that is not
+                            // a provider failure; the loop head settles the run 'stopped'.
+                        }
+                        else if (stepFailed && (zeroWork || res.resultIsError || !res.text.trim())) {
                             consecutiveAiFailures += 1;
                             if (consecutiveAiFailures >= AI_FAILFAST_THRESHOLD) {
                                 logLine(`Loop aborted: provider appears down — ${consecutiveAiFailures} AI steps failed with no output and no successful AI call in between${stepErrorText ? ` — ${stepErrorText}` : ''}`, 'stderr');
@@ -1138,12 +1333,14 @@ class LoopRunManager {
                         const command = (0, loop_constants_1.resolveConstants)(resolveRunVars((0, loop_graph_1.interpolateSpec)(String(node.data?.command ?? ''), req.spec), runVars), constMap);
                         emitStep('shell', `⚡ ${nodeLabel || 'Shell'}`, node.id, iteration + 1);
                         logLine(`$ ${command}`);
-                        const sh = await this.executors.runShell({ command, cwd: req.cwd, onLine: logLine, onSpawn: (c) => this._activeChild.set(runId, c) });
+                        const shellRepo = req.executionManifest?.repositories.find((repo) => repo.repositoryId === node.data?.repositoryId)?.worktreePath ?? req.repoDir;
+                        const sh = await this.executors.runShell({ command, cwd: req.executionManifest ? shellRepo ?? req.cwd : req.cwd, repoDir: shellRepo, timeoutMs: Math.min(10 * 60_000, Math.max(1, deadline - this.now())), onLine: logLine, onSpawn: (c) => this._activeChild.set(runId, c) });
                         if (this._disposed)
                             return neverAfterDispose();
                         this._activeChild.delete(runId);
                         logLine(`(exit ${sh.exitCode})`);
                         emitStepEnd({ status: sh.exitCode === 0 ? 'ok' : 'failed', exitCode: sh.exitCode, durationMs: sh.durationMs });
+                        passFailed ||= sh.exitCode !== 0;
                         totalDuration += sh.durationMs ?? 0;
                         (0, loop_runs_store_1.updateLoopRunCounters)(this.db, runId, {
                             iterationCount: iteration,
@@ -1169,23 +1366,27 @@ class LoopRunManager {
                         // BUG-32: capture the real Decider start BEFORE the await (see AI step).
                         const deciderStart = new Date(this.now()).toISOString();
                         const deciderRecoveryKey = stageAiAccounting('decider', node.id, `loop:${runId}:decider`, deciderStart);
-                        const dec = await this.executors.runDecider({
+                        let dec = await this.executors.runDecider({
                             systemPrompt: (0, loop_decider_1.buildDeciderSystemPrompt)(),
                             // Give the Decider the spec so it can verify completeness against the
                             // FULL scope instead of trusting a step's self-reported success.
-                            userPrompt: (0, loop_decider_1.buildDeciderUserPrompt)({ goal, history, spec: req.spec ? { title: req.spec.title, description: req.spec.description } : undefined }),
+                            userPrompt: [(0, multi_repo_execution_store_1.executionManifestPrompt)(req.executionManifest), (0, loop_decider_1.buildDeciderUserPrompt)({ goal, history, spec: req.spec })].filter(Boolean).join('\n\n'),
                             provider: nodeProvider,
                             model: nodeModel,
                             effort: nodeEffort,
                             cwd: req.cwd,
-                            repoDir: req.repoDir,
+                            repoDir: req.repoDir, executionManifest: req.executionManifest,
                             onLine: logLine,
                             onRawLine,
                             onSpawn: (c) => this._activeChild.set(runId, c),
+                            timeoutMs: Math.min(3 * 60_000, Math.max(1, deadline - this.now())),
                         });
                         if (this._disposed)
                             return neverAfterDispose();
                         this._activeChild.delete(runId);
+                        if ((!dec.parsed || passFailed) && !dec.blocked && !dec.continue) {
+                            dec = { ...dec, continue: true, reasoning: `A required step failed in this pass; repair and verify before completion. ${dec.reasoning}` };
+                        }
                         // A parseable Decider verdict means this AI invocation (SAME
                         // provider/model) succeeded → the provider is alive, so clear any
                         // ai-step failure streak. This stops a false fail-fast abort when an
@@ -1257,14 +1458,16 @@ class LoopRunManager {
                         // (e.g. verify keeps passing a baseline while the feature is never
                         // written) — abort `stalled` rather than cycle to the cap.
                         if (dec.continue && this.executors.repoStateHash) {
-                            const dir = req.repoDir ?? req.cwd;
-                            const hash = this.executors.repoStateHash(dir);
+                            const dirs = req.executionManifest?.repositories.map((repo) => repo.worktreePath) ?? [req.repoDir ?? req.cwd];
+                            const hashes = dirs.map((dir) => this.executors.repoStateHash(dir));
+                            const hash = hashes.some((value) => value === null) ? null : JSON.stringify(hashes);
                             // null = couldn't read the tree → skip (never let null match null).
                             if (hash !== null) {
                                 if (hash === lastRepoHash) {
                                     consecutiveNoProgress += 1;
                                     if (consecutiveNoProgress >= NO_PROGRESS_LIMIT) {
                                         logLine(`\n■ Loop stalled — ${consecutiveNoProgress + 1} iterations changed nothing in the working tree; stopping instead of cycling.`, 'stderr');
+                                        stallReason = 'no_progress';
                                         outcome = 'stalled';
                                         settled = true;
                                         break;
@@ -1277,9 +1480,17 @@ class LoopRunManager {
                             }
                         }
                         if (dec.continue) {
+                            // Enforce the cap before another expensive implementation pass,
+                            // not after that extra pass reaches the next decider.
+                            if (iteration >= maxIterations) {
+                                outcome = 'max-iterations';
+                                settled = true;
+                                break;
+                            }
                             const target = branchTarget('continue') ?? conts[0]?.id;
-                            if (target) {
+                            if (target && byId.get(target)?.type !== 'end') {
                                 nodeId = target;
+                                passFailed = false;
                                 // The next step acts on this verdict → let it see the history.
                                 justContinued = true;
                                 // New iteration of an iterate-per-item loop → drop the resumed AI
@@ -1291,7 +1502,7 @@ class LoopRunManager {
                                     aiSessionId = undefined;
                             }
                             else {
-                                outcome = 'success';
+                                outcome = 'failed';
                                 settled = true;
                             }
                         }
@@ -1307,7 +1518,14 @@ class LoopRunManager {
                         break;
                     }
                     case 'condition':
-                        // v1: AND/OR branching is not yet evaluated — follow the first edge.
+                        // Multiple branches have no implemented condition evaluator. Never
+                        // silently skip required branches and report a successful run.
+                        if (succs.length !== 1) {
+                            logLine('Condition branching is unsupported; replace it with explicit sequential steps or a Decider.', 'stderr');
+                            outcome = 'failed';
+                            settled = true;
+                            break;
+                        }
                         nodeId = succs[0]?.id;
                         break;
                     default:
@@ -1315,7 +1533,7 @@ class LoopRunManager {
                 }
                 // Ran off the end of the graph without an explicit End node.
                 if (nodeId === undefined && !settled) {
-                    outcome = 'success';
+                    outcome = 'failed';
                     settled = true;
                 }
             }
@@ -1455,6 +1673,8 @@ class LoopRunManager {
             outcome,
             iterations: iteration,
             totalCostUsd: usageTelemetryAvailable ? totalCost : null,
+            ...(outcome === 'stalled' && stallReason ? { stallReason } : {}),
+            ...(providerLimitHit ? { providerLimit: providerLimitHit } : {}),
         };
     }
     /**
@@ -1486,6 +1706,9 @@ class LoopRunManager {
         return new Promise((resolve) => {
             let timer = null;
             let timedOut = false;
+            // Set by the session's wedge detector BEFORE it aborts: tells a stall
+            // (no raw output for the idle budget) apart from any other 'crashed'.
+            let stalled = false;
             const session = new interactive_job_session_1.InteractiveJobSession({
                 jobId: input.runId,
                 projectId: input.projectId,
@@ -1493,8 +1716,13 @@ class LoopRunManager {
                 adapter: input.plan.adapter,
                 broadcast: this.broadcast,
                 settleMode: 'auto',
-                // The step timeout below is the sole watchdog — the one-shot loop path
-                // has no zombie detector either (byte-parity), so none is armed here.
+                // Inactivity bound (loop-step-idle): the session's zombie timer IS the
+                // step's idle watchdog — reset on any raw child output, silence for the
+                // whole budget folds the in-flight turn and settles 'crashed' with the
+                // `stalled` flag set so the engine retries the step once by resume.
+                // Absent (0) ⇒ the step timeout below is the only watchdog.
+                zombieTimeoutMs: input.plan.idleTimeoutMs ?? input.plan.inactivityTimeoutMs ?? 0,
+                onZombieTimeout: () => { stalled = true; },
                 nextEventSeq: input.nextEventSeq,
                 persistTurnUsage: (usage, completedEventSeq, checkpoint) => {
                     const checkpointTurn = this.db.transaction(() => {
@@ -1529,7 +1757,7 @@ class LoopRunManager {
                     // command:` frame) is a FAILED step — the command never ran. Judged
                     // by the session itself over the accumulated totals (whole-session,
                     // so a multi-turn step ending on one synthetic frame is unaffected).
-                    const failed = info.reason === 'crashed' || info.zeroWork;
+                    const failed = info.reason === 'crashed' || info.zeroWork || info.resultIsError;
                     resolve({
                         // The last turn's `result` payload — the same terminal text the
                         // one-shot path captures (feeds history/changeId/fail-fast).
@@ -1554,9 +1782,13 @@ class LoopRunManager {
                         zeroWork: info.zeroWork,
                         errorText: failed
                             ? (info.reason === 'crashed'
-                                ? (timedOut ? 'AI step timed out' : 'interactive step session crashed')
-                                : `zero work performed — the command never ran${info.resultText ? `: ${info.resultText}` : ''}`)
+                                ? (stalled ? loop_step_idle_1.AI_STEP_STALLED_ERROR : timedOut ? AI_STEP_TIMEOUT_ERROR : info.resultIsError && info.resultText ? info.resultText : 'interactive step session crashed')
+                                : info.zeroWork
+                                    ? `zero work performed — the command never ran${info.resultText ? `: ${info.resultText}` : ''}`
+                                    : `provider returned an error result${info.resultText ? ` — ${info.resultText}` : ''}`)
                             : undefined,
+                        ...(stalled && info.reason === 'crashed' ? { stalled: true } : {}),
+                        ...(info.resultIsError ? { resultIsError: true } : {}),
                     });
                 },
             });