teamshare-bridge 0.28.0 → 0.28.3

package/dist/lib/harness/spawn.js

@@ -30,6 +30,7 @@ const line_assembler_1 = require("./line-assembler");
 const opencode_outcome_1 = require("./opencode-outcome");
 const opencode_control_1 = require("./opencode-control");
 const activity_1 = require("./activity");
+const turn_watch_1 = require("./turn-watch");
 const agy_pty_1 = require("./agy-pty");
 const prompt_file_1 = require("./prompt-file");
 const diagnostics_1 = require("../diagnostics");
@@ -38,12 +39,6 @@ const relay_key_1 = require("./relay-key");
 const resume_1 = require("./resume");
 const opencode_session_1 = require("../llm/opencode-session");
 const session_store_1 = require("./session-store");
-/**
- * Maximum prompt length before we switch to temp-file delivery.
- * Windows argv limit is 32,767 chars; we leave headroom for binary path,
- * MCP config, and other flags.
- */
-const PROMPT_FILE_THRESHOLD = 28_000;
 /** Real task ids are UUIDs; synthetic keys (chat-<projectId>) are not reported. */
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 let warnedNoPty = false;
@@ -376,45 +371,56 @@ function spawnHarness(harness, prompt, log, settings = {}) {
         log(`[harness] ${adapter.binary} not found - install it first.`, 'stderr');
         return Promise.resolve(1);
     }
-    // ── Long prompt → temp file delivery (decided once) ────────────────────
-    let promptFile = null;
-    const longPrompt = prompt.length > PROMPT_FILE_THRESHOLD && adapter.supportsPromptFile;
-    if (longPrompt) {
-        promptFile = (0, prompt_file_1.writePromptFile)(prompt);
-        log(`[harness] prompt too long for argv (${prompt.length} chars) — wrote to temp file`, 'stderr');
-    }
-    const buildArgs = (req, promptOverride) => {
+    // ── Long prompt → file delivery (decided PER INVOCATION) ───────────────
+    //
+    // The decision is taken inside buildArgs, from the prompt ACTUALLY being
+    // sent and from the whole assembled command line. It used to be taken once,
+    // above, and then skipped whenever `promptOverride` was passed - and runOnce
+    // ALWAYS passes it (`buildArgs(req, effectivePrompt)`), so the file was
+    // written, logged, and never used: the 62 KB brief still went on argv and
+    // `spawn` died with `ENAMETOOLONG` before the harness started.
+    const promptFiles = [];
+    /**
+     * Write one prompt file. Never throws: a full/read-only disk must produce a
+     * log line, not an exception on a path that is already a last resort.
+     */
+    const openPromptFile = (text) => {
+        try {
+            const file = (0, prompt_file_1.writePromptFile)(text);
+            promptFiles.push(file);
+            return file;
+        }
+        catch (err) {
+            log(`[harness] could not write the prompt file (${msg(err)}) - ` +
+                'falling back to argv, which may exceed the OS command-line limit', 'stderr');
+            return null;
+        }
+    };
+    const buildArgs = (req, promptForRun) => {
         const opts = req
             ? { ...settings, resume: req }
             : { ...settings, resume: undefined };
-        const effectivePrompt = promptOverride ?? prompt;
-        // A continuation prompt is a SHORT follow-up ("your shell finished, carry
-        // on"), so it is always passed on argv - never re-written to the prompt
-        // file, which still holds the original task brief.
-        if (promptFile && !promptOverride) {
-            if (adapter.promptFileFlag) {
-                const a = adapter.headlessArgs(effectivePrompt, opts, true);
-                a.push(adapter.promptFileFlag, promptFile);
-                return a;
-            }
-            const a = adapter.headlessArgs(effectivePrompt, opts, true);
-            const shortInstruction = `Read the file "${promptFile}" and follow its instructions exactly. ` +
-                'Do not respond until you have read the file. ' +
-                'The file contains your complete instructions including project context, tools, and the user message.';
-            if (['claude', 'gemini', 'antigravity', 'copilot', 'cursor'].includes(resolved)) {
-                a.unshift('-p', shortInstruction);
-            }
-            else {
-                a.push(shortInstruction);
-            }
-            return a;
+        const effectivePrompt = promptForRun ?? prompt;
+        const inline = adapter.headlessArgs(effectivePrompt, opts);
+        const commandLineChars = (0, prompt_file_1.argvChars)([
+            bin.file,
+            ...(bin.args ?? []),
+            ...inline,
+        ]);
+        if (!(0, prompt_file_1.shouldUsePromptFile)(adapter, effectivePrompt.length, commandLineChars)) {
+            return inline;
         }
-        return adapter.headlessArgs(effectivePrompt, opts);
+        const file = openPromptFile(effectivePrompt);
+        if (!file)
+            return inline;
+        log(`[harness] prompt too long for argv (${effectivePrompt.length} chars, ` +
+            `~${commandLineChars} char command line) — wrote it to ${file} and ` +
+            'passed the path instead', 'stderr');
+        return (0, prompt_file_1.promptFileArgs)(adapter, adapter.headlessArgs(effectivePrompt, opts, true), file);
     };
     const cleanupPrompt = () => {
-        if (promptFile)
-            (0, prompt_file_1.cleanupPromptFile)(promptFile);
-        promptFile = null;
+        for (const file of promptFiles.splice(0))
+            (0, prompt_file_1.cleanupPromptFile)(file);
     };
     // Shell/subagent lifecycle (2026-10): OpenCode V2 executes `shell` and
     // subagent work on the SHARED SERVICE. Both can outlive the CLI client and
@@ -469,6 +475,17 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                 `run-${Date.now()}`);
             if (scope)
                 (0, run_scope_1.excludeRunScopeFromGit)(scope.projectRoot);
+            else {
+                // Degraded, and it must never be silent: without a scope the run's
+                // identity config lands in the SHARED folder, where a concurrent run
+                // and the human's own session overwrite it (see
+                // docs/agent-guides/mcp-identity.md). The run still starts - refusing
+                // would block every run on a read-only folder - but the operator can
+                // see why an identity went wrong.
+                log('[scope] could not create a private run scope - this run shares the ' +
+                    "folder's own opencode config (identity may be taken from the " +
+                    'human). Fix: check permissions on <folder>/.teamshare-runs.', 'stderr');
+            }
         }
         const runCwd = scope?.dir ?? settings.cwd;
         lastRunCwd = runCwd ?? lastRunCwd;
@@ -516,7 +533,13 @@ function spawnHarness(harness, prompt, log, settings = {}) {
         // because the OpenCode service keeps the MCP connection it opened with it.
         // Derived from task + mode so a resume, a retry or a question pause reuses
         // the SAME token instead of 401ing the cached connection.
-        const runKey = (0, relay_key_1.relayKeyFor)(settings.sessionStore?.taskId ?? null, settings.sessionStore?.mode ?? settings.sessionMode ?? null, settings.sessionId);
+        //
+        // The AGENT is part of the key (2026-11): a chat run's "task" is the
+        // CONVERSATION (`chat-<projectId>-<channelId>`), which two agents answering
+        // in one channel share. Without it, agent B's spawn dropped agent A's live
+        // token (a different key must never inherit a session's token) and the two
+        // revoked each other in a loop - the "only when multiple agents run" bug.
+        const runKey = (0, relay_key_1.relayKeyFor)(settings.sessionStore?.taskId ?? null, settings.sessionStore?.mode ?? settings.sessionMode ?? null, settings.sessionId, settings.sessionStore?.agentId);
         if (adapter.supportsMcp && settings.agentKey && adapter.writeMcpConfig) {
             const relay = await resolveMcpRelay(settings.agentKey, settings.sessionStore?.agentId, runKey);
             mcpOverride = adapter.writeMcpConfig(settings.agentKey, settings.apiBaseUrl ?? process.env.TEAMSHARE_API_URL ?? api_1.DEFAULT_API_URL, settings.sessionMode, settings, relay?.url, relay?.token);
@@ -675,6 +698,26 @@ function spawnHarness(harness, prompt, log, settings = {}) {
              * kill fires only when ALL of them have been quiet for the budget.
              */
             const activity = new activity_1.ActivityClock();
+            /**
+             * TURN WATCHDOG (2026-11) - see `turn-watch.ts`.
+             *
+             * The activity clock answers "is anything happening ANYWHERE"; this one
+             * answers "is a model turn OPEN with no output", which is the exact shape
+             * of a hung provider call (a `step_start` and then nothing for 19 minutes
+             * was the Aga Autos chat incident). Ladder: warn -> interrupt -> kill.
+             * The kill reuses exit 124 so the conversation is kept for a retry.
+             */
+            const turn = new turn_watch_1.TurnWatch();
+            const turnBudget = (0, turn_watch_1.turnBudgets)(settings.sessionMode ?? store?.mode, settings.timeoutMin ?? 60);
+            const turnWatchOff = /^(0|off|false|no)$/i.test(process.env.TEAMSHARE_TURN_WATCH ?? '');
+            /** One warning / interrupt per step id, so a recovered run is not re-hit. */
+            let turnWarnedStep = null;
+            let turnInterruptedStep = null;
+            let turnInterruptedAt = null;
+            /** Hard guard inputs - real work in flight is never a stall. */
+            let shellRunningNow = false;
+            let relayInFlight = 0;
+            let lastSubagentAt = 0;
             const STALL_WARN_MS = Math.max(1, Number(process.env.TEAMSHARE_STALL_WARN_MIN) || 5) * 60_000;
             const HEARTBEAT_MS = 5 * 60_000;
             /**
@@ -811,6 +854,9 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                                     // A NEW subagent message is real work even though the parent
                                     // CLI printed nothing for it (the line below is OUR log).
                                     activity.note('subagent', `ses ${shortId(child.id)}`);
+                                    // The turn watchdog's "active subagent" guard: a child that
+                                    // is producing output must never be killed as a stalled turn.
+                                    lastSubagentAt = Date.now();
                                     startIdleTimer();
                                     if (Date.now() - lastHeartbeatAt >= HEARTBEAT_MS) {
                                         lastHeartbeatAt = Date.now();
@@ -860,6 +906,14 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                 // reach the console or the diagnostic sidecar (2026-10-25).
                 const safeLine = (0, diagnostics_1.scrubSecrets)(line);
                 rawCapture(safeLine);
+                // TURN WATCHDOG (2026-11): feed the open-step clock. A `step_start`
+                // that never reaches `step_finish` is a hung model turn, not work -
+                // see `turn-watch.ts` for the incident this exists for.
+                if (!turnWatchOff && resolved === 'opencode') {
+                    const turnEvent = (0, turn_watch_1.extractTurnEvent)(line);
+                    if (turnEvent)
+                        turn.note(turnEvent);
+                }
                 // SCOPE GUARD (2026-10-25): see `tripScopeGuard`.
                 tripScopeGuard(safeLine);
                 // Track background shells the service owns (they outlive this CLI).
@@ -999,6 +1053,9 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                         if (settled)
                             return;
                         const now = Date.now();
+                        // Turn-watchdog guards are re-derived every tick: a shell that
+                        // exited must stop shielding a stalled turn, and vice versa.
+                        shellRunningNow = false;
                         // 1) Relayed MCP/tool activity. Give up after 3 misses (no daemon,
                         //    or a run wired straight to the API) instead of polling forever.
                         if (runKey && relayProbeMisses < 3) {
@@ -1008,6 +1065,9 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                             }
                             else {
                                 relayProbeMisses = 0;
+                                // A call that has not answered yet is work in flight, so the
+                                // turn watchdog must not treat the silence as a hung model.
+                                relayInFlight = signal.inFlight;
                                 if (signal.lastActivityAt > lastRelayActivityAt) {
                                     lastRelayActivityAt = signal.lastActivityAt;
                                     activity.note('mcp', signal.lastTool ?? 'tool call');
@@ -1053,6 +1113,9 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                                     continue;
                                 }
                                 activity.note('shell', shell.command ?? shell.id);
+                                // HARD GUARD: a running command is real work. The turn
+                                // watchdog must never kill a run the agent is waiting on.
+                                shellRunningNow = true;
                                 startIdleTimer();
                                 heartbeat(`background shell ${shell.command ?? shell.id} is still running`);
                                 break;
@@ -1075,6 +1138,110 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                                 ? 'no output at all'
                                 : `no stdout for ${Math.max(1, stdoutMin)} min`} - last activity: ${(0, activity_1.activityActor)(latest)}. Not dead unless the ${Math.max(1, Math.round(timeoutMs / 60_000))} min idle timeout is reached.`, 'stdout');
                         }
+                        // 4) TURN WATCHDOG (2026-11) - see `turn-watch.ts`.
+                        //
+                        //    The activity clock above is deliberately patient: it cannot
+                        //    tell a hung provider call from a long tool call, so it waits the
+                        //    full idle budget. A model turn that OPENED and never produced a
+                        //    token is not ambiguous, though - it is the Aga Autos incident
+                        //    (a `step_start`, then 19 silent minutes). Ladder: warn ->
+                        //    interrupt -> kill, and the kill reuses exit 124 so the
+                        //    conversation is kept for a retry instead of starting over.
+                        if (!turnWatchOff && !settled) {
+                            const turnSnap = turn.snapshot();
+                            // A DIFFERENT step means the session recovered (or moved on):
+                            // clear the interrupt latch so the next stall is judged fresh.
+                            if (turnInterruptedStep !== null &&
+                                turnSnap.stepId !== turnInterruptedStep) {
+                                turnInterruptedStep = null;
+                                turnInterruptedAt = null;
+                            }
+                            const verdict = (0, turn_watch_1.decideTurnAction)({
+                                snapshot: turnSnap,
+                                now,
+                                budgets: turnBudget,
+                                inFlightMcp: relayInFlight,
+                                shellRunning: shellRunningNow,
+                                subagentActive: lastSubagentAt > 0 &&
+                                    now - lastSubagentAt < turnBudget.subagentGraceMs,
+                                interruptedAt: turnInterruptedAt,
+                            });
+                            const stepKey = turnSnap.stepId ?? `step-${turnSnap.stepOpenedAt ?? 0}`;
+                            if (verdict.action === 'warn' && turnWarnedStep !== stepKey) {
+                                turnWarnedStep = stepKey;
+                                log(`[harness] ${adapter.name} model turn open for ` +
+                                    `${Math.max(1, Math.round(verdict.openForMs / 60_000))} min, ` +
+                                    `silent for ${Math.max(1, Math.round(verdict.silentMs / 60_000))} min ` +
+                                    `(${verdict.reason}) - interrupting at ` +
+                                    `${Math.round(turnBudget.interruptMs / 60_000)} min, killing at ` +
+                                    `${Math.round(turnBudget.killMs / 60_000)} min unless it recovers`, 'stderr');
+                                (0, diagnostics_1.reportDiagnosticSafe)({
+                                    kind: 'harness_failure',
+                                    severity: 'warning',
+                                    message: `model turn stalled (${verdict.reason})`,
+                                    errorCode: 'HARNESS_TURN_STALL',
+                                    agentId: store?.agentId,
+                                    context: {
+                                        harness: resolved,
+                                        mode: settings.sessionMode ?? null,
+                                        runKey: runKey ?? null,
+                                        openForMs: verdict.openForMs,
+                                        silentMs: verdict.silentMs,
+                                    },
+                                });
+                            }
+                            if (verdict.action === 'interrupt' &&
+                                turnInterruptedStep !== stepKey) {
+                                turnInterruptedStep = stepKey;
+                                turnInterruptedAt = now;
+                                log(`[harness] ${adapter.name} model turn stalled ` +
+                                    `(${verdict.reason}) - interrupting the service session so it ` +
+                                    'can continue; no work is lost', 'stderr');
+                                if (capturedId)
+                                    await (0, opencode_control_1.interruptOpencodeSession)(capturedId);
+                            }
+                            if (verdict.action === 'kill') {
+                                // Same contract as the idle kill: OUR decision, not a broken
+                                // conversation, so the session is kept for a retry.
+                                idleKilled = true;
+                                forcedExitCode = 124;
+                                log(`[harness] ${adapter.name} model turn stalled ` +
+                                    `(${verdict.reason}) - killing the run; the conversation is ` +
+                                    'kept so the next attempt continues it', 'stderr');
+                                (0, diagnostics_1.reportDiagnosticSafe)({
+                                    kind: 'harness_failure',
+                                    severity: 'error',
+                                    message: `model turn stalled and was killed (${verdict.reason})`,
+                                    errorCode: 'HARNESS_TURN_STALLED',
+                                    agentId: store?.agentId,
+                                    context: {
+                                        harness: resolved,
+                                        mode: settings.sessionMode ?? null,
+                                        runKey: runKey ?? null,
+                                        openForMs: verdict.openForMs,
+                                        silentMs: verdict.silentMs,
+                                    },
+                                });
+                                // Interrupt FIRST: on OpenCode V2 the shared service owns the
+                                // turn, so killing the CLI alone leaves a silent session.
+                                if (capturedId)
+                                    await (0, opencode_control_1.interruptOpencodeSession)(capturedId);
+                                try {
+                                    killTree(child.pid);
+                                }
+                                catch {
+                                    /* ignore */
+                                }
+                                try {
+                                    child.kill();
+                                }
+                                catch {
+                                    /* ignore */
+                                }
+                                done(124);
+                                return;
+                            }
+                        }
                     })();
                 }, 30_000);
                 activityTimer.unref?.();
@@ -1130,11 +1297,33 @@ function spawnHarness(harness, prompt, log, settings = {}) {
                 done(forcedExitCode ?? code ?? 1);
             });
         }).catch((err) => {
-            // `spawn()` can throw SYNCHRONOUSLY (bad argv, ENOENT before the `error`
-            // event), which means the `close` handler above never runs. Clean the MCP
-            // config here too - it holds the agent API key - and let the rejection
-            // reach the caller.
+            // `spawn()` can throw SYNCHRONOUSLY (bad argv, ENAMETOOLONG, ENOENT
+            // before the `error` event), which means the `close` handler above never
+            // runs. Clean the MCP config here too - it holds the agent API key - and
+            // let the rejection reach the caller.
             cleanMcp();
+            // A raw `spawn ENAMETOOLONG` reaches the task as if the binary were
+            // broken. Name the real cause and the fix instead.
+            if (err?.code === 'ENAMETOOLONG') {
+                const detail = `the ${adapter.name} command line was too long to start ` +
+                    `(prompt ${prompt.length} chars, OS limit ~32k). Shorten the task ` +
+                    'brief / project context, or use a harness that supports prompt files.';
+                log(`[harness] ${detail}`, 'stderr');
+                (0, diagnostics_1.reportDiagnosticSafe)({
+                    kind: 'harness_failure',
+                    severity: 'error',
+                    message: detail,
+                    errorCode: 'HARNESS_ENAMETOOLONG',
+                    agentId: settings.sessionStore?.agentId,
+                    context: {
+                        harness: adapter.name,
+                        promptChars: prompt.length,
+                        mode: settings.sessionMode ?? null,
+                        taskId: settings.sessionStore?.taskId ?? null,
+                    },
+                });
+                throw new Error(`spawn ENAMETOOLONG: ${detail}`);
+            }
             throw err;
         });
     };
@@ -1384,11 +1573,17 @@ function spawnHarness(harness, prompt, log, settings = {}) {
             // session is consumed) exists because a retry must not resume a poisoned
             // planning session; a conversation is different, so it gets one retry and
             // is replaced only after the SECOND consecutive failure.
+            //
+            // `failedRun` is REQUIRED here (2026-11): without it a SUCCESSFUL turn
+            // took this branch too, and logged "run failed (attempt 0) - KEEPING the
+            // conversation" on every single reply. Live case: a chat mention answered
+            // the question, posted the message through MCP, exited 0 - and the
+            // console still said the run failed.
             const conversational = session_store_1.CONVERSATIONAL_MODES.has(store.mode);
             const failureCount = failedRun
                 ? Math.max(1, (0, session_store_1.noteHarnessSessionFailure)(store.agentId, store.taskId, store.mode, `${adapter.name} exited with code ${finalCode}`))
                 : 0;
-            const keepForConversation = conversational && failureCount < 2;
+            const keepForConversation = failedRun && conversational && failureCount < 2;
             if (failedRun &&
                 !questionStillOpen &&
                 !keepForIdleResume &&
@@ -1398,13 +1593,8 @@ function spawnHarness(harness, prompt, log, settings = {}) {
             }
             else if (spec.capture !== 'none') {
                 if (keepForConversation) {
-                    log(`[resume] run failed (attempt ${failureCount}) - KEEPING the ` +
-                        `conversation so the next message continues instead of starting over`);
-                    if (failureCount >= 2) {
-                        // Unreachable by construction (`keepForConversation` is false at
-                        // 2), kept as a guard if the threshold ever changes.
-                        log('[resume] session failed twice - replacing it', 'stderr');
-                    }
+                    log(`[resume] run failed (attempt ${failureCount} of 2) - keeping the ` +
+                        `conversation so the next message continues instead of starting over`, 'stderr');
                 }
                 if (questionStillOpen) {
                     log(`[resume] run failed but a question is still open - keeping the ` +
@@ -1502,9 +1692,11 @@ async function readRelayActivity(runKey) {
         const at = Number(body.lastActivityAt);
         if (!Number.isFinite(at) || at <= 0)
             return null;
+        const inFlight = Number(body.inFlight);
         return {
             lastActivityAt: at,
             lastTool: typeof body.lastTool === 'string' ? body.lastTool : null,
+            inFlight: Number.isFinite(inFlight) && inFlight > 0 ? inFlight : 0,
         };
     }
     catch {
@@ -1568,28 +1760,33 @@ function spawnThroughPty(binary, prompt, log, settings, name) {
     const adapterLike = (0, harness_adapter_1.getAdapter)(name);
     let promptFile = null;
     let args;
-    if (prompt.length > PROMPT_FILE_THRESHOLD && adapterLike.supportsPromptFile) {
-        promptFile = (0, prompt_file_1.writePromptFile)(prompt);
-        log(`[harness] prompt too long for argv (${prompt.length} chars) — wrote to temp file`, 'stderr');
-        if (adapterLike.promptFileFlag) {
-            args = adapterLike.headlessArgs(prompt, settings, true);
-            args.push(adapterLike.promptFileFlag, promptFile);
+    const inlineArgs = adapterLike.headlessArgs(prompt, settings);
+    const commandLineChars = (0, prompt_file_1.argvChars)([
+        bin.file,
+        ...(bin.args ?? []),
+        ...inlineArgs,
+    ]);
+    if ((0, prompt_file_1.shouldUsePromptFile)(adapterLike, prompt.length, commandLineChars)) {
+        try {
+            promptFile = (0, prompt_file_1.writePromptFile)(prompt);
+        }
+        catch (err) {
+            promptFile = null;
+            log(`[harness] could not write the prompt file (${msg(err)}) - ` +
+                'falling back to argv, which may exceed the OS command-line limit', 'stderr');
+        }
+        if (promptFile) {
+            log(`[harness] prompt too long for argv (${prompt.length} chars, ` +
+                `~${commandLineChars} char command line) — wrote it to ` +
+                `${promptFile} and passed the path instead`, 'stderr');
+            args = (0, prompt_file_1.promptFileArgs)(adapterLike, adapterLike.headlessArgs(prompt, settings, true), promptFile);
         }
         else {
-            args = adapterLike.headlessArgs(prompt, settings, true);
-            const shortInstruction = `Read the file "${promptFile}" and follow its instructions exactly. ` +
-                'Do not respond until you have read the file. ' +
-                'The file contains your complete instructions including project context, tools, and the user message.';
-            if (['claude', 'gemini', 'antigravity', 'copilot', 'cursor'].includes(name)) {
-                args.unshift('-p', shortInstruction);
-            }
-            else {
-                args.push(shortInstruction);
-            }
+            args = inlineArgs;
         }
     }
     else {
-        args = adapterLike.headlessArgs(prompt, settings);
+        args = inlineArgs;
     }
     let mcpOverride = null;
     if (adapterLike.supportsMcp &&