teamshare-bridge 0.21.57 → 0.21.58

package/dist/bridge/daemon.js

@@ -6,6 +6,7 @@ exports.runDaemon = runDaemon;
  * agent and spawns a session when a `wake` event arrives (contract section 4).
  * Also serves the localhost probe so the website can detect it.
  */
+const node_child_process_1 = require("node:child_process");
 const node_fs_1 = require("node:fs");
 const process_tree_1 = require("../lib/process-tree");
 const git_runner_1 = require("../lib/git/git-runner");
@@ -25,6 +26,12 @@ const rate_gate_1 = require("../lib/rate-gate");
 const harness_models_1 = require("../lib/harness/harness-models");
 /** IMP-917: wakes already deferred once for a rate-limit pause (bounded). */
 const rateDeferred = new Set();
+/**
+ * Spawn watchdog hook: assigned by runDaemon to `scheduleSpawnWatch` so the
+ * module-level handleWake can flag every real spawn for session-start
+ * verification (a spawned session that never came up used to fail silently).
+ */
+let spawnWatchScheduler = null;
 const bus_1 = require("../lib/workspace/bus");
 const presence_1 = require("../lib/workspace/presence");
 const worktree_1 = require("../lib/orchestrator/worktree");
@@ -459,38 +466,151 @@ async function runDaemon(portOverride) {
     if (config.agents.length === 0) {
         log('no agents configured - run: teamshare-agent connect --agent <id> --key ts_...');
     }
+    // ── Spawn watchdog (2026-09) ────────────────────────────────────────────
+    // A spawned session that never reaches session:start used to fail in
+    // complete silence (Windows Script Host blocked, native-module ABI crash
+    // after a Node upgrade, missing/stale CLI build): only a [spawn] line in
+    // daemon.log, no comment, no hub signal. 20s after a wake reports 'spawned',
+    // verify a session actually came up and report a spawn_failure diagnostic
+    // with the capture-file tail when it did not.
+    const spawnWatchTimers = new Map();
+    function scheduleSpawnWatch(agentId, wakeType) {
+        const prev = spawnWatchTimers.get(agentId);
+        if (prev)
+            clearTimeout(prev);
+        const startedAt = Date.now();
+        const timer = setTimeout(() => {
+            spawnWatchTimers.delete(agentId);
+            try {
+                if (sessionRelay.current(agentId))
+                    return; // session came up
+                const lock = (0, lock_1.readLock)(agentId);
+                if (lock?.pid) {
+                    try {
+                        process.kill(lock.pid, 0);
+                        return; // a live session process holds the lock - assume running
+                    }
+                    catch {
+                        /* dead pid - fall through to forensics */
+                    }
+                }
+                // No session, no live lock: look for a capture file written by this
+                // spawn (hidden-launch creates it only if the chain actually ran).
+                let launched = false;
+                let captureInfo = 'no capture file - the hidden launcher never ran (Windows Script Host/wscript.exe may be disabled or blocked by antivirus)';
+                const dir = (0, node_path_1.join)((0, node_os_1.homedir)(), '.teamshare', 'sessions');
+                let newest = null;
+                for (const f of (0, node_fs_1.readdirSync)(dir)) {
+                    if (!f.startsWith('spawn-') || !f.endsWith('.log'))
+                        continue;
+                    const st = (0, node_fs_1.statSync)((0, node_path_1.join)(dir, f), { throwIfNoEntry: false });
+                    if (!st)
+                        continue;
+                    if (st.mtimeMs >= startedAt - 2_000 &&
+                        (!newest || st.mtimeMs > newest.mtime)) {
+                        newest = { file: f, mtime: st.mtimeMs };
+                    }
+                }
+                if (newest) {
+                    launched = true;
+                    const content = (0, node_fs_1.readFileSync)((0, node_path_1.join)(dir, newest.file), 'utf8');
+                    const tail = content
+                        .split(/\r?\n/)
+                        .filter(Boolean)
+                        .slice(-15)
+                        .join('\n');
+                    captureInfo = tail
+                        ? `capture ${newest.file} tail:\n${tail}`
+                        : `capture ${newest.file} exists but is empty - the CLI exited before logging anything (native-module load failure?)`;
+                }
+                log(`agent ${agentId}: spawn watch - no session started ${Math.round((Date.now() - startedAt) / 1000)}s after ${wakeType} wake`);
+                (0, diagnostics_1.reportDiagnosticSafe)({
+                    kind: 'spawn_failure',
+                    severity: 'error',
+                    agentId,
+                    message: `Session never started after a ${wakeType} wake (${launched ? 'CLI exited before session:start' : 'hidden launcher never ran'})`,
+                    errorCode: launched
+                        ? 'SESSION_PREMATURE_EXIT'
+                        : 'SESSION_NEVER_LAUNCHED',
+                    context: { wakeType, capture: captureInfo.slice(0, 4_000) },
+                    includeDoctor: true,
+                });
+            }
+            catch (err) {
+                log(`agent ${agentId}: spawn watch failed: ${(0, config_1.msg)(err)}`);
+            }
+        }, 20_000);
+        timer.unref();
+        spawnWatchTimers.set(agentId, timer);
+    }
+    spawnWatchScheduler = scheduleSpawnWatch;
     // ── Stale lock sweep (safety net) ────────────────────────────────────
     // On startup + every 5 minutes, scan all agent lock files. If the
-    // holder PID is dead, remove the lock. This catches orphaned locks from
-    // force-killed sessions, crashes, OOM kills, or daemon restarts.
-    // NEVER touches a lock held by a live process — only dead PIDs.
+    // holder PID is dead OR has been recycled by a foreign (non-session)
+    // process, remove the lock. This catches orphaned locks from force-killed
+    // sessions, crashes, OOM kills, daemon restarts, and PID reuse (which
+    // used to keep the agent "busy" for up to 24h, queueing every wake).
+    // NEVER touches a lock held by a live session-like process.
+    function holderLooksLikeSession(pid) {
+        try {
+            if (process.platform === 'win32') {
+                const out = (0, node_child_process_1.spawnSync)('tasklist', ['/FI', `PID eq ${pid}`, '/NH'], {
+                    encoding: 'utf8',
+                    windowsHide: true,
+                    timeout: 3_000,
+                });
+                const text = (out.stdout ?? '').trim();
+                // No match / odd output - stay conservative (trust the pid).
+                if (!text || /^INFO:/i.test(text))
+                    return true;
+                return /node|wscript|teamshare|opencode|claude|conhost|bun\.exe/i.test(text);
+            }
+            const out = (0, node_child_process_1.spawnSync)('ps', ['-p', String(pid), '-o', 'comm='], {
+                encoding: 'utf8',
+                timeout: 3_000,
+            });
+            const comm = (out.stdout ?? '').trim().toLowerCase();
+            if (!comm)
+                return true; // ps unavailable - stay conservative
+            return /node|bun|deno|teamshare/.test(comm);
+        }
+        catch {
+            return true;
+        }
+    }
     function sweepStaleLocks() {
         for (const agent of config.agents) {
             const lock = (0, lock_1.readLock)(agent.agentId);
             if (!lock)
                 continue;
+            let reason = null;
             try {
                 process.kill(lock.pid, 0); // probe — throws ESRCH if dead
+                if (!holderLooksLikeSession(lock.pid)) {
+                    reason = `pid ${lock.pid} is now a foreign process (PID reuse)`;
+                }
             }
             catch {
-                // PID is dead — remove the orphaned lock.
-                (0, lock_1.clearLock)(agent.agentId);
-                log(`lock sweep: cleared stale lock for ${agent.agentId.slice(0, 8)} (dead pid ${lock.pid})`);
-                // Also end the active session for this agent so the backend marks
-                // it as ended immediately rather than waiting for the heartbeat
-                // sweep. Without this, the daemon's own heartbeat keeps
-                // lastHeartbeatAt fresh and the backend sweep skips the session.
-                const activeSessionId = sessionRelay.current(agent.agentId);
-                if (activeSessionId) {
-                    sessionRelay.sessionEnd({ sessionId: activeSessionId, exitCode: 1 });
-                    log(`lock sweep: ended session ${activeSessionId.slice(0, 12)} for agent ${agent.agentId.slice(0, 8)}`);
-                }
-                // Push busy=false immediately so the frontend indicator updates
-                // without waiting for the next 5-min pushState interval.
-                const handle = handles.get(agent.agentId);
-                if (handle)
-                    handle.pushState();
+                reason = `dead pid ${lock.pid}`;
             }
+            if (!reason)
+                continue;
+            (0, lock_1.clearLock)(agent.agentId);
+            log(`lock sweep: cleared stale lock for ${agent.agentId.slice(0, 8)} (${reason})`);
+            // Also end the active session for this agent so the backend marks
+            // it as ended immediately rather than waiting for the heartbeat
+            // sweep. Without this, the daemon's own heartbeat keeps
+            // lastHeartbeatAt fresh and the backend sweep skips the session.
+            const activeSessionId = sessionRelay.current(agent.agentId);
+            if (activeSessionId) {
+                sessionRelay.sessionEnd({ sessionId: activeSessionId, exitCode: 1 });
+                log(`lock sweep: ended session ${activeSessionId.slice(0, 12)} for agent ${agent.agentId.slice(0, 8)}`);
+            }
+            // Push busy=false immediately so the frontend indicator updates
+            // without waiting for the next 5-min pushState interval.
+            const handle = handles.get(agent.agentId);
+            if (handle)
+                handle.pushState();
         }
     }
     // Run immediately on startup.
@@ -600,6 +720,10 @@ async function runDaemon(portOverride) {
                 queued: agentState.queued ?? 0,
                 machineId: (0, config_1.machineId)(),
                 machineLabel: (0, config_1.machineLabel)(),
+                // DETERMINISTIC PORT DISCOVERY (2026-09): the website reads this via
+                // GET /agents/:id/state and probes the EXACT bound port on 127.0.0.1
+                // instead of only scanning the static candidate set.
+                port: boundPort,
                 projects: Object.fromEntries(Object.entries(status.projects).map(([id, link]) => [id, { path: link.path, guard: link.guard ?? true }])),
             };
             const snapshot = JSON.stringify(payload);
@@ -806,8 +930,11 @@ async function runDaemon(portOverride) {
                             }
                         }
                     }
-                    catch {
-                        // best-effort: don't crash the daemon on periodic poll failure
+                    catch (err) {
+                        // best-effort: don't crash the daemon - but DO log it. This used
+                        // to be a bare catch, so an inbox outage silently disabled the
+                        // 20-minute recovery sweep with zero trace.
+                        log(`[error] mention/stranded poll failed: ${(0, config_1.msg)(err)}`);
                     }
                 }, MENTION_POLL_MS));
             }
@@ -1006,6 +1133,12 @@ async function runDaemon(portOverride) {
         });
         socket.on('wake', (event) => {
             lastWakeByAgent.set(agent.agentId, new Date().toISOString());
+            // Receipt ack (2026-09): tells the hub the emit actually landed, so a
+            // half-open socket is caught by the hub's 15s watchdog instead of
+            // swallowing the wake silently. Older hubs simply ignore the event.
+            if (event.wakeId) {
+                socket.emit('wake.received', { wakeId: event.wakeId });
+            }
             // Manual interrupt (2026-08): session.stop bypasses cooldown, dedupe
             // and the queue entirely - a human pressing "Stop work" must always
             // land immediately. Kills the live session (harness/CLI and its
@@ -1288,7 +1421,12 @@ async function runDaemon(portOverride) {
             const agent = credsOf(h);
             void (async () => {
                 const settings = await freshSettings(api_1.TeamshareApi.forAgent(apiBase, agent), agentId);
-                return spawnWake(api_1.TeamshareApi.forAgent(apiBase, agent), agentId, agent.apiKey, event, getConfig(), { settings, recentSpawns });
+                const outcome = await spawnWake(api_1.TeamshareApi.forAgent(apiBase, agent), agentId, agent.apiKey, event, getConfig(), { settings, recentSpawns });
+                // Same spawn watchdog as the live wake path - a queued wake that
+                // spawns later must also be verified (session:start or report).
+                if (outcome === 'spawned')
+                    spawnWatchScheduler?.(agentId, event.type);
+                return outcome;
             })()
                 .then(() => {
                 log(`agent ${agentId}: spawned queued wake ${event.type}`);
@@ -1415,18 +1553,10 @@ async function handleWake(api, agentId, apiKey, event, configInput, extra) {
         log(`agent ${agentId}: wake ${event.type} skipped (quiet hours)`);
         return 'skipped';
     }
-    if (event.taskId && config.wakeRules?.priorities?.length) {
-        try {
-            const task = await api.getTask(event.taskId);
-            if (!config.wakeRules.priorities.includes(task.priority)) {
-                log(`agent ${agentId}: wake ${event.type} skipped (priority ${task.priority})`);
-                return 'skipped';
-            }
-        }
-        catch {
-            /* fetch failed - proceed anyway */
-        }
-    }
+    // wakeRules.priorities enforcement REMOVED (2026-09): the gate was already
+    // declared unsupported in the guides, yet a stale config.json entry still
+    // silently skipped assignment wakes with zero feedback ("agent idle, no
+    // response"). The config field is tolerated but ignored from now on.
     // Stop signals (2026-09): land IMMEDIATELY, before the busy queue. A
     // `build.stop` used to be queued behind the very build it targeted (the
     // build loop holds the agent lock for its whole run), so the stop file was
@@ -1447,9 +1577,29 @@ async function handleWake(api, agentId, apiKey, event, configInput, extra) {
         queue.push(event);
         wakeQueue.set(agentId, queue);
         log(`agent ${agentId}: wake ${event.type} queued (busy since ${lock.startedAt}, ${lock.kind}) - ${queue.length} waiting`);
+        // Stuck-queue visibility: a lock held far longer than any real session
+        // (PID reuse / hung process) queues every wake invisibly - the hub keeps
+        // showing idle. Report once the lock crosses 10 minutes; the fingerprint
+        // includes the lock's startedAt, so all wakes behind the SAME lock
+        // coalesce into one report (10-min cooldown).
+        const lockAgeMs = Date.now() - new Date(lock.startedAt).getTime();
+        if (lockAgeMs > 10 * 60_000) {
+            (0, diagnostics_1.reportDiagnosticSafe)({
+                kind: 'wake_failure',
+                severity: 'warning',
+                agentId,
+                errorCode: 'WAKE_QUEUE_STUCK',
+                message: `wake ${event.type} queued behind a ${lock.kind} lock held over 10 minutes (since ${lock.startedAt}) - if no session is actually running: teamshare-bridge clear-lock --agent ${agentId}`,
+            });
+        }
         return 'queued';
     }
-    return await spawnWake(api, agentId, apiKey, event, config, { settings, recentSpawns, orchestratorClient });
+    const outcome = await spawnWake(api, agentId, apiKey, event, config, { settings, recentSpawns, orchestratorClient });
+    // Spawn watchdog: flag every real spawn so a session that never reaches
+    // session:start is reported instead of failing in complete silence.
+    if (outcome === 'spawned')
+        spawnWatchScheduler?.(agentId, event.type);
+    return outcome;
 }
 async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
     log(`agent ${agentId}: wake ${event.type}${event.taskId ? ` task=${event.taskId}` : ''}`);
@@ -1784,6 +1934,10 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
             log(`agent ${agentId}: question.answered -> orchestrator signaled for OT ${event.orchestratorTaskId}`);
         }
         // Also resume build mode if a build session was active
+        // `spawned` is ONLY true when a process was actually launched - a
+        // no-local-session answer must not claim 'spawned' (callers treat it as
+        // a live spawn, and the spawn watchdog would false-positive).
+        let spawned = false;
         if (hasBuildState) {
             // Write a signal file so the running build loop wakes immediately
             // instead of waiting for the 15s API poll fallback.
@@ -1795,6 +1949,7 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
                 /* best-effort */
             }
             (0, spawn_1.spawnBuildSession)(agentId, apiKey, event.projectId, extra.settings, undefined, cwd);
+            spawned = true;
         }
         else if (event.taskId) {
             // Resume-on-answer for EVERY mode: if THIS PC still holds the harness
@@ -1819,19 +1974,22 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
             if ((0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'run')) {
                 log(`agent ${agentId}: question.answered -> resuming run session for task ${event.taskId}`);
                 (0, spawn_1.spawnAgentSession)(agentId, apiKey, event.taskId, extra.settings, undefined, cwd);
+                spawned = true;
             }
             else if (planAllowed && (0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'plan')) {
                 log(`agent ${agentId}: question.answered -> resuming plan session for task ${event.taskId}`);
                 (0, spawn_1.spawnPlanSession)(agentId, apiKey, event.taskId, event.projectId ?? undefined, extra.settings, undefined, cwd);
+                spawned = true;
             }
             else if ((0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'build')) {
                 (0, spawn_1.spawnBuildSession)(agentId, apiKey, event.projectId, extra.settings, undefined, cwd);
+                spawned = true;
             }
             else {
                 log(`agent ${agentId}: question.answered - no local session for task ${event.taskId} (fresh on next run)`);
             }
         }
-        return 'spawned';
+        return spawned ? 'spawned' : 'skipped';
     }
     if (event.type === 'question.expired') {
         // Expired-on-timeout: same wake path as question.answered — the agent
@@ -1848,6 +2006,9 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
             extra.orchestratorClient.signalQuestionAnswered(event.orchestratorTaskId, event.commentId ?? event.messageId ?? 'wake-expired');
             log(`agent ${agentId}: question.expired -> orchestrator signaled for OT ${event.orchestratorTaskId}`);
         }
+        // Same spawn-tracking rule as question.answered: only claim 'spawned'
+        // when a process was actually launched.
+        let spawned = false;
         if (hasBuildState) {
             try {
                 (0, node_fs_1.mkdirSync)(SIGNAL_DIR, { recursive: true });
@@ -1857,6 +2018,7 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
                 /* best-effort */
             }
             (0, spawn_1.spawnBuildSession)(agentId, apiKey, event.projectId, extra.settings, undefined, cwd);
+            spawned = true;
         }
         else if (event.taskId) {
             // Same resume-on-expiry path as question.answered (local session only).
@@ -1873,15 +2035,18 @@ async function spawnWake(api, agentId, apiKey, event, config, extra = {}) {
             const planAllowed = !isToolTask && workMode !== 'build';
             if ((0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'run')) {
                 (0, spawn_1.spawnAgentSession)(agentId, apiKey, event.taskId, extra.settings, undefined, cwd);
+                spawned = true;
             }
             else if (planAllowed && (0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'plan')) {
                 (0, spawn_1.spawnPlanSession)(agentId, apiKey, event.taskId, event.projectId ?? undefined, extra.settings, undefined, cwd);
+                spawned = true;
             }
             else if ((0, session_store_1.hasHarnessSession)(agentId, event.taskId, 'build')) {
                 (0, spawn_1.spawnBuildSession)(agentId, apiKey, event.projectId, extra.settings, undefined, cwd);
+                spawned = true;
             }
         }
-        return 'spawned';
+        return spawned ? 'spawned' : 'skipped';
     }
     if (event.type === 'comment.mention') {
         // Comment mentions on tasks spawn a task session (agent responds in