teleportation-cli 1.2.2 → 1.4.0

package/lib/daemon/teleportation-daemon.js

@@ -54,6 +54,7 @@ import {
   executeTaskTurn,
   stopTask,
   stopAllTasks,
+  stopTasksForSession,
 } from './task-executor-v2.js';
 
 // Transcript ingestion for timeline completeness
@@ -229,6 +230,10 @@ const OUTPUT_PREVIEW_LONG = 1000;   // Full output displays
 const heartbeatState = new Map();
 let lastHeartbeatTime = 0;
 
+// Track which sessions have had their first heartbeat failure logged.
+// Prevents log spam while ensuring operators see at least one failure per session.
+const heartbeatFailureLogged = new Set();
+
 // In-memory registry of active teleportation sessions handled by this daemon
 const sessions = new Map();
 
@@ -265,6 +270,7 @@ setInterval(() => {
       sessions.delete(sessionId);
       sessionActivity.delete(sessionId);
       heartbeatState.delete(sessionId);
+      heartbeatFailureLogged.delete(sessionId);
       sessionCleanedCount++;
 
       if (process.env.DEBUG) {
@@ -1037,7 +1043,8 @@ async function ackAndLogCancellation(session_id, message, triggeredBy = {}) {
   try {
     await fetch(`${RELAY_API_URL}/api/messages/${encodeURIComponent(message.id)}/ack`, {
       method: 'POST',
-      headers: { 'Authorization': `Bearer ${RELAY_API_KEY}` }
+      headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+      body: JSON.stringify({ session_id })
     });
     logDebug(`[daemon] Acknowledged message ${message.id}`);
   } catch (ackError) {
@@ -1098,7 +1105,8 @@ async function handleInboxMessage(session_id, message) {
         logWarn(`[daemon] Failed to check is_away state, skipping auto-continue for safety`);
         await fetch(`${RELAY_API_URL}/api/messages/${encodeURIComponent(message.id)}/ack`, {
           method: 'POST',
-          headers: { 'Authorization': `Bearer ${RELAY_API_KEY}` }
+          headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+          body: JSON.stringify({ session_id })
         }).catch(() => {});
         return;
       }
@@ -1131,11 +1139,73 @@ async function handleInboxMessage(session_id, message) {
         // Still acknowledge the message to prevent re-delivery
         await fetch(`${RELAY_API_URL}/api/messages/${encodeURIComponent(message.id)}/ack`, {
           method: 'POST',
-          headers: { 'Authorization': `Bearer ${RELAY_API_KEY}` }
+          headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+          body: JSON.stringify({ session_id })
         });
         return;
       }
 
+      // Check for paused tasks — route message to task instead of spawning new process.
+      // This adds one relay round-trip per inbox message, but only fires for 'command' type
+      // messages (user-initiated from mobile), not for auto-continue or approval messages.
+      // Future optimization: include paused task info in the session polling response.
+      try {
+        const tasksResp = await fetch(
+          `${RELAY_API_URL}/api/sessions/${encodeURIComponent(session_id)}/tasks`,
+          { headers: { 'Authorization': `Bearer ${RELAY_API_KEY}` }, signal: AbortSignal.timeout(5000) }
+        );
+        if (tasksResp.ok) {
+          const tasks = await tasksResp.json();
+          const pausedTasks = tasks.filter(t => t.status === 'paused' || t.status === 'waiting_input');
+          if (pausedTasks.length > 1) {
+            logWarn(`[daemon] ⚠️  Multiple paused tasks (${pausedTasks.length}) for session ${session_id} — routing to first`);
+          }
+          const pausedTask = pausedTasks[0];
+          if (pausedTask) {
+            logInfo(`[daemon] 📨 Routing message to paused task ${pausedTask.id.slice(0, 20)}... (status: ${pausedTask.status})`);
+
+            let routeResp;
+            if (pausedTask.status === 'waiting_input') {
+              // Task is waiting for user input — use the answer endpoint
+              routeResp = await fetch(
+                `${RELAY_API_URL}/api/sessions/${encodeURIComponent(session_id)}/tasks/${encodeURIComponent(pausedTask.id)}/answer`,
+                {
+                  method: 'POST',
+                  headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+                  body: JSON.stringify({ answer: commandText })
+                }
+              );
+            } else {
+              // Task is paused — use redirect to set new instructions and resume
+              routeResp = await fetch(
+                `${RELAY_API_URL}/api/sessions/${encodeURIComponent(session_id)}/tasks/${encodeURIComponent(pausedTask.id)}/redirect`,
+                {
+                  method: 'POST',
+                  headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+                  body: JSON.stringify({ instruction: commandText })
+                }
+              );
+            }
+
+            if (routeResp.ok) {
+              logInfo(`[daemon] ✅ Message routed to task, will resume on next poll cycle`);
+              // Acknowledge the inbox message
+              await fetch(`${RELAY_API_URL}/api/messages/${encodeURIComponent(message.id)}/ack`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${RELAY_API_KEY}` },
+                body: JSON.stringify({ session_id })
+              }).catch(() => {});
+              return; // Don't spawn a new process
+            }
+            // If route failed (e.g. wrong status), fall through to normal execution
+            logWarn(`[daemon] Failed to route message to task (${routeResp.status}), falling back to normal execution`);
+          }
+        }
+      } catch (taskCheckError) {
+        // Non-critical — fall through to normal execution
+        logWarn(`[daemon] Task check failed: ${taskCheckError.message}`);
+      }
+
       // Invalidate pending approvals BEFORE executing new command
       // This prevents race conditions where stale approvals could be acted upon
       try {
@@ -1355,8 +1425,10 @@ async function handleInboxMessage(session_id, message) {
     const ackResponse = await fetch(`${RELAY_API_URL}/api/messages/${encodeURIComponent(message.id)}/ack`, {
       method: 'POST',
       headers: {
+        'Content-Type': 'application/json',
         'Authorization': `Bearer ${RELAY_API_KEY}`
-      }
+      },
+      body: JSON.stringify({ session_id, is_user_prompt: true })
     });
 
     if (ackResponse.ok) {
@@ -1484,6 +1556,77 @@ async function pollRelayAPI() {
       // Update activity timestamp for cleanup tracking
       sessionActivity.set(session_id, Date.now());
 
+      // 0) Check for stop_requested flag (mobile stop button)
+      // Uses daemon-state endpoint (lightweight) instead of full session fetch
+      try {
+        const stopCheckResponse = await fetch(
+          `${RELAY_API_URL}/api/sessions/${encodeURIComponent(session_id)}/daemon-state`,
+          {
+            headers: { 'Authorization': `Bearer ${RELAY_API_KEY}` },
+            signal: AbortSignal.timeout(5000)
+          }
+        );
+
+        if (stopCheckResponse.ok) {
+          const daemonState = await stopCheckResponse.json();
+
+          if (daemonState.stop_requested) {
+            logInfo(`[daemon] 🛑 Stop requested for session ${session_id} — killing running processes`);
+
+            // Kill any running approval execution processes for this session
+            let killedExecution = false;
+            for (const [approval_id, exec] of executions) {
+              if (exec.session_id === session_id && exec.status === 'executing' && exec.child_process) {
+                try {
+                  const child = exec.child_process;
+                  child.kill('SIGTERM');
+                  // Track SIGKILL timer so it can be cancelled if process exits cleanly
+                  const killTimer = setTimeout(() => {
+                    try { child.kill('SIGKILL'); } catch {}
+                  }, 2000);
+                  child.once('exit', () => clearTimeout(killTimer));
+                  killedExecution = true;
+                  logInfo(`[daemon] Killed execution process for approval ${approval_id}`);
+                } catch (killErr) {
+                  logWarn(`[daemon] Failed to kill execution ${approval_id}: ${killErr.message}`);
+                }
+              }
+            }
+
+            // Kill any running task processes for this session
+            const killedTaskCount = stopTasksForSession(session_id);
+            const killedTask = killedTaskCount > 0;
+            if (killedTask) {
+              logInfo(`[daemon] Killed ${killedTaskCount} task process(es) for session ${session_id}`);
+            }
+
+            // Clear stop_requested flag
+            try {
+              await fetch(`${RELAY_API_URL}/api/sessions/${encodeURIComponent(session_id)}/daemon-state`, {
+                method: 'PATCH',
+                headers: {
+                  'Content-Type': 'application/json',
+                  'Authorization': `Bearer ${RELAY_API_KEY}`
+                },
+                body: JSON.stringify({ stop_requested: false })
+              });
+              logInfo(`[daemon] Cleared stop_requested flag for session ${session_id}`);
+            } catch (clearErr) {
+              logWarn(`[daemon] Failed to clear stop_requested: ${clearErr.message}`);
+            }
+
+            if (!killedExecution && !killedTask) {
+              logInfo(`[daemon] No running processes found to stop for session ${session_id}`);
+            }
+          }
+        }
+      } catch (stopCheckError) {
+        // Don't block polling if stop check fails
+        if (stopCheckError.name !== 'AbortError') {
+          logWarn(`[daemon] Stop check error for ${session_id}: ${stopCheckError.message}`);
+        }
+      }
+
       // 1) Approvals polling (existing behavior)
       try {
         const response = await fetch(
@@ -1593,8 +1736,8 @@ async function pollRelayAPI() {
 
           // Process each task (stateless - queries timeline each time)
           for (const task of tasks) {
-            // Skip stopped/completed tasks
-            if (task.status === 'stopped' || task.status === 'completed') {
+            // Skip non-runnable tasks (paused tasks wait for user message to resume)
+            if (task.status === 'stopped' || task.status === 'completed' || task.status === 'paused') {
               continue;
             }
 
@@ -1633,11 +1776,21 @@ async function pollRelayAPI() {
       const claude_session_id = sessionData.claude_session_id || session_id;
       const cwd = sessionData.cwd || process.cwd();
 
+      // 4) Heartbeat - send periodically to keep session alive
+      // Must run before ingestion throttle check — ingestion `continue` must not skip heartbeats
+      const now = Date.now();
+      const sessionHeartbeat = heartbeatState.get(session_id);
+      const lastSent = sessionHeartbeat?.lastSent || 0;
+      if (now - lastSent >= SESSION_HEARTBEAT_INTERVAL_MS) {
+        await sendHeartbeat(session_id);
+      }
+
+      // 5) Transcript ingestion - backup to stop hook for timeline completeness
       // Throttling: Check if ingestion is already in progress for this session
       // Prevents concurrent ingestion runs that could cause race conditions
       if (ingestionInProgress.has(session_id)) {
         debugLog('daemon-transcript-debug.log', `Ingestion already in progress for ${session_id}, skipping`);
-        // Skip this polling cycle for this session
+        // Skip ingestion this cycle (heartbeat already sent above)
         continue;
       }
 
@@ -1670,15 +1823,6 @@ async function pollRelayAPI() {
 
       // Track the promise (but don't await - fire-and-forget)
       ingestionInProgress.set(session_id, ingestionPromise);
-
-      // 5) Heartbeat - send periodically to keep session alive
-      // Only send heartbeat if enough time has passed since last one (throttled per session)
-      const now = Date.now();
-      const sessionHeartbeat = heartbeatState.get(session_id);
-      const lastSent = sessionHeartbeat?.lastSent || 0;
-      if (now - lastSent >= SESSION_HEARTBEAT_INTERVAL_MS) {
-        await sendHeartbeat(session_id);
-      }
     }
 
     // Process approval queue
@@ -1715,11 +1859,12 @@ function cleanupOldExecutions() {
     console.log(`[daemon] Cleaned up ${removed} old execution(s) from cache`);
   }
 
-  // Clean up heartbeatState for sessions that no longer exist
+  // Clean up heartbeatState and heartbeatFailureLogged for sessions that no longer exist
   let heartbeatRemoved = 0;
   for (const sessionId of heartbeatState.keys()) {
     if (!sessions.has(sessionId)) {
       heartbeatState.delete(sessionId);
+      heartbeatFailureLogged.delete(sessionId);
       heartbeatRemoved++;
     }
   }
@@ -1812,6 +1957,7 @@ async function processQueue() {
   // Mark as executing (child_process will be set when spawnClaudeProcess is called)
   executions.set(approval_id, {
     approval_id,
+    session_id,
     status: 'executing',
     started_at: Date.now(),
     completed_at: null,
@@ -2868,7 +3014,7 @@ async function main() {
       if (isShuttingDown || sessions.size === 0) return;
       for (const [sessionId] of sessions) {
         try {
-          await fetch(`${RELAY_API_URL}/api/sessions/${encodeURIComponent(sessionId)}/heartbeat`, {
+          const hbResponse = await fetch(`${RELAY_API_URL}/api/sessions/${encodeURIComponent(sessionId)}/heartbeat`, {
             method: 'POST',
             headers: {
               'Content-Type': 'application/json',
@@ -2877,8 +3023,58 @@ async function main() {
             body: JSON.stringify({ timestamp: Date.now() }),
             signal: AbortSignal.timeout(5000)
           });
+          if (!hbResponse.ok) {
+            const errMsg = `HTTP ${hbResponse.status}`;
+
+            // 404 means session expired from Redis — try to re-register it.
+            // The relay heartbeat endpoint also attempts recovery from mech-storage,
+            // but if that fails (e.g., session never persisted), daemon-side re-registration
+            // ensures the session is recreated with correct metadata.
+            if (hbResponse.status === 404) {
+              const sessionData = sessions.get(sessionId);
+              try {
+                const regResponse = await fetch(`${RELAY_API_URL}/api/sessions/register`, {
+                  method: 'POST',
+                  headers: {
+                    'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${RELAY_API_KEY}`
+                  },
+                  body: JSON.stringify({
+                    session_id: sessionId,
+                    claude_session_id: sessionData?.claude_session_id || undefined,
+                    cwd: sessionData?.cwd || process.cwd(),
+                    meta: sessionData?.meta || {}
+                  }),
+                  signal: AbortSignal.timeout(5000)
+                });
+                if (regResponse.ok) {
+                  console.log(`[daemon] Re-registered expired session ${sessionId} after heartbeat 404`);
+                  // Clear failure tracking so next heartbeat is treated fresh
+                  heartbeatFailureLogged.delete(sessionId);
+                  continue; // Skip failure logging — session recovered
+                } else {
+                  console.warn(`[daemon] Failed to re-register session ${sessionId}: HTTP ${regResponse.status}`);
+                }
+              } catch (regErr) {
+                console.warn(`[daemon] Re-registration attempt failed for ${sessionId}: ${regErr.message}`);
+              }
+            }
+
+            if (!heartbeatFailureLogged.has(sessionId)) {
+              heartbeatFailureLogged.add(sessionId);
+              console.warn(`[daemon] Heartbeat rejected for ${sessionId}: ${errMsg} (further failures for this session suppressed unless DEBUG is set)`);
+            } else if (process.env.DEBUG) {
+              console.error(`[daemon] Heartbeat rejected for ${sessionId}: ${errMsg}`);
+            }
+          }
         } catch (err) {
-          if (process.env.DEBUG) {
+          // Always log the first heartbeat failure per session so operators
+          // know heartbeats are not reaching the relay. Subsequent failures
+          // for the same session are only logged in DEBUG mode to avoid spam.
+          if (!heartbeatFailureLogged.has(sessionId)) {
+            heartbeatFailureLogged.add(sessionId);
+            console.warn(`[daemon] Heartbeat failed for ${sessionId}: ${err.message} (further failures for this session suppressed unless DEBUG is set)`);
+          } else if (process.env.DEBUG) {
             console.error(`[daemon] Heartbeat failed for ${sessionId}: ${err.message}`);
           }
         }

package/lib/daemon/timeline-analyzer.js

@@ -16,7 +16,7 @@
 export async function fetchTimeline(session_id, config) {
   const { relayApiUrl, apiKey } = config;
 
-  const response = await fetch(`${relayApiUrl}/api/timeline/${session_id}`, {
+  const response = await fetch(`${relayApiUrl}/api/sessions/${session_id}/timeline`, {
     headers: { 'Authorization': `Bearer ${apiKey}` }
   });
 
@@ -36,7 +36,7 @@ export async function fetchTimeline(session_id, config) {
  */
 export function analyzeTaskState(events, task_id) {
   // Filter events for this specific task
-  const taskEvents = events.filter(e => e.meta?.task_id === task_id);
+  const taskEvents = events.filter(e => e.data?.task_id === task_id);
 
   if (taskEvents.length === 0) {
     return {
@@ -53,16 +53,17 @@ export function analyzeTaskState(events, task_id) {
 
   const lastEvent = taskEvents[taskEvents.length - 1];
 
-  // Count turns (each assistant_response indicates a completed turn)
+  // Count turns (each assistant_response with matching task_id indicates a completed turn)
+  // Note: relay stores source inside data object, not at event top level
   const turn_count = taskEvents.filter(e =>
-    e.type === 'assistant_response' && e.source === 'autonomous_task'
+    e.type === 'assistant_response'
   ).length;
 
   // Find the latest claude_session_id from task execution
   let claude_session_id = null;
   for (let i = taskEvents.length - 1; i >= 0; i--) {
-    if (taskEvents[i].meta?.claude_session_id) {
-      claude_session_id = taskEvents[i].meta.claude_session_id;
+    if (taskEvents[i].data?.claude_session_id) {
+      claude_session_id = taskEvents[i].data.claude_session_id;
       break;
     }
   }
@@ -72,7 +73,7 @@ export function analyzeTaskState(events, task_id) {
     e.type === 'approval_requested' &&
     !taskEvents.some(later =>
       later.type === 'approval_decided' &&
-      later.meta?.approval_id === e.meta?.approval_id
+      later.data?.approval_id === e.data?.approval_id
     )
   );
 
@@ -93,7 +94,7 @@ export function analyzeTaskState(events, task_id) {
       state: 'paused',
       turn_count,
       ready_for_execution: false,
-      reason: lastEvent.meta?.reason || 'Task paused',
+      reason: lastEvent.data?.reason || 'Task paused',
       claude_session_id,
       waiting_for_user_message: true,
     };
@@ -105,7 +106,7 @@ export function analyzeTaskState(events, task_id) {
       state: 'stopped',
       turn_count,
       ready_for_execution: false,
-      reason: lastEvent.meta?.reason || 'Task stopped',
+      reason: lastEvent.data?.reason || 'Task stopped',
       claude_session_id,
     };
   }
@@ -134,7 +135,7 @@ export function analyzeTaskState(events, task_id) {
   // If last event is assistant_response, ready for next turn
   // Use stop_reason to determine if Claude is done (works like CLI)
   if (lastEvent.type === 'assistant_response') {
-    const stopReason = lastEvent.meta?.stop_reason;
+    const stopReason = lastEvent.data?.stop_reason;
 
     // Claude uses "end_turn" when it's done with the current turn and waiting for input
     // This is the natural stopping point, just like in the CLI
@@ -183,9 +184,14 @@ export function getNextPrompt(state, task) {
     return task.task;
   }
 
-  // If paused and resumed with user message, use that message
-  if (task.pending_question) {
-    return task.pending_question;
+  // If resumed with a user answer/message, use that as the prompt
+  if (task.pending_answer) {
+    return task.pending_answer;
+  }
+
+  // If redirected with new instructions, use those
+  if (task.pending_redirect) {
+    return task.pending_redirect;
   }
 
   // Default continuation prompt