teleportation-cli 1.1.5 → 1.2.1

package/lib/daemon/transcript-ingestion.js

@@ -0,0 +1,696 @@
+/**
+ * Transcript Ingestion
+ *
+ * Reads Claude Code transcript and pushes events to timeline.
+ * Ensures timeline completeness even if hooks fail to log events.
+ *
+ * @module lib/daemon/transcript-ingestion
+ */
+
+import { readFile, readdir } from 'fs/promises';
+import { homedir, tmpdir } from 'os';
+import { join } from 'path';
+
+// ============================================================================
+// Configuration Constants
+// ============================================================================
+
+/**
+ * Maximum events to process in real-time mode (post_tool_use hook)
+ *
+ * Rationale for 10 events:
+ * - Fast execution: Hook runs after each tool, must not block Claude
+ * - Low latency: ~100ms total (10 events * 10ms/event)
+ * - Recent context: Most relevant events are the last few
+ * - Hook timeout: Well under Claude's hook timeout limits
+ *
+ * Tuning:
+ * - Increase to 20 if seeing missing events in timeline
+ * - Decrease to 5 if hooks are timing out
+ */
+const DEFAULT_MAX_EVENTS_REALTIME = 10;
+
+/**
+ * Maximum events to process in daemon mode (polling loop)
+ *
+ * Rationale for 100 events:
+ * - Thorough: Catches most backlog from inactive periods
+ * - Non-blocking: Daemon polls every 5 seconds, 100 events = ~1-2 seconds
+ * - Memory: 100 events * ~5KB = ~500KB (acceptable footprint)
+ * - Balance: Processes recent activity without freezing on huge backlogs
+ *
+ * Tuning:
+ * - Increase to 200 if daemon falls behind on active sessions
+ * - Decrease to 50 if daemon is spending too long on ingestion
+ */
+const DEFAULT_MAX_EVENTS_DAEMON = 100;
+
+/**
+ * Batch chunk size for timeline event ingestion
+ *
+ * Rationale for 50 events/batch:
+ * - Network efficiency: 100 events = 2 batches (98% reduction from 100 individual POSTs)
+ * - Resilience: Batch failure only affects 50 events (not entire backlog)
+ * - API limits: ~250KB per batch (50 events * 5KB) - well under typical limits
+ * - Timeout risk: ~500ms per batch (50 * 10ms) - low timeout probability
+ * - Memory: Small footprint, safe for daemon/hook contexts
+ * - Fallback: Individual POST fallback ensures no data loss on batch failure
+ *
+ * Performance comparison (100 events):
+ * - Individual POSTs: 100 requests * 50ms = 5,000ms (5 seconds)
+ * - Batched (size=50): 2 requests * 500ms = 1,000ms (1 second) ← 80% faster
+ * - Batched (size=100): 1 request * 1,000ms = 1,000ms (same speed, but higher risk)
+ *
+ * Tuning considerations:
+ * - Increase to 100: Better network efficiency, but batch timeout risk increases
+ * - Decrease to 20: More resilient, but more HTTP overhead (100 events = 5 batches)
+ * - Monitor: Check relay API `/api/timeline/batch` endpoint for failures/timeouts
+ *
+ * @see pushEventsInChunks for implementation details
+ */
+const BATCH_CHUNK_SIZE = 50;
+
+/**
+ * Maximum length for assistant response text in timeline events
+ *
+ * Rationale for 2000 characters:
+ * - Matches stop hook truncation limit (consistency)
+ * - Provides enough context for timeline viewing
+ * - Keeps event payload sizes reasonable (<10KB)
+ * - Prevents oversized events from breaking timeline queries
+ *
+ * @see extractTimelineEvents
+ */
+const MAX_ASSISTANT_RESPONSE_LENGTH = 2000;
+
+/**
+ * Timestamp validation range: ±1 year from current time
+ *
+ * Rationale for 1 year:
+ * - Prevents clock skew issues (system time drift, NTP sync)
+ * - Catches obviously invalid dates (epoch=0, far future timestamps)
+ * - Allows timezone differences and daylight saving time adjustments
+ * - Lenient enough for legitimate use cases (resuming old sessions)
+ * - Stricter than necessary (could be ±1 week) but safe for edge cases
+ *
+ * @see parseTimestamp
+ */
+const TIMESTAMP_MAX_AGE_MS = 365 * 24 * 60 * 60 * 1000;
+
+/**
+ * Timeline query limit for deduplication
+ *
+ * Rationale for 50 events:
+ * - Catches recent duplicates: Hook and daemon running concurrently
+ * - Small enough: Minimal API overhead (~5KB response)
+ * - Large enough: Covers typical concurrent ingestion window
+ * - Prevents race conditions: If hook posts events 1-10, daemon queries last 50 to detect overlap
+ *
+ * @see ingestTranscriptToTimeline timeline query logic
+ */
+const TIMELINE_QUERY_LIMIT = 50;
+
+/**
+ * Network error retry configuration
+ *
+ * MAX_RETRIES: Maximum retry attempts for individual event POSTs
+ * - Handles transient network errors (timeout, connection reset, DNS failures)
+ * - Non-network errors (4xx, 5xx) are not retried (permanent failures)
+ *
+ * RETRY_BASE_DELAY_MS: Base delay for exponential backoff
+ * - Retry 1: 100ms
+ * - Retry 2: 200ms
+ * - Retry 3: 400ms
+ * - Total max wait: 700ms for 3 retries
+ */
+const MAX_RETRIES = 3;
+const RETRY_BASE_DELAY_MS = 100;
+
+/**
+ * Debug mode flag
+ * Set TELEPORTATION_DEBUG=true to enable verbose logging to tmpdir()
+ */
+const DEBUG = process.env.TELEPORTATION_DEBUG === 'true';
+
+/**
+ * Sleep helper for retry backoff
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Derive Claude Code project slug from working directory
+ * Matches Claude Code's slug generation logic
+ * @param {string} cwd - Working directory
+ * @returns {string} Project slug
+ */
+function getProjectSlug(cwd) {
+  // Claude Code converts absolute path to slug by replacing separators with dashes
+  const absPath = cwd || process.cwd();
+  return '-' + absPath.replace(/[\\/]/g, '-');
+}
+
+/**
+ * Search all project directories for transcript (fallback)
+ * @param {string} claude_session_id - Claude Code session ID
+ * @returns {Promise<Array>} Transcript messages or empty array
+ */
+async function searchForTranscript(claude_session_id) {
+  const projectsDir = join(homedir(), '.claude', 'projects');
+
+  try {
+    const projects = await readdir(projectsDir);
+
+    for (const project of projects) {
+      const transcriptPath = join(projectsDir, project, `${claude_session_id}.jsonl`);
+      try {
+        const content = await readFile(transcriptPath, 'utf8');
+        console.log(`[transcript] Found transcript in project: ${project}`);
+        const lines = content.trim().split('\n').filter(Boolean);
+        return lines.map(line => JSON.parse(line));
+      } catch (e) {
+        // Not in this project, continue searching
+      }
+    }
+  } catch (error) {
+    console.error(`[transcript] Failed to search projects: ${error.message}`);
+  }
+
+  return [];
+}
+
+/**
+ * Read transcript file for a Claude session
+ * @param {string} claude_session_id - Claude Code session ID
+ * @param {string} cwd - Working directory (to derive project slug)
+ * @returns {Promise<Array>} Transcript messages
+ */
+async function readTranscript(claude_session_id, cwd) {
+  // Try reading from derived project slug first
+  const projectSlug = getProjectSlug(cwd);
+  const transcriptPath = join(homedir(), '.claude', 'projects', projectSlug, `${claude_session_id}.jsonl`);
+
+  try {
+    const content = await readFile(transcriptPath, 'utf8');
+    const lines = content.trim().split('\n').filter(Boolean);
+    return lines.map(line => JSON.parse(line));
+  } catch (error) {
+    // If not found at derived path, search all project directories
+    console.log(`[transcript] Not found at ${transcriptPath}, searching all projects...`);
+    return await searchForTranscript(claude_session_id);
+  }
+}
+
+/**
+ * Push events to timeline using chunked batching with individual fallback
+ * @param {Array} events - Timeline events to push
+ * @param {string} relayApiUrl - Relay API URL
+ * @param {string} apiKey - API key
+ * @param {string} parent_session_id - Parent session ID
+ * @param {string} task_id - Task ID (optional)
+ * @param {string} claude_session_id - Claude session ID
+ * @param {Function} log - Logging function
+ * @returns {Promise<{successCount: number, failCount: number}>}
+ */
+async function pushEventsInChunks(events, relayApiUrl, apiKey, parent_session_id, task_id, claude_session_id, log) {
+  let successCount = 0;
+  let failCount = 0;
+
+  // Split events into chunks
+  for (let i = 0; i < events.length; i += BATCH_CHUNK_SIZE) {
+    const chunk = events.slice(i, i + BATCH_CHUNK_SIZE);
+
+    // Try batch POST first
+    try {
+      const response = await fetch(`${relayApiUrl}/api/timeline/batch`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${apiKey}`
+        },
+        body: JSON.stringify({
+          session_id: parent_session_id,
+          events: chunk.map(event => ({
+            type: event.type,
+            source: event.source,
+            data: {
+              ...event.meta,
+              task_id,
+              claude_session_id,
+              timestamp: event.timestamp,
+            }
+          }))
+        })
+      });
+
+      if (response.ok) {
+        successCount += chunk.length;
+        log(`✓ Batch ${Math.floor(i / BATCH_CHUNK_SIZE) + 1}: ${chunk.length} events pushed`);
+        continue; // Success, move to next chunk
+      } else {
+        log(`✗ Batch ${Math.floor(i / BATCH_CHUNK_SIZE) + 1} failed (${response.status}), falling back to individual posts`);
+      }
+    } catch (error) {
+      log(`✗ Batch ${Math.floor(i / BATCH_CHUNK_SIZE) + 1} error: ${error.message}, falling back to individual posts`);
+    }
+
+    // Fallback: Post individually if batch failed
+    for (const event of chunk) {
+      let lastError = null;
+      let success = false;
+
+      // Retry loop with exponential backoff for network errors
+      for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        try {
+          const response = await fetch(`${relayApiUrl}/api/timeline`, {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              'Authorization': `Bearer ${apiKey}`
+            },
+            body: JSON.stringify({
+              session_id: parent_session_id,
+              type: event.type,
+              source: event.source,
+              data: {
+                ...event.meta,
+                task_id,
+                claude_session_id,
+                timestamp: event.timestamp,
+              }
+            })
+          });
+
+          if (response.ok) {
+            successCount++;
+            log(`  ✓ ${event.type}${attempt > 0 ? ` (retry ${attempt})` : ''}`);
+            success = true;
+            break; // Success, exit retry loop
+          } else {
+            // HTTP errors (4xx, 5xx) are permanent failures - don't retry
+            failCount++;
+            const errorText = await response.text();
+            log(`  ✗ ${event.type}: ${response.status} - ${errorText}`);
+            success = false;
+            break; // Don't retry HTTP errors
+          }
+        } catch (error) {
+          // Network errors (timeout, DNS, connection reset) - retry with backoff
+          lastError = error;
+          if (attempt < MAX_RETRIES) {
+            const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
+            log(`  ⟳ ${event.type}: ${error.message} (retry ${attempt + 1}/${MAX_RETRIES} after ${delay}ms)`);
+            await sleep(delay);
+          } else {
+            // Max retries exceeded
+            failCount++;
+            log(`  ✗ ${event.type}: ${error.message} (max retries exceeded)`);
+          }
+        }
+      }
+    }
+  }
+
+  return { successCount, failCount };
+}
+
+/**
+ * Validate and parse timestamp from transcript entry
+ * @param {Object} entry - Transcript entry
+ * @returns {number} Timestamp in milliseconds
+ */
+function parseTimestamp(entry) {
+  if (!entry.timestamp) {
+    console.warn('[transcript] Missing timestamp in entry, using current time');
+    return Date.now();
+  }
+
+  const parsed = new Date(entry.timestamp).getTime();
+
+  // Validate timestamp is reasonable (not NaN, not in distant past/future)
+  if (isNaN(parsed)) {
+    console.warn(`[transcript] Invalid timestamp: ${entry.timestamp}, using current time`);
+    return Date.now();
+  }
+
+  const now = Date.now();
+  const oneYearAgo = now - TIMESTAMP_MAX_AGE_MS;
+  const oneYearFromNow = now + TIMESTAMP_MAX_AGE_MS;
+
+  if (parsed < oneYearAgo || parsed > oneYearFromNow) {
+    console.warn(`[transcript] Timestamp out of reasonable range: ${entry.timestamp}, using current time`);
+    return Date.now();
+  }
+
+  return parsed;
+}
+
+/**
+ * Extract events from transcript that should be in timeline
+ * Supports both old format (message at root) and new format (message nested)
+ * @param {Array} transcript - Transcript messages
+ * @param {number} fromIndex - Start extracting from this index (to avoid duplicates)
+ * @returns {Array} Timeline events to push
+ */
+function extractTimelineEvents(transcript, fromIndex = 0) {
+  const events = [];
+
+  console.log(`[transcript] extractTimelineEvents: processing ${transcript.length - fromIndex} messages from index ${fromIndex}`);
+
+  // Build a lookup map from tool_use_id → { name, input } so that
+  // tool_completed/tool_failed events can include the tool name and input
+  // (transcript tool_result blocks only carry tool_use_id, not the tool name)
+  const toolUseLookup = new Map();
+  for (let i = fromIndex; i < transcript.length; i++) {
+    const entry = transcript[i];
+    const message = entry.message || entry;
+    if (message.role === 'assistant' && Array.isArray(message.content)) {
+      for (const block of message.content) {
+        if (block.type === 'tool_use' && block.id) {
+          toolUseLookup.set(block.id, { name: block.name, input: block.input });
+        }
+      }
+    }
+  }
+
+  for (let i = fromIndex; i < transcript.length; i++) {
+    const entry = transcript[i];
+
+    // Handle both old format (message at root) and new format (message nested)
+    const message = entry.message || entry;
+    const role = message.role;
+    const content = message.content;
+    const timestamp = parseTimestamp(entry);
+
+    // Extract assistant responses
+    // Schema matches stop hook batch: data.message (canonical field name)
+    if (role === 'assistant' && content) {
+      const textContent = Array.isArray(content)
+        ? content.find(c => c.type === 'text')?.text
+        : content;
+
+      const trimmedContent = textContent ? textContent.trim() : '';
+      if (trimmedContent) {
+        events.push({
+          type: 'assistant_response',
+          source: entry.type === 'autonomous_task' ? 'autonomous_task' : 'cli_interactive',
+          timestamp,
+          meta: {
+            message: trimmedContent.slice(0, MAX_ASSISTANT_RESPONSE_LENGTH),
+            full_length: trimmedContent.length,
+            truncated: trimmedContent.length > MAX_ASSISTANT_RESPONSE_LENGTH,
+            stop_reason: message.stop_reason || null,
+            model: message.model || null,
+          }
+        });
+      }
+    }
+
+    // Extract tool uses
+    if (role === 'assistant' && content) {
+      const toolUses = Array.isArray(content)
+        ? content.filter(c => c.type === 'tool_use')
+        : [];
+
+      for (const toolUse of toolUses) {
+        events.push({
+          type: 'tool_use',
+          source: entry.type === 'autonomous_task' ? 'autonomous_task' : 'cli_interactive',
+          timestamp,
+          meta: {
+            tool_name: toolUse.name,
+            tool_use_id: toolUse.id,
+            tool_input: toolUse.input,
+          }
+        });
+      }
+    }
+
+    // Extract tool results as tool_completed/tool_failed events
+    // Schema matches post_tool_use hook canonical format:
+    //   tool_name, tool_input, exit_code, stdout, stderr, tool_use_id, is_error
+    // Note: Transcripts don't carry exit_code or stderr, so we infer exit_code
+    // from is_error (0 for success, 1 for failure) and put content into stdout.
+    if (role === 'user' && content) {
+      const toolResults = Array.isArray(content)
+        ? content.filter(c => c.type === 'tool_result')
+        : [];
+
+      for (const toolResult of toolResults) {
+        const isError = toolResult.is_error || false;
+        const toolInfo = toolUseLookup.get(toolResult.tool_use_id);
+        const resultContent = typeof toolResult.content === 'string'
+          ? toolResult.content.slice(0, 1000)
+          : JSON.stringify(toolResult.content).slice(0, 1000);
+
+        events.push({
+          type: isError ? 'tool_failed' : 'tool_completed',
+          source: entry.type === 'autonomous_task' ? 'autonomous_task' : 'cli_interactive',
+          timestamp,
+          meta: {
+            tool_use_id: toolResult.tool_use_id,
+            tool_name: toolInfo?.name || null,
+            tool_input: toolInfo?.input || null,
+            exit_code: isError ? 1 : 0,
+            stdout: isError ? null : resultContent,
+            stderr: isError ? resultContent : null,
+            is_error: isError,
+          }
+        });
+      }
+    }
+  }
+
+  console.log(`[transcript] extractTimelineEvents: found ${events.length} events`);
+  return events;
+}
+
+/**
+ * Fetch task metadata including last ingested transcript index
+ * @param {string} task_id - Task ID
+ * @param {string} session_id - Session ID
+ * @param {Object} config - Config with relayApiUrl, apiKey
+ * @returns {Promise<Object|null>} Task object or null
+ */
+async function fetchTask(task_id, session_id, config) {
+  const { relayApiUrl, apiKey } = config;
+
+  try {
+    const response = await fetch(
+      `${relayApiUrl}/api/sessions/${session_id}/tasks/${task_id}/status`,
+      {
+        headers: { 'Authorization': `Bearer ${apiKey}` }
+      }
+    );
+
+    if (!response.ok) {
+      return null;
+    }
+
+    return await response.json();
+  } catch (error) {
+    console.error(`[transcript] Failed to fetch task: ${error.message}`);
+    return null;
+  }
+}
+
+/**
+ * Update task's last ingested transcript index
+ * @param {string} task_id - Task ID
+ * @param {string} session_id - Session ID
+ * @param {number} lastIndex - Last ingested index
+ * @param {Object} config - Config with relayApiUrl, apiKey
+ */
+async function updateLastIngestedIndex(task_id, session_id, lastIndex, config) {
+  const { relayApiUrl, apiKey } = config;
+
+  try {
+    await fetch(`${relayApiUrl}/api/sessions/${session_id}/tasks/${task_id}`, {
+      method: 'PATCH',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({ last_transcript_index: lastIndex })
+    });
+  } catch (error) {
+    console.error(`[transcript] Failed to update last index: ${error.message}`);
+  }
+}
+
+/**
+ * Push transcript events to timeline via relay API
+ * Called by daemon after each turn execution
+ *
+ * @param {Object} options
+ * @param {string} options.claude_session_id - Claude session ID
+ * @param {string} options.parent_session_id - Teleportation session ID (for timeline)
+ * @param {string} options.task_id - Task ID (for event metadata)
+ * @param {string} options.cwd - Working directory (to derive project slug)
+ * @param {Object} options.config - Config with relayApiUrl, apiKey
+ * @param {boolean} options.realTimeMode - If true, only process last 10 events (fast, for hooks)
+ * @param {number} options.maxEvents - Maximum events to process (default: 100 for daemon, 10 for realTime)
+ * @returns {Promise<Object>} Result { events_pushed: number }
+ */
+export async function ingestTranscriptToTimeline(options) {
+  const { claude_session_id, parent_session_id, task_id, cwd, config, realTimeMode = false, maxEvents } = options;
+  const { relayApiUrl, apiKey } = config;
+
+  // Determine max events based on mode
+  const effectiveMaxEvents = maxEvents || (realTimeMode ? DEFAULT_MAX_EVENTS_REALTIME : DEFAULT_MAX_EVENTS_DAEMON);
+
+  // File logging for debugging (cross-platform)
+  const { appendFileSync } = await import('node:fs');
+  const logFile = join(tmpdir(), 'daemon-transcript-ingestion.log');
+  const log = (msg) => {
+    if (!DEBUG) return; // Only log when DEBUG mode is enabled
+    try {
+      appendFileSync(logFile, `[${new Date().toISOString()}] ${msg}\n`);
+    } catch (e) {
+      console.error(`[transcript] Failed to write to log: ${e.message}`);
+    }
+  };
+
+  log(`===== INGESTION START (${realTimeMode ? 'REAL-TIME' : 'DAEMON'}, max=${effectiveMaxEvents}) =====`);
+  log(`claude_session: ${claude_session_id}, parent: ${parent_session_id}, task: ${task_id}`);
+
+  console.log(`[transcript] ===== INGESTION START =====`);
+  console.log(`[transcript] claude_session: ${claude_session_id}, parent: ${parent_session_id}, task: ${task_id}`);
+
+  // 1. Read transcript with cwd for project slug derivation
+  const transcript = await readTranscript(claude_session_id, cwd);
+  log(`Read transcript: ${transcript.length} messages`);
+  if (transcript.length === 0) {
+    log(`No transcript found or empty - returning`);
+    console.log(`[transcript] No transcript found or empty`);
+    return { events_pushed: 0 };
+  }
+
+  // 2. Determine starting index for event extraction
+  let fromIndex = 0;
+
+  if (task_id) {
+    // For tasks: Use task's last ingested index
+    const task = await fetchTask(task_id, parent_session_id, config);
+    fromIndex = task?.last_transcript_index || 0;
+    console.log(`[transcript] Using task's last ingested index: ${fromIndex}`);
+  } else {
+    // For regular sessions: Query timeline to find recent events for deduplication
+    // Queries last 50 events (not just 1) to catch concurrent hook/daemon ingestion
+    // This prevents duplicates when hook and daemon run simultaneously
+    console.log(`[transcript] Querying timeline for session ${parent_session_id} to find recent events...`);
+    try {
+      const timelineResponse = await fetch(
+        `${relayApiUrl}/api/sessions/${parent_session_id}/timeline?limit=${TIMELINE_QUERY_LIMIT}`,
+        {
+          headers: { 'Authorization': `Bearer ${apiKey}` }
+        }
+      );
+
+      console.log(`[transcript] Timeline query status: ${timelineResponse.status}`);
+
+      if (timelineResponse.ok) {
+        const response = await timelineResponse.json();
+        // PRD-0022: Timeline API returns { events: [...], pagination: {...} } format
+        const timelineEvents = Array.isArray(response) ? response : (response.events || []);
+        console.log(`[transcript] Timeline returned ${timelineEvents.length} events`);
+
+        if (timelineEvents.length > 0 && timelineEvents[0].timestamp) {
+          const lastTimestamp = timelineEvents[0].timestamp;
+          console.log(`[transcript] Last timeline event timestamp: ${lastTimestamp} (${new Date(lastTimestamp).toISOString()})`);
+
+          // Sample first and last transcript timestamps
+          if (transcript.length > 0) {
+            const firstMsg = transcript[0];
+            const lastMsg = transcript[transcript.length - 1];
+            console.log(`[transcript] First transcript msg: ${firstMsg.timestamp}`);
+            console.log(`[transcript] Last transcript msg: ${lastMsg.timestamp}`);
+          }
+
+          // Find the index of the first message after this timestamp
+          fromIndex = transcript.findIndex(msg => {
+            const msgTimestamp = new Date(msg.timestamp || 0).getTime();
+            return msgTimestamp > lastTimestamp;
+          });
+
+          // If not found, start from end of transcript (all messages are older)
+          if (fromIndex === -1) {
+            log(`All messages older than last timeline event (${new Date(lastTimestamp).toISOString()}) - no new events`);
+            console.log(`[transcript] All ${transcript.length} messages are older than last timeline event - no new events to ingest`);
+            fromIndex = transcript.length;
+          } else {
+            log(`Found ${transcript.length - fromIndex} new messages starting from index ${fromIndex}/${transcript.length}`);
+            console.log(`[transcript] Found ${transcript.length - fromIndex} new messages starting from index ${fromIndex}/${transcript.length}`);
+          }
+        } else {
+          console.log(`[transcript] No timeline events found - will process all transcript messages`);
+          fromIndex = 0;
+        }
+      } else {
+        console.log(`[transcript] Timeline query failed with status ${timelineResponse.status} - processing all messages`);
+        fromIndex = 0;
+      }
+    } catch (error) {
+      console.log(`[transcript] Failed to query timeline: ${error.message}, processing all messages`);
+      fromIndex = 0;
+    }
+  }
+
+  // 3. Extract only NEW events from determined index
+  const allEvents = extractTimelineEvents(transcript, fromIndex);
+
+  log(`Extracted ${allEvents.length} events from transcript`);
+  if (allEvents.length === 0) {
+    log(`No new events since index ${fromIndex} - returning`);
+    console.log(`[transcript] No new events since index ${fromIndex} (transcript length: ${transcript.length})`);
+    return { events_pushed: 0 };
+  }
+
+  // 4. Limit events to process (most recent N events)
+  // This prevents blocking the daemon or hooks with large backlogs
+  const events = allEvents.slice(-effectiveMaxEvents);
+  const skipped = allEvents.length - events.length;
+
+  if (skipped > 0) {
+    log(`Limited to most recent ${events.length} events (skipped ${skipped} older events)`);
+    console.log(`[transcript] Limited to most recent ${events.length}/${allEvents.length} events`);
+  } else {
+    log(`Processing all ${events.length} events`);
+    console.log(`[transcript] Processing all ${events.length} events`);
+  }
+
+  // 5. Push events using chunked batching (with individual fallback on failure)
+  // This balances network efficiency (fewer requests) with resilience (individual fallback)
+  const { successCount, failCount } = await pushEventsInChunks(
+    events,
+    relayApiUrl,
+    apiKey,
+    parent_session_id,
+    task_id,
+    claude_session_id,
+    log
+  );
+
+  log(`===== INGESTION COMPLETE: ${successCount}/${events.length} events pushed (${failCount} failed) =====`);
+  console.log(`[transcript] Pushed ${successCount}/${events.length} events (${failCount} failed)`);
+
+  // 6. Update last ingested index to prevent duplicates on next call (tasks only)
+  if (task_id && successCount > 0) {
+    await updateLastIngestedIndex(task_id, parent_session_id, transcript.length, config);
+  }
+
+  return { events_pushed: successCount, events_failed: failCount };
+}
+
+/**
+ * Get transcript length to track ingestion progress
+ * @param {string} claude_session_id - Claude session ID
+ * @returns {Promise<number>} Number of messages in transcript
+ */
+export async function getTranscriptLength(claude_session_id) {
+  const transcript = await readTranscript(claude_session_id);
+  return transcript.length;
+}