tracegist-mcp-bridge 0.2.7 → 0.2.8

package/bin/tracegist-mcp-bridge.mjs

@@ -957,9 +957,12 @@ async function transcribeLiveVoice(voiceBlobDataUrl) {
   return { error: "No transcription available (set OPENROUTER_API_KEY or install Whisper)" };
 }
 
-/** Push a live event into the ring buffer with auto-incrementing seq. */
+/** Push a live event into the ring buffer with auto-incrementing seq.
+ *  Seq is always higher than any existing event (including extension-originated ones). */
 function pushLiveEvent(eventType, data) {
-  liveEventSeq++;
+  // Ensure our seq is always higher than the max seq in the buffer
+  const maxExistingSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
+  liveEventSeq = Math.max(liveEventSeq, maxExistingSeq) + 1;
   liveEvents.push({ seq: liveEventSeq, ts: Date.now(), eventType, data });
   if (liveEvents.length > LIVE_SHADOW_EVENT_BUFFER_SIZE) {
     const evictCount = Math.floor(LIVE_SHADOW_EVENT_BUFFER_SIZE * 0.25);
@@ -1045,6 +1048,7 @@ function handleExtensionMessage(msg) {
         startedAt: Date.now(),
       };
       liveEvents.length = 0;
+      liveEventSeq = 0;
       pendingQuestions.length = 0;
       questionResponses.clear();
       liveInteractions.length = 0;
@@ -1089,6 +1093,54 @@ function handleExtensionMessage(msg) {
         highlightCount: msg.highlightCaptures?.length || 0,
         hint: "Use get_tester_response to retrieve the full response with transcription and images.",
       });
+      // Eagerly transcribe the voice response and log the interaction immediately
+      // (don't wait for get_tester_response to be called)
+      if (msg.voiceBlobDataUrl) {
+        transcribeLiveVoice(msg.voiceBlobDataUrl)
+          .then((result) => {
+            // Store transcription with the response for later retrieval
+            const stored = questionResponses.get(msg.questionId);
+            if (stored && !stored.dismissed) {
+              stored.transcription = result.transcription || null;
+            }
+            // Log the interaction with transcription
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: result.transcription || null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+            // Push transcription as a live event so polling agents see it
+            pushLiveEvent("question-response-transcribed", {
+              questionId: msg.questionId,
+              transcription: result.transcription || null,
+              error: result.error || undefined,
+            });
+          })
+          .catch((err) => {
+            console.error(`[${BRIDGE_NAME}] Question response transcription failed:`, err);
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+          });
+      } else {
+        // No voice — log interaction immediately
+        liveInteractions.push({
+          type: "tester-response",
+          questionId: msg.questionId,
+          timestamp: msg.timestamp || Date.now(),
+          transcription: null,
+          screenshot: !!msg.screenshot,
+          highlightCount: msg.highlightCaptures?.length || 0,
+        });
+      }
       break;
     }
 
@@ -1188,8 +1240,11 @@ server.tool(
     "- You MUST keep calling this tool every 2-4 seconds in a loop until sessionActive becomes false.\n" +
     "- Do NOT stop polling just because there are no new events — the tester is still active.\n" +
     "- When you see 'voice-transcription' events, the tester recorded a voice marker — read the transcription and react to it.\n" +
-    "- When you see 'question-response-received' events, use get_tester_response to retrieve the full answer.\n" +
+    "- When you see 'question-response-received' events, use get_tester_response to retrieve the full answer with images.\n" +
+    "- When you see 'question-response-transcribed' events, the tester's voice response has been transcribed — read it for context.\n" +
+    "- Marker events with hasPendingVoice=true will have their voiceTranscription field populated once transcription completes.\n" +
     "- The tester may set voice markers at any time to communicate with you — keep watching for them.\n" +
+    "- If you need clarification, use ask_tester_question to show a toast notification to the tester.\n" +
     "- Only stop polling when sessionActive is false (the recording ended).",
   {
     since_seq: z
@@ -1209,12 +1264,33 @@ server.tool(
 
     const latestSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
 
+    // Enrich marker events with their transcriptions (if available in the buffer)
+    // so agents don't have to correlate separate marker + voice-transcription events
+    const transcriptionsByMarkerId = new Map();
+    for (const e of liveEvents) {
+      if (e.eventType === "voice-transcription" && e.data?.markerId) {
+        transcriptionsByMarkerId.set(e.data.markerId, e.data.transcription);
+      }
+    }
+    const enrichedEvents = events.map((e) => {
+      if (e.eventType === "marker" && e.data?.markerId && transcriptionsByMarkerId.has(e.data.markerId)) {
+        return {
+          ...e,
+          data: {
+            ...e.data,
+            voiceTranscription: transcriptionsByMarkerId.get(e.data.markerId),
+          },
+        };
+      }
+      return e;
+    });
+
     const responseData = {
       sessionActive: liveSessionActive,
       session: liveSessionMeta,
-      events,
+      events: enrichedEvents,
       latestSeq,
-      eventCount: events.length,
+      eventCount: enrichedEvents.length,
     };
 
     if (liveSessionActive) {
@@ -1241,12 +1317,23 @@ server.tool(
           waitingSince: Math.round((Date.now() - q.sentAt) / 1000) + "s ago",
         }));
       }
-    } else if (liveInteractions.length > 0) {
-      // Session ended — include full interaction log for the agent to incorporate
-      responseData.sessionEndedNote =
-        "Session has ended. The interactionLog below contains all agent-tester exchanges from this session. " +
-        "Include these in your analysis — they represent collaborative context between agent and tester.";
-      responseData.interactionLog = liveInteractions;
+    } else {
+      // Session ended — include full interaction log and voice transcriptions
+      if (liveInteractions.length > 0 || liveEvents.some((e) => e.eventType === "voice-transcription")) {
+        responseData.sessionEndedNote =
+          "Session has ended. The interactionLog and voiceMarkerTranscriptions below contain all agent-tester exchanges from this session. " +
+          "Include these in your analysis — they represent collaborative context between agent and tester. " +
+          "You can also call get_live_session_summary for the complete summary.";
+        responseData.interactionLog = liveInteractions;
+        // Include all voice marker transcriptions
+        responseData.voiceMarkerTranscriptions = liveEvents
+          .filter((e) => e.eventType === "voice-transcription" && e.data?.transcription)
+          .map((e) => ({
+            timestamp: e.ts,
+            markerId: e.data.markerId,
+            transcription: e.data.transcription,
+          }));
+      }
     }
 
     // Always include interaction count so the agent knows exchanges happened
@@ -1428,33 +1515,30 @@ server.tool(
       }
     }
 
-    let transcriptionText = null;
     if (response.voiceBlobDataUrl) {
-      const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
-      if (voiceResult.transcription) {
-        transcriptionText = voiceResult.transcription;
+      // Use pre-transcribed result if available (from eager transcription on response arrival)
+      if (response.transcription) {
         content.push({
           type: "text",
-          text: `Voice transcription:\n${voiceResult.transcription}`,
+          text: `Voice transcription:\n${response.transcription}`,
         });
       } else {
-        content.push({
-          type: "text",
-          text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
-        });
+        // Fallback: transcribe now if eager transcription hasn't completed yet
+        const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
+        if (voiceResult.transcription) {
+          content.push({
+            type: "text",
+            text: `Voice transcription:\n${voiceResult.transcription}`,
+          });
+        } else {
+          content.push({
+            type: "text",
+            text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
+          });
+        }
       }
     }
 
-    // Log the interaction for the session history
-    liveInteractions.push({
-      type: "tester-response",
-      questionId: question_id,
-      timestamp: Date.now(),
-      transcription: transcriptionText,
-      screenshot: !!response.screenshot,
-      highlightCount: response.highlightCaptures?.length || 0,
-    });
-
     content.push({
       type: "text",
       text: "\nIMPORTANT: Resume polling watch_live_session to continue watching the session.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tracegist-mcp-bridge",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "Local-first MCP bridge for reading and transcribing TraceGist package zips.",
   "type": "module",
   "bin": {