tracegist-mcp-bridge 0.2.6 → 0.2.8

package/bin/tracegist-mcp-bridge.mjs

@@ -866,6 +866,14 @@ const pendingQuestions = [];
 /** @type {Map<string, object>} */
 const questionResponses = new Map();
 
+/**
+ * Persistent log of all agent-tester interactions during the live session.
+ * Survives until the next session-start. Used to build the complete interaction
+ * history for the final artifact.
+ * @type {Array<{type: string, questionId: string, question?: string, timestamp: number, transcription?: string, screenshot?: boolean, highlightCount?: number, dismissed?: boolean}>}
+ */
+const liveInteractions = [];
+
 /** @type {Array<{resolve: Function, reject: Function, timeout: ReturnType<typeof setTimeout>}>} */
 const screenshotWaiters = [];
 
@@ -949,9 +957,12 @@ async function transcribeLiveVoice(voiceBlobDataUrl) {
   return { error: "No transcription available (set OPENROUTER_API_KEY or install Whisper)" };
 }
 
-/** Push a live event into the ring buffer with auto-incrementing seq. */
+/** Push a live event into the ring buffer with auto-incrementing seq.
+ *  Seq is always higher than any existing event (including extension-originated ones). */
 function pushLiveEvent(eventType, data) {
-  liveEventSeq++;
+  // Ensure our seq is always higher than the max seq in the buffer
+  const maxExistingSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
+  liveEventSeq = Math.max(liveEventSeq, maxExistingSeq) + 1;
   liveEvents.push({ seq: liveEventSeq, ts: Date.now(), eventType, data });
   if (liveEvents.length > LIVE_SHADOW_EVENT_BUFFER_SIZE) {
     const evictCount = Math.floor(LIVE_SHADOW_EVENT_BUFFER_SIZE * 0.25);
@@ -1037,8 +1048,10 @@ function handleExtensionMessage(msg) {
         startedAt: Date.now(),
       };
       liveEvents.length = 0;
+      liveEventSeq = 0;
       pendingQuestions.length = 0;
       questionResponses.clear();
+      liveInteractions.length = 0;
       console.error(`[${BRIDGE_NAME}] Live shadow: session started (${msg.sessionId})`);
       // Notify MCP clients that resources changed
       try {
@@ -1072,6 +1085,62 @@ function handleExtensionMessage(msg) {
       });
       const rIdx = pendingQuestions.findIndex((q) => q.questionId === msg.questionId);
       if (rIdx >= 0) pendingQuestions.splice(rIdx, 1);
+      // Push a live event so polling agents see the response immediately
+      pushLiveEvent("question-response-received", {
+        questionId: msg.questionId,
+        hasVoice: !!msg.voiceBlobDataUrl,
+        hasScreenshot: !!msg.screenshot,
+        highlightCount: msg.highlightCaptures?.length || 0,
+        hint: "Use get_tester_response to retrieve the full response with transcription and images.",
+      });
+      // Eagerly transcribe the voice response and log the interaction immediately
+      // (don't wait for get_tester_response to be called)
+      if (msg.voiceBlobDataUrl) {
+        transcribeLiveVoice(msg.voiceBlobDataUrl)
+          .then((result) => {
+            // Store transcription with the response for later retrieval
+            const stored = questionResponses.get(msg.questionId);
+            if (stored && !stored.dismissed) {
+              stored.transcription = result.transcription || null;
+            }
+            // Log the interaction with transcription
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: result.transcription || null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+            // Push transcription as a live event so polling agents see it
+            pushLiveEvent("question-response-transcribed", {
+              questionId: msg.questionId,
+              transcription: result.transcription || null,
+              error: result.error || undefined,
+            });
+          })
+          .catch((err) => {
+            console.error(`[${BRIDGE_NAME}] Question response transcription failed:`, err);
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+          });
+      } else {
+        // No voice — log interaction immediately
+        liveInteractions.push({
+          type: "tester-response",
+          questionId: msg.questionId,
+          timestamp: msg.timestamp || Date.now(),
+          transcription: null,
+          screenshot: !!msg.screenshot,
+          highlightCount: msg.highlightCaptures?.length || 0,
+        });
+      }
       break;
     }
 
@@ -1079,6 +1148,14 @@ function handleExtensionMessage(msg) {
       questionResponses.set(msg.questionId, { dismissed: true });
       const dIdx = pendingQuestions.findIndex((q) => q.questionId === msg.questionId);
       if (dIdx >= 0) pendingQuestions.splice(dIdx, 1);
+      // Log dismissal in interactions
+      liveInteractions.push({
+        type: "question-dismissed",
+        questionId: msg.questionId,
+        timestamp: Date.now(),
+        dismissed: true,
+      });
+      pushLiveEvent("question-dismissed", { questionId: msg.questionId });
       break;
     }
 
@@ -1099,6 +1176,13 @@ function handleExtensionMessage(msg) {
             transcription: result.transcription || null,
             error: result.error || undefined,
           });
+          // Log voice marker in interactions for the final artifact
+          liveInteractions.push({
+            type: "voice-marker",
+            questionId: markerId,
+            timestamp: Date.now(),
+            transcription: result.transcription || null,
+          });
         })
         .catch((err) => {
           console.error(`[${BRIDGE_NAME}] Live voice transcription failed:`, err);
@@ -1151,9 +1235,17 @@ server.resource(
 // Tool: watch_live_session
 server.tool(
   "watch_live_session",
-  "Watch a live TraceGist shadowing session. Returns buffered events since the given sequence number. " +
-    "Poll every 3-5 seconds to watch events in real-time. " +
-    "Returns sessionActive: false when the session ends.",
+  "Watch a live TraceGist shadowing session in real-time. Returns buffered events since the given sequence number.\n\n" +
+    "IMPORTANT POLLING BEHAVIOR:\n" +
+    "- You MUST keep calling this tool every 2-4 seconds in a loop until sessionActive becomes false.\n" +
+    "- Do NOT stop polling just because there are no new events — the tester is still active.\n" +
+    "- When you see 'voice-transcription' events, the tester recorded a voice marker — read the transcription and react to it.\n" +
+    "- When you see 'question-response-received' events, use get_tester_response to retrieve the full answer with images.\n" +
+    "- When you see 'question-response-transcribed' events, the tester's voice response has been transcribed — read it for context.\n" +
+    "- Marker events with hasPendingVoice=true will have their voiceTranscription field populated once transcription completes.\n" +
+    "- The tester may set voice markers at any time to communicate with you — keep watching for them.\n" +
+    "- If you need clarification, use ask_tester_question to show a toast notification to the tester.\n" +
+    "- Only stop polling when sessionActive is false (the recording ended).",
   {
     since_seq: z
       .number()
@@ -1172,37 +1264,88 @@ server.tool(
 
     const latestSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
 
+    // Enrich marker events with their transcriptions (if available in the buffer)
+    // so agents don't have to correlate separate marker + voice-transcription events
+    const transcriptionsByMarkerId = new Map();
+    for (const e of liveEvents) {
+      if (e.eventType === "voice-transcription" && e.data?.markerId) {
+        transcriptionsByMarkerId.set(e.data.markerId, e.data.transcription);
+      }
+    }
+    const enrichedEvents = events.map((e) => {
+      if (e.eventType === "marker" && e.data?.markerId && transcriptionsByMarkerId.has(e.data.markerId)) {
+        return {
+          ...e,
+          data: {
+            ...e.data,
+            voiceTranscription: transcriptionsByMarkerId.get(e.data.markerId),
+          },
+        };
+      }
+      return e;
+    });
+
+    const responseData = {
+      sessionActive: liveSessionActive,
+      session: liveSessionMeta,
+      events: enrichedEvents,
+      latestSeq,
+      eventCount: enrichedEvents.length,
+    };
+
+    if (liveSessionActive) {
+      responseData.pollingGuidance =
+        "KEEP POLLING every 2-4 seconds. Do NOT stop — the tester is still active. " +
+        "Watch for 'voice-transcription' events (tester voice markers) and 'question-response-received' events. " +
+        "React to voice markers by acknowledging them or asking follow-up questions.";
+      responseData.availableActions = [
+        {
+          tool: "ask_tester_question",
+          description:
+            "Ask the tester a short question (max 200 chars). They respond with voice + highlights.",
+        },
+        {
+          tool: "get_live_screenshot",
+          description:
+            "Capture a screenshot of the tester's current browser tab.",
+        },
+      ];
+      if (pendingQuestions.length > 0) {
+        responseData.pendingQuestions = pendingQuestions.map((q) => ({
+          questionId: q.questionId,
+          question: q.question,
+          waitingSince: Math.round((Date.now() - q.sentAt) / 1000) + "s ago",
+        }));
+      }
+    } else {
+      // Session ended — include full interaction log and voice transcriptions
+      if (liveInteractions.length > 0 || liveEvents.some((e) => e.eventType === "voice-transcription")) {
+        responseData.sessionEndedNote =
+          "Session has ended. The interactionLog and voiceMarkerTranscriptions below contain all agent-tester exchanges from this session. " +
+          "Include these in your analysis — they represent collaborative context between agent and tester. " +
+          "You can also call get_live_session_summary for the complete summary.";
+        responseData.interactionLog = liveInteractions;
+        // Include all voice marker transcriptions
+        responseData.voiceMarkerTranscriptions = liveEvents
+          .filter((e) => e.eventType === "voice-transcription" && e.data?.transcription)
+          .map((e) => ({
+            timestamp: e.ts,
+            markerId: e.data.markerId,
+            transcription: e.data.transcription,
+          }));
+      }
+    }
+
+    // Always include interaction count so the agent knows exchanges happened
+    if (liveInteractions.length > 0) {
+      responseData.interactionCount = liveInteractions.length;
+    }
+
     return {
       content: [
         {
           type: "text",
-          text: JSON.stringify(
-            {
-              sessionActive: liveSessionActive,
-              session: liveSessionMeta,
-              events,
-              latestSeq,
-              eventCount: events.length,
-              ...(liveSessionActive
-                ? {
-                    availableActions: [
-                      {
-                        tool: "ask_tester_question",
-                        description:
-                          "Ask the tester a short question (max 200 chars). They respond with voice + highlights.",
-                      },
-                      {
-                        tool: "get_live_screenshot",
-                        description:
-                          "Capture a screenshot of the tester's current browser tab.",
-                      },
-                    ],
-                  }
-                : {}),
-            },
-            null,
-            2,
-          ),
+          text: JSON.stringify(responseData, null, 2),
         },
       ],
     };
@@ -1248,6 +1391,17 @@ server.tool(
       pendingQuestions.shift();
     }
 
+    // Log the interaction
+    liveInteractions.push({
+      type: "agent-question",
+      questionId,
+      question,
+      timestamp: Date.now(),
+    });
+
+    // Push a live event so the polling loop sees it
+    pushLiveEvent("agent-question-sent", { questionId, question });
+
     return {
       content: [
         {
@@ -1260,7 +1414,9 @@ server.tool(
             "- Voice marker: Alt/Option + Shift + M (start/stop recording)",
             "- Highlight captures: Alt/Option + Shift + S (draw on screen)",
             "",
-            "Use `get_tester_response` with this question ID to check for their answer.",
+            "IMPORTANT: Keep polling watch_live_session — you will see a 'question-response-received' event when they respond.",
+            "Then use `get_tester_response` with this question ID to retrieve their full answer.",
+            "The tester may take 10-30 seconds to record their voice response. Be patient.",
           ].join("\n"),
         },
       ],
@@ -1272,7 +1428,11 @@ server.tool(
 server.tool(
   "get_tester_response",
   "Check for the tester's response to a question asked during live shadowing. " +
-    "Returns the tester's screenshot, highlight captures, and voice note availability.",
+    "Returns the tester's screenshot, highlight captures, and transcribed voice note.\n\n" +
+    "IMPORTANT: The tester needs time to record their answer (10-60 seconds). " +
+    "Instead of calling this tool repeatedly, prefer polling watch_live_session — " +
+    "you will see a 'question-response-received' event when the response arrives. " +
+    "Then call this tool once to get the full response with transcription and images.",
   {
     question_id: z.string().describe("The question ID returned by ask_tester_question."),
   },
@@ -1280,11 +1440,22 @@ server.tool(
     const response = questionResponses.get(question_id);
 
     if (!response) {
+      // Check how long we've been waiting
+      const pending = pendingQuestions.find((q) => q.questionId === question_id);
+      const waitingSec = pending ? Math.round((Date.now() - pending.sentAt) / 1000) : null;
       return {
         content: [
           {
             type: "text",
-            text: "No response yet. The tester may still be recording their answer. Try again in a few seconds.",
+            text: [
+              "No response yet." +
+                (waitingSec !== null ? ` Waiting for ${waitingSec}s.` : ""),
+              "The tester may still be recording their voice answer.",
+              "",
+              "TIP: Instead of polling this tool, go back to polling watch_live_session — " +
+                "a 'question-response-received' event will appear when the tester responds. " +
+                "Then call this tool once to retrieve the full answer.",
+            ].join("\n"),
           },
         ],
       };
@@ -1345,20 +1516,34 @@ server.tool(
     }
 
     if (response.voiceBlobDataUrl) {
-      const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
-      if (voiceResult.transcription) {
+      // Use pre-transcribed result if available (from eager transcription on response arrival)
+      if (response.transcription) {
         content.push({
           type: "text",
-          text: `Voice transcription:\n${voiceResult.transcription}`,
+          text: `Voice transcription:\n${response.transcription}`,
         });
       } else {
-        content.push({
-          type: "text",
-          text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
-        });
+        // Fallback: transcribe now if eager transcription hasn't completed yet
+        const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
+        if (voiceResult.transcription) {
+          content.push({
+            type: "text",
+            text: `Voice transcription:\n${voiceResult.transcription}`,
+          });
+        } else {
+          content.push({
+            type: "text",
+            text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
+          });
+        }
       }
     }
 
+    content.push({
+      type: "text",
+      text: "\nIMPORTANT: Resume polling watch_live_session to continue watching the session.",
+    });
+
     // Clean up after retrieval
     questionResponses.delete(question_id);
 
@@ -1420,6 +1605,50 @@ server.tool(
   },
 );
 
+// Tool: get_live_session_summary
+server.tool(
+  "get_live_session_summary",
+  "Get a summary of the live shadowing session including all agent-tester interactions. " +
+    "Call this after the session ends (sessionActive becomes false) to get the complete interaction log " +
+    "for incorporating into your analysis. This includes all questions asked, tester responses, " +
+    "voice transcriptions, and voice marker transcriptions from the session.",
+  {},
+  async () => {
+    if (!liveSessionMeta && liveInteractions.length === 0) {
+      return toolError("No live session data available. Start a live shadowing session first.");
+    }
+
+    // Collect voice-transcription events from the event buffer
+    const voiceTranscriptions = liveEvents
+      .filter((e) => e.eventType === "voice-transcription" && e.data?.transcription)
+      .map((e) => ({
+        timestamp: e.ts,
+        markerId: e.data.markerId,
+        transcription: e.data.transcription,
+      }));
+
+    const summary = {
+      session: liveSessionMeta,
+      sessionActive: liveSessionActive,
+      interactionLog: liveInteractions,
+      voiceMarkerTranscriptions: voiceTranscriptions,
+      totalEvents: liveEvents.length,
+      guidance: liveSessionActive
+        ? "Session is still active. Resume polling watch_live_session."
+        : "Session has ended. Use this interaction log and voice transcriptions to enrich your analysis of the exported session package.",
+    };
+
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(summary, null, 2),
+        },
+      ],
+    };
+  },
+);
+
 // ---------------------------------------------------------------------------
 // Startup
 // ---------------------------------------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tracegist-mcp-bridge",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "Local-first MCP bridge for reading and transcribing TraceGist package zips.",
   "type": "module",
   "bin": {