tracegist-mcp-bridge 0.2.7 → 0.2.11

package/README.md

@@ -43,6 +43,10 @@ Set `TRACEGIST_DIR` to change where the bridge looks for packages (defaults to `
 - `read_tracegist_package_file` — read any text file from inside the ZIP without writing to disk (e.g. `network/api-requests.jsonl`, the Playwright repro script)
 - `extract_tracegist_package_file` — write a binary or text file from the ZIP to a local directory
 - `transcribe_tracegist_package_voice_notes` — transcribe voice notes using local Python Whisper
+- `watch_live_session` — poll real-time session events while recording is active
+- `get_live_screenshot` — request and retrieve a screenshot of the current tab
+- `ask_tester_question` — send a question to the tester (shown as a browser toast, max 200 chars)
+- `get_tester_response` — retrieve the tester's voice + screenshot answer
 
 ## Package contents
 

package/bin/lib.mjs

@@ -89,6 +89,14 @@ const SECTION_HINTS = {
   "marker visual evidence": "Screenshot references for each marker",
   "environment at marker time": "Environment snapshot at a specific marker",
   "marker timeline logs (context window)": "Logs within the ±5 s marker window",
+  "webapp testing reproduction":
+    "Server setup, Python repro script, verification points, key selectors",
+  "tester intent summary": "Classified intent: specifications, issues found, observations",
+  "iterative context": "Previous session reference, verification checklist for build-test cycle",
+  "tier 0: quick summary": "Session overview, intent, reproduction command (~200 tokens)",
+  "tier 1: findings": "Marker timeline with structured findings (~500 tokens)",
+  "tier 2: full context": "Interaction timeline, network/console context, environment",
+  "tier 3: deep diagnostics": "Full session timelines (deep profile only)",
 };
 
 function getSectionHint(sectionName) {

package/bin/tracegist-mcp-bridge.mjs

@@ -928,7 +928,9 @@ async function transcribeLiveVoice(voiceBlobDataUrl) {
       const text = result.choices?.[0]?.message?.content?.trim();
       return text ? { transcription: text } : { error: "Empty transcription response" };
     } catch (err) {
-      return { error: `OpenRouter transcription failed: ${err instanceof Error ? err.message : String(err)}` };
+      return {
+        error: `OpenRouter transcription failed: ${err instanceof Error ? err.message : String(err)}`,
+      };
     }
   }
 
@@ -950,16 +952,21 @@ async function transcribeLiveVoice(voiceBlobDataUrl) {
         await fs.rm(tmpDir, { recursive: true, force: true });
       }
     } catch (err) {
-      return { error: `Whisper transcription failed: ${err instanceof Error ? err.message : String(err)}` };
+      return {
+        error: `Whisper transcription failed: ${err instanceof Error ? err.message : String(err)}`,
+      };
     }
   }
 
   return { error: "No transcription available (set OPENROUTER_API_KEY or install Whisper)" };
 }
 
-/** Push a live event into the ring buffer with auto-incrementing seq. */
+/** Push a live event into the ring buffer with auto-incrementing seq.
+ *  Seq is always higher than any existing event (including extension-originated ones). */
 function pushLiveEvent(eventType, data) {
-  liveEventSeq++;
+  // Ensure our seq is always higher than the max seq in the buffer
+  const maxExistingSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
+  liveEventSeq = Math.max(liveEventSeq, maxExistingSeq) + 1;
   liveEvents.push({ seq: liveEventSeq, ts: Date.now(), eventType, data });
   if (liveEvents.length > LIVE_SHADOW_EVENT_BUFFER_SIZE) {
     const evictCount = Math.floor(LIVE_SHADOW_EVENT_BUFFER_SIZE * 0.25);
@@ -1045,6 +1052,7 @@ function handleExtensionMessage(msg) {
         startedAt: Date.now(),
       };
       liveEvents.length = 0;
+      liveEventSeq = 0;
       pendingQuestions.length = 0;
       questionResponses.clear();
       liveInteractions.length = 0;
@@ -1089,6 +1097,54 @@ function handleExtensionMessage(msg) {
         highlightCount: msg.highlightCaptures?.length || 0,
         hint: "Use get_tester_response to retrieve the full response with transcription and images.",
       });
+      // Eagerly transcribe the voice response and log the interaction immediately
+      // (don't wait for get_tester_response to be called)
+      if (msg.voiceBlobDataUrl) {
+        transcribeLiveVoice(msg.voiceBlobDataUrl)
+          .then((result) => {
+            // Store transcription with the response for later retrieval
+            const stored = questionResponses.get(msg.questionId);
+            if (stored && !stored.dismissed) {
+              stored.transcription = result.transcription || null;
+            }
+            // Log the interaction with transcription
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: result.transcription || null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+            // Push transcription as a live event so polling agents see it
+            pushLiveEvent("question-response-transcribed", {
+              questionId: msg.questionId,
+              transcription: result.transcription || null,
+              error: result.error || undefined,
+            });
+          })
+          .catch((err) => {
+            console.error(`[${BRIDGE_NAME}] Question response transcription failed:`, err);
+            liveInteractions.push({
+              type: "tester-response",
+              questionId: msg.questionId,
+              timestamp: msg.timestamp || Date.now(),
+              transcription: null,
+              screenshot: !!msg.screenshot,
+              highlightCount: msg.highlightCaptures?.length || 0,
+            });
+          });
+      } else {
+        // No voice — log interaction immediately
+        liveInteractions.push({
+          type: "tester-response",
+          questionId: msg.questionId,
+          timestamp: msg.timestamp || Date.now(),
+          transcription: null,
+          screenshot: !!msg.screenshot,
+          highlightCount: msg.highlightCaptures?.length || 0,
+        });
+      }
       break;
     }
 
@@ -1188,8 +1244,11 @@ server.tool(
     "- You MUST keep calling this tool every 2-4 seconds in a loop until sessionActive becomes false.\n" +
     "- Do NOT stop polling just because there are no new events — the tester is still active.\n" +
     "- When you see 'voice-transcription' events, the tester recorded a voice marker — read the transcription and react to it.\n" +
-    "- When you see 'question-response-received' events, use get_tester_response to retrieve the full answer.\n" +
+    "- When you see 'question-response-received' events, use get_tester_response to retrieve the full answer with images.\n" +
+    "- When you see 'question-response-transcribed' events, the tester's voice response has been transcribed — read it for context.\n" +
+    "- Marker events with hasPendingVoice=true will have their voiceTranscription field populated once transcription completes.\n" +
     "- The tester may set voice markers at any time to communicate with you — keep watching for them.\n" +
+    "- If you need clarification, use ask_tester_question to show a toast notification to the tester.\n" +
     "- Only stop polling when sessionActive is false (the recording ended).",
   {
     since_seq: z
@@ -1209,12 +1268,37 @@ server.tool(
 
     const latestSeq = liveEvents.length > 0 ? liveEvents[liveEvents.length - 1].seq : 0;
 
+    // Enrich marker events with their transcriptions (if available in the buffer)
+    // so agents don't have to correlate separate marker + voice-transcription events
+    const transcriptionsByMarkerId = new Map();
+    for (const e of liveEvents) {
+      if (e.eventType === "voice-transcription" && e.data?.markerId) {
+        transcriptionsByMarkerId.set(e.data.markerId, e.data.transcription);
+      }
+    }
+    const enrichedEvents = events.map((e) => {
+      if (
+        e.eventType === "marker" &&
+        e.data?.markerId &&
+        transcriptionsByMarkerId.has(e.data.markerId)
+      ) {
+        return {
+          ...e,
+          data: {
+            ...e.data,
+            voiceTranscription: transcriptionsByMarkerId.get(e.data.markerId),
+          },
+        };
+      }
+      return e;
+    });
+
     const responseData = {
       sessionActive: liveSessionActive,
       session: liveSessionMeta,
-      events,
+      events: enrichedEvents,
       latestSeq,
-      eventCount: events.length,
+      eventCount: enrichedEvents.length,
     };
 
     if (liveSessionActive) {
@@ -1230,8 +1314,7 @@ server.tool(
         },
         {
           tool: "get_live_screenshot",
-          description:
-            "Capture a screenshot of the tester's current browser tab.",
+          description: "Capture a screenshot of the tester's current browser tab.",
         },
       ];
       if (pendingQuestions.length > 0) {
@@ -1241,12 +1324,26 @@ server.tool(
           waitingSince: Math.round((Date.now() - q.sentAt) / 1000) + "s ago",
         }));
       }
-    } else if (liveInteractions.length > 0) {
-      // Session ended — include full interaction log for the agent to incorporate
-      responseData.sessionEndedNote =
-        "Session has ended. The interactionLog below contains all agent-tester exchanges from this session. " +
-        "Include these in your analysis — they represent collaborative context between agent and tester.";
-      responseData.interactionLog = liveInteractions;
+    } else {
+      // Session ended — include full interaction log and voice transcriptions
+      if (
+        liveInteractions.length > 0 ||
+        liveEvents.some((e) => e.eventType === "voice-transcription")
+      ) {
+        responseData.sessionEndedNote =
+          "Session has ended. The interactionLog and voiceMarkerTranscriptions below contain all agent-tester exchanges from this session. " +
+          "Include these in your analysis — they represent collaborative context between agent and tester. " +
+          "You can also call get_live_session_summary for the complete summary.";
+        responseData.interactionLog = liveInteractions;
+        // Include all voice marker transcriptions
+        responseData.voiceMarkerTranscriptions = liveEvents
+          .filter((e) => e.eventType === "voice-transcription" && e.data?.transcription)
+          .map((e) => ({
+            timestamp: e.ts,
+            markerId: e.data.markerId,
+            transcription: e.data.transcription,
+          }));
+      }
     }
 
     // Always include interaction count so the agent knows exchanges happened
@@ -1361,8 +1458,7 @@ server.tool(
           {
             type: "text",
             text: [
-              "No response yet." +
-                (waitingSec !== null ? ` Waiting for ${waitingSec}s.` : ""),
+              "No response yet." + (waitingSec !== null ? ` Waiting for ${waitingSec}s.` : ""),
               "The tester may still be recording their voice answer.",
               "",
               "TIP: Instead of polling this tool, go back to polling watch_live_session — " +
@@ -1428,33 +1524,30 @@ server.tool(
       }
     }
 
-    let transcriptionText = null;
     if (response.voiceBlobDataUrl) {
-      const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
-      if (voiceResult.transcription) {
-        transcriptionText = voiceResult.transcription;
+      // Use pre-transcribed result if available (from eager transcription on response arrival)
+      if (response.transcription) {
         content.push({
           type: "text",
-          text: `Voice transcription:\n${voiceResult.transcription}`,
+          text: `Voice transcription:\n${response.transcription}`,
         });
       } else {
-        content.push({
-          type: "text",
-          text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
-        });
+        // Fallback: transcribe now if eager transcription hasn't completed yet
+        const voiceResult = await transcribeLiveVoice(response.voiceBlobDataUrl);
+        if (voiceResult.transcription) {
+          content.push({
+            type: "text",
+            text: `Voice transcription:\n${voiceResult.transcription}`,
+          });
+        } else {
+          content.push({
+            type: "text",
+            text: `Voice note is attached but could not be transcribed${voiceResult.error ? ` (${voiceResult.error})` : ""}. Set OPENROUTER_API_KEY or install local Whisper for transcription.`,
+          });
+        }
       }
     }
 
-    // Log the interaction for the session history
-    liveInteractions.push({
-      type: "tester-response",
-      questionId: question_id,
-      timestamp: Date.now(),
-      transcription: transcriptionText,
-      screenshot: !!response.screenshot,
-      highlightCount: response.highlightCaptures?.length || 0,
-    });
-
     content.push({
       type: "text",
       text: "\nIMPORTANT: Resume polling watch_live_session to continue watching the session.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tracegist-mcp-bridge",
-  "version": "0.2.7",
+  "version": "0.2.11",
   "description": "Local-first MCP bridge for reading and transcribing TraceGist package zips.",
   "type": "module",
   "bin": {