testdriverai 7.11.283-test → 7.11.285-test

package/mcp-server/dist/server.mjs

@@ -255,6 +255,51 @@ function storeImage(data, type) {
 function getStoredImage(id) {
     return imageStoreState().images.get(id);
 }
+/**
+ * `_meta` key carrying the image inline in a tool result, for the MCP App.
+ *
+ * The app used to get every image with a follow-up `resources/read` against the
+ * per-connection store above. That second round trip is fragile on hosted
+ * clients: when it fails the app has nothing to show and the preview collapses.
+ * Sending the bytes with the result removes the round trip. `_meta` is passed
+ * through to the app but, unlike `content`/`structuredContent`, is not shown to
+ * the model, so the base64 stays out of the agent's context. The resource URI
+ * is still set as a fallback (and for `check`'s `referenceImageUri`).
+ */
+const INLINE_IMAGE_META_KEY = "testdriver/image";
+/** Above this (base64 chars) the image is left to the resource fallback, so one
+ *  huge frame can't push a tool result past a host's size limit. */
+const MAX_INLINE_IMAGE_BASE64 = 2_000_000;
+function inlineImageMeta(base64) {
+    if (!base64)
+        return undefined;
+    if (base64.length > MAX_INLINE_IMAGE_BASE64) {
+        logger.debug("inlineImageMeta: Image too large to inline, relying on resource URI", { length: base64.length });
+        return undefined;
+    }
+    return { [INLINE_IMAGE_META_KEY]: { mimeType: "image/png", data: base64 } };
+}
+/**
+ * Store a result's images and record their resource URIs on `data`. Returns the
+ * base64 of the image the app displays (full screenshot wins over the crop,
+ * matching the app's own preference) so the caller can inline it.
+ */
+function attachImages(data, images) {
+    let screenshot;
+    let cropped;
+    for (const img of images ?? []) {
+        const uri = storeImage(img.base64, img.kind);
+        if (img.kind === "cropped") {
+            data.croppedImageResourceUri = uri;
+            cropped = img.base64;
+        }
+        else {
+            data.screenshotResourceUri = uri;
+            screenshot = img.base64;
+        }
+    }
+    return screenshot ?? cropped;
+}
 /** Diagnostics for the active connection's image store (used in debug logs). */
 function imageStoreDiagnostics() {
     const images = imageStoreState().images;
@@ -297,14 +342,8 @@ function noSessionResult(err) {
  */
 function resultToMcp(r) {
     const data = { ...r.data };
-    for (const img of r.images ?? []) {
-        const uri = storeImage(img.base64, img.kind);
-        if (img.kind === "cropped")
-            data.croppedImageResourceUri = uri;
-        else
-            data.screenshotResourceUri = uri;
-    }
-    return createToolResult(r.ok, r.text, data, r.code);
+    const image = attachImages(data, r.images);
+    return createToolResult(r.ok, r.text, data, r.code, image);
 }
 const DEFAULT_HEARTBEAT_MS = 3000;
 function makeProgressReporter(extra) {
@@ -410,7 +449,7 @@ function cancelledResultOrNull(error, tool) {
  * If generatedCode is provided, it's appended to the text response with instructions
  * for the agent to write it to the test file.
  */
-function createToolResult(success, textContent, structuredData, generatedCode) {
+function createToolResult(success, textContent, structuredData, generatedCode, inlineImageBase64) {
     // Build text content - append generated code if provided with directive instructions
     let fullText = textContent;
     if (generatedCode && success) {
@@ -444,6 +483,7 @@ function createToolResult(success, textContent, structuredData, generatedCode) {
             generatedCode: generatedCode && success ? generatedCode : undefined,
             testFile: session?.testFile || undefined,
         },
+        _meta: inlineImageMeta(inlineImageBase64),
     };
 }
 /**
@@ -631,13 +671,7 @@ Debug mode (connect to existing sandbox):
             // Reproduce the server's exact success output (text + data) which differs
             // from the core's neutral text. Images are mapped to resource URIs.
             const data = { ...result.data, duration };
-            for (const img of result.images ?? []) {
-                const uri = storeImage(img.base64, img.kind);
-                if (img.kind === "cropped")
-                    data.croppedImageResourceUri = uri;
-                else
-                    data.screenshotResourceUri = uri;
-            }
+            const image = attachImages(data, result.images);
             if (result.data.debugMode) {
                 // Debug (existing-sandbox) success text — preserve original wording.
                 const text = `Connected to existing sandbox (debug mode)
@@ -646,7 +680,7 @@ Sandbox: ${result.data.sandboxId}
 Expires in: ${Math.round(params.keepAlive / 1000)}s
 
 You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`;
-                return createToolResult(true, text, data, result.code);
+                return createToolResult(true, text, data, result.code, image);
             }
             // Normal provisioning success — append the EXACT dependency guidance block.
             const text = `${result.text}
@@ -662,7 +696,7 @@ IMPORTANT - If creating a new test project, use these EXACT dependencies in pack
     "test": "vitest"
   }
 }`;
-            return createToolResult(true, text, data, result.code);
+            return createToolResult(true, text, data, result.code, image);
         }
         catch (error) {
             // On client cancellation, tear down the half-provisioned session so we
@@ -1502,6 +1536,7 @@ Useful for:
                     screenshotResourceUri,
                     duration
                 },
+                _meta: inlineImageMeta(imageBase64),
             };
         }
         catch (error) {
@@ -1533,17 +1568,19 @@ Only use 'screenshot' when you explicitly want to show something to the human us
         logger.info("screenshot: Starting");
         try {
             const result = await core.screenshot();
-            // Store the captured image as a resource URI (kept OUT of AI context — the
-            // MCP app fetches it via resources/read). Preserve the original user-facing
-            // text rather than the core's neutral text.
+            // The image goes to the MCP app only (inline in `_meta`, with a resource
+            // URI as fallback) and stays OUT of AI context. Preserve the original
+            // user-facing text rather than the core's neutral text.
             const data = { action: "screenshot" };
+            let image;
             for (const img of result.images ?? []) {
                 data.screenshotResourceUri = storeImage(img.base64, "screenshot");
+                image = img.base64;
             }
             const duration = Date.now() - startTime;
             data.duration = duration;
             logger.info("screenshot: Completed", { duration, hasImage: (result.images?.length ?? 0) > 0 });
-            return createToolResult(true, "Screenshot captured and displayed to user", data);
+            return createToolResult(true, "Screenshot captured and displayed to user", data, undefined, image);
         }
         catch (error) {
             if (error instanceof NoActiveSessionError) {

package/mcp-server/src/mcp-app.ts

@@ -66,6 +66,20 @@ interface ToolResultData {
   debuggerUrl?: string;
   sessionId?: string;
   duration?: number;
+  imageError?: string;  // Why the image could not be shown (set client-side)
+}
+
+// `_meta` key the server uses to send the image inline with a tool result.
+const INLINE_IMAGE_META_KEY = "testdriver/image";
+
+/**
+ * Read the image the server inlined in the tool result's `_meta`, as a data URL.
+ * Null when the result carries none (older server, or an image too large to inline).
+ */
+function inlineImageUrl(result: CallToolResult): string | null {
+  const image = result._meta?.[INLINE_IMAGE_META_KEY] as { mimeType?: string; data?: string } | undefined;
+  if (!image || typeof image.data !== "string" || !image.data) return null;
+  return `data:${image.mimeType || "image/png"};base64,${image.data}`;
 }
 
 // Store session info globally for display
@@ -345,6 +359,9 @@ function renderResult(data: ToolResultData) {
   if (data.error) {
     statusText += ` - ${data.error}`;
   }
+  if (data.imageError) {
+    statusText += ` - ${data.imageError}`;
+  }
   
   actionStatusEl.textContent = statusText;
   actionStatusEl.className = statusClass;
@@ -450,28 +467,22 @@ const app = new App({ name: "TestDriver Screenshot", version: "1.0.0" });
 /**
  * Fetch screenshot blob from resource URI and convert to data URL
  */
-async function fetchScreenshotFromResource(resourceUri: string): Promise<string | null> {
-  try {
-    console.info("Fetching screenshot from resource:", resourceUri);
-    const result = await app.request(
-      { method: "resources/read", params: { uri: resourceUri } },
-      ReadResourceResultSchema,
-    );
-
-    const content = result.contents[0];
-    if (!content || !("blob" in content)) {
-      console.error("Resource did not contain blob data");
-      return null;
-    }
-
-    // Convert base64 blob to data URL
-    const dataUrl = `data:${content.mimeType || "image/png"};base64,${content.blob}`;
-    console.info("Screenshot fetched successfully, blob length:", content.blob.length);
-    return dataUrl;
-  } catch (error) {
-    console.error("Failed to fetch screenshot resource:", error);
-    return null;
+async function fetchScreenshotFromResource(resourceUri: string): Promise<string> {
+  console.info("Fetching screenshot from resource:", resourceUri);
+  const result = await app.request(
+    { method: "resources/read", params: { uri: resourceUri } },
+    ReadResourceResultSchema,
+  );
+
+  const content = result.contents[0];
+  if (!content || !("blob" in content)) {
+    throw new Error("Resource did not contain image data");
   }
+
+  // Convert base64 blob to data URL
+  const dataUrl = `data:${content.mimeType || "image/png"};base64,${content.blob}`;
+  console.info("Screenshot fetched successfully, blob length:", content.blob.length);
+  return dataUrl;
 }
 
 // 2. Register handlers BEFORE connecting
@@ -517,13 +528,22 @@ app.ontoolresult = async (result) => {
   console.info("Has screenshotResourceUri:", !!data.screenshotResourceUri);
   console.info("Has croppedImageResourceUri:", !!data.croppedImageResourceUri);
   
-  // If a screenshot or cropped image resource URI is provided, fetch the image from it
+  // Prefer the image the server sent inline with the result: no extra request.
+  if (!data.imageUrl) {
+    data.imageUrl = inlineImageUrl(result) ?? undefined;
+  }
+
+  // Otherwise fall back to fetching it by resource URI. If that fails, say so in
+  // the status bar instead of silently rendering an empty frame.
   const resourceUri = data.screenshotResourceUri || data.croppedImageResourceUri;
   if (resourceUri && !data.imageUrl) {
     showLoading("Fetching image...");
-    const imageUrl = await fetchScreenshotFromResource(resourceUri);
-    if (imageUrl) {
-      data.imageUrl = imageUrl;
+    try {
+      data.imageUrl = await fetchScreenshotFromResource(resourceUri);
+    } catch (error) {
+      console.error("Failed to fetch screenshot resource:", error);
+      const reason = error instanceof Error ? error.message : String(error);
+      data.imageError = `Screenshot unavailable: ${reason}`;
     }
   }
   

package/mcp-server/src/server.ts

@@ -326,6 +326,53 @@ function getStoredImage(id: string): StoredImage | undefined {
   return imageStoreState().images.get(id);
 }
 
+/**
+ * `_meta` key carrying the image inline in a tool result, for the MCP App.
+ *
+ * The app used to get every image with a follow-up `resources/read` against the
+ * per-connection store above. That second round trip is fragile on hosted
+ * clients: when it fails the app has nothing to show and the preview collapses.
+ * Sending the bytes with the result removes the round trip. `_meta` is passed
+ * through to the app but, unlike `content`/`structuredContent`, is not shown to
+ * the model, so the base64 stays out of the agent's context. The resource URI
+ * is still set as a fallback (and for `check`'s `referenceImageUri`).
+ */
+const INLINE_IMAGE_META_KEY = "testdriver/image";
+
+/** Above this (base64 chars) the image is left to the resource fallback, so one
+ *  huge frame can't push a tool result past a host's size limit. */
+const MAX_INLINE_IMAGE_BASE64 = 2_000_000;
+
+function inlineImageMeta(base64: string | undefined): CallToolResult["_meta"] | undefined {
+  if (!base64) return undefined;
+  if (base64.length > MAX_INLINE_IMAGE_BASE64) {
+    logger.debug("inlineImageMeta: Image too large to inline, relying on resource URI", { length: base64.length });
+    return undefined;
+  }
+  return { [INLINE_IMAGE_META_KEY]: { mimeType: "image/png", data: base64 } };
+}
+
+/**
+ * Store a result's images and record their resource URIs on `data`. Returns the
+ * base64 of the image the app displays (full screenshot wins over the crop,
+ * matching the app's own preference) so the caller can inline it.
+ */
+function attachImages(data: Record<string, unknown>, images: ActionResult["images"]): string | undefined {
+  let screenshot: string | undefined;
+  let cropped: string | undefined;
+  for (const img of images ?? []) {
+    const uri = storeImage(img.base64, img.kind);
+    if (img.kind === "cropped") {
+      data.croppedImageResourceUri = uri;
+      cropped = img.base64;
+    } else {
+      data.screenshotResourceUri = uri;
+      screenshot = img.base64;
+    }
+  }
+  return screenshot ?? cropped;
+}
+
 /** Diagnostics for the active connection's image store (used in debug logs). */
 function imageStoreDiagnostics(): { imageStoreSize: number; availableKeys: string[] } {
   const images = imageStoreState().images;
@@ -378,12 +425,8 @@ function noSessionResult(err: NoActiveSessionError): CallToolResult {
  */
 function resultToMcp(r: ActionResult): CallToolResult {
   const data: Record<string, unknown> = { ...r.data };
-  for (const img of r.images ?? []) {
-    const uri = storeImage(img.base64, img.kind);
-    if (img.kind === "cropped") data.croppedImageResourceUri = uri;
-    else data.screenshotResourceUri = uri;
-  }
-  return createToolResult(r.ok, r.text, data, r.code);
+  const image = attachImages(data, r.images);
+  return createToolResult(r.ok, r.text, data, r.code, image);
 }
 
 // =============================================================================
@@ -534,7 +577,8 @@ function createToolResult(
   success: boolean,
   textContent: string,
   structuredData: Record<string, unknown>,
-  generatedCode?: string
+  generatedCode?: string,
+  inlineImageBase64?: string
 ): CallToolResult {
   // Build text content - append generated code if provided with directive instructions
   let fullText = textContent;
@@ -572,6 +616,7 @@ function createToolResult(
       generatedCode: generatedCode && success ? generatedCode : undefined,
       testFile: session?.testFile || undefined,
     },
+    _meta: inlineImageMeta(inlineImageBase64),
   };
 }
 
@@ -814,11 +859,7 @@ Debug mode (connect to existing sandbox):
       // Reproduce the server's exact success output (text + data) which differs
       // from the core's neutral text. Images are mapped to resource URIs.
       const data: Record<string, unknown> = { ...result.data, duration };
-      for (const img of result.images ?? []) {
-        const uri = storeImage(img.base64, img.kind);
-        if (img.kind === "cropped") data.croppedImageResourceUri = uri;
-        else data.screenshotResourceUri = uri;
-      }
+      const image = attachImages(data, result.images);
 
       if (result.data.debugMode) {
         // Debug (existing-sandbox) success text — preserve original wording.
@@ -828,7 +869,7 @@ Sandbox: ${result.data.sandboxId}
 Expires in: ${Math.round(params.keepAlive / 1000)}s
 
 You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`;
-        return createToolResult(true, text, data, result.code);
+        return createToolResult(true, text, data, result.code, image);
       }
 
       // Normal provisioning success — append the EXACT dependency guidance block.
@@ -845,7 +886,7 @@ IMPORTANT - If creating a new test project, use these EXACT dependencies in pack
     "test": "vitest"
   }
 }`;
-      return createToolResult(true, text, data, result.code);
+      return createToolResult(true, text, data, result.code, image);
     } catch (error) {
       // On client cancellation, tear down the half-provisioned session so we
       // don't leak a connected sandbox. The underlying SDK call may still be
@@ -1829,6 +1870,7 @@ Useful for:
           screenshotResourceUri,
           duration 
         },
+        _meta: inlineImageMeta(imageBase64),
       };
     } catch (error) {
       logger.error("view_local_screenshot: Failed", { error: String(error), path: params.path });
@@ -1867,19 +1909,21 @@ Only use 'screenshot' when you explicitly want to show something to the human us
     try {
       const result = await core.screenshot();
 
-      // Store the captured image as a resource URI (kept OUT of AI context — the
-      // MCP app fetches it via resources/read). Preserve the original user-facing
-      // text rather than the core's neutral text.
+      // The image goes to the MCP app only (inline in `_meta`, with a resource
+      // URI as fallback) and stays OUT of AI context. Preserve the original
+      // user-facing text rather than the core's neutral text.
       const data: Record<string, unknown> = { action: "screenshot" };
+      let image: string | undefined;
       for (const img of result.images ?? []) {
         data.screenshotResourceUri = storeImage(img.base64, "screenshot");
+        image = img.base64;
       }
 
       const duration = Date.now() - startTime;
       data.duration = duration;
       logger.info("screenshot: Completed", { duration, hasImage: (result.images?.length ?? 0) > 0 });
 
-      return createToolResult(true, "Screenshot captured and displayed to user", data);
+      return createToolResult(true, "Screenshot captured and displayed to user", data, undefined, image);
     } catch (error) {
       if (error instanceof NoActiveSessionError) {
         logger.warn("screenshot: No active session");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.11.283-test",
+  "version": "7.11.285-test",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",