testdriverai 6.2.2 → 7.1.0

package/agent/lib/redraw.js

@@ -3,9 +3,28 @@ const fs = require("fs");
 const { events } = require("../events");
 const theme = require("./theme");
 
+// Default redraw options
+const DEFAULT_REDRAW_OPTIONS = {
+  enabled: true,           // Master switch to enable/disable redraw detection
+  screenRedraw: true,      // Enable screen redraw detection
+  networkMonitor: true,    // Enable network activity monitoring
+  diffThreshold: 0.1,      // Percentage threshold for screen diff (0.1 = 0.1%)
+};
+
 // Factory function that creates redraw functionality with the provided system instance
-const createRedraw = (emitter, system, sandbox) => {
-  const redrawThresholdPercent = 0.1;
+const createRedraw = (
+  emitter,
+  system,
+  sandbox,
+  defaultOptions = {},
+) => {
+  // Merge default options with provided defaults
+  const baseOptions = { ...DEFAULT_REDRAW_OPTIONS, ...defaultOptions };
+  // Support legacy redrawThresholdPercent number argument
+  if (typeof defaultOptions === 'number') {
+    baseOptions.diffThreshold = defaultOptions;
+  }
+  
   const networkUpdateInterval = 15000;
 
   let lastTxBytes = null;
@@ -69,7 +88,6 @@ const createRedraw = (emitter, system, sandbox) => {
   async function updateNetwork() {
     if (sandbox && sandbox.instanceSocketConnected) {
       let network = await sandbox.send({
-        os: "linux",
         type: "system.network",
       });
       parseNetworkStats(
@@ -113,16 +131,13 @@ const createRedraw = (emitter, system, sandbox) => {
         { threshold: 0.1 },
       );
 
-      if (differentPixels === 0) {
-        return false;
-      } else {
-        // Calculate percentage difference based on pixel differences
-        const diffPercentage = (differentPixels / totalPixels) * 100;
-        return diffPercentage.toFixed(1);
-      }
+      // Calculate percentage difference based on pixel differences
+      // Always return a number (0 if no difference)
+      const diffPercentage = (differentPixels / totalPixels) * 100;
+      return parseFloat(diffPercentage.toFixed(2));
     } catch (error) {
       console.error("Error comparing images:", error);
-      return false;
+      return 0; // Return 0 on error instead of false
     }
   }
 
@@ -143,46 +158,104 @@ const createRedraw = (emitter, system, sandbox) => {
     }
   }
 
-  async function start() {
+  // Current options for the active redraw cycle
+  let currentOptions = { ...baseOptions };
+
+  async function start(options = {}) {
+    // Merge base options with per-call options
+    currentOptions = { ...baseOptions, ...options };
+    
+    console.log('[redraw] start() called with options:', JSON.stringify(currentOptions));
+    
+    // If redraw is completely disabled, return early
+    if (!currentOptions.enabled) {
+      console.log('[redraw] start() - redraw disabled, returning null');
+      return null;
+    }
+    
+    // If both screenRedraw and networkMonitor are disabled, disable redraw
+    if (!currentOptions.screenRedraw && !currentOptions.networkMonitor) {
+      currentOptions.enabled = false;
+      console.log('[redraw] start() - both screenRedraw and networkMonitor disabled, returning null');
+      return null;
+    }
+    
     resetState();
-    startNetworkMonitoring();
-    startImage = await system.captureScreenPNG(0.25, true);
+    
+    // Only start network monitoring if enabled
+    if (currentOptions.networkMonitor) {
+      startNetworkMonitoring();
+    }
+    
+    // Only capture start image if screen redraw is enabled
+    if (currentOptions.screenRedraw) {
+      startImage = await system.captureScreenPNG(0.25, true);
+      console.log('[redraw] start() - captured startImage:', startImage);
+    }
+    
     return startImage;
   }
 
-  async function checkCondition(resolve, startTime, timeoutMs) {
-    let nowImage = await system.captureScreenPNG(0.25, true);
+  async function checkCondition(resolve, startTime, timeoutMs, options) {
+    const { enabled, screenRedraw, networkMonitor, diffThreshold } = options;
+    
+    // If redraw is disabled, resolve immediately
+    if (!enabled) {
+      resolve("true");
+      return;
+    }
+    
+    let nowImage = screenRedraw ? await system.captureScreenPNG(0.25, true) : null;
     let timeElapsed = Date.now() - startTime;
     let diffPercent = 0;
     let isTimeout = timeElapsed > timeoutMs;
 
-    if (!screenHasRedrawn) {
+    // Check screen redraw if enabled and we have a start image to compare against
+    if (screenRedraw && !screenHasRedrawn && startImage && nowImage) {
+      console.log('[redraw] checkCondition() - comparing images:', { startImage, nowImage });
       diffPercent = await imageDiffPercent(startImage, nowImage);
-      screenHasRedrawn = diffPercent > redrawThresholdPercent;
+      console.log('[redraw] checkCondition() - diffPercent:', diffPercent, 'threshold:', diffThreshold);
+      screenHasRedrawn = diffPercent > diffThreshold;
+      console.log('[redraw] checkCondition() - screenHasRedrawn:', screenHasRedrawn);
+    } else if (screenRedraw && !startImage) {
+      // If no start image was captured, capture one now and wait for next check
+      console.log('[redraw] checkCondition() - no startImage, capturing now');
+      startImage = await system.captureScreenPNG(0.25, true);
     }
-
-    // // log redraw as output
-    let redrawText = screenHasRedrawn
-      ? theme.green(`y`)
-      : theme.dim(`${diffPercent}/${redrawThresholdPercent}%`);
-    let networkText = networkSettled
-      ? theme.green(`y`)
-      : theme.dim(
-          `${Math.trunc((diffRxBytes + diffTxBytes) / networkUpdateInterval)}b/s`,
-        );
+    
+    // If screen redraw is disabled, consider it as "redrawn"
+    const effectiveScreenRedrawn = screenRedraw ? screenHasRedrawn : true;
+    // If network monitor is disabled, consider it as "settled"
+    const effectiveNetworkSettled = networkMonitor ? networkSettled : true;
+
+    // Log redraw status
+    let redrawText = !screenRedraw
+      ? theme.dim(`disabled`)
+      : effectiveScreenRedrawn
+        ? theme.green(`y`)
+        : theme.dim(`${diffPercent}/${diffThreshold}%`);
+    let networkText = !networkMonitor
+      ? theme.dim(`disabled`)
+      : effectiveNetworkSettled
+        ? theme.green(`y`)
+        : theme.dim(
+            `${Math.trunc((diffRxBytes + diffTxBytes) / networkUpdateInterval)}b/s`,
+          );
     let timeoutText = isTimeout
       ? theme.green(`y`)
       : theme.dim(`${Math.floor(timeElapsed / 1000)}/${timeoutMs / 1000}s`);
 
     emitter.emit(events.redraw.status, {
       redraw: {
-        hasRedrawn: screenHasRedrawn,
+        enabled: screenRedraw,
+        hasRedrawn: effectiveScreenRedrawn,
         diffPercent,
-        threshold: redrawThresholdPercent,
+        threshold: diffThreshold,
         text: redrawText,
       },
       network: {
-        settled: networkSettled,
+        enabled: networkMonitor,
+        settled: effectiveNetworkSettled,
         rxBytes: diffRxBytes,
         txBytes: diffTxBytes,
         text: networkText,
@@ -195,27 +268,42 @@ const createRedraw = (emitter, system, sandbox) => {
       },
     });
 
-    if ((screenHasRedrawn && networkSettled) || isTimeout) {
+    if ((effectiveScreenRedrawn && effectiveNetworkSettled) || isTimeout) {
       emitter.emit(events.redraw.complete, {
-        screenHasRedrawn,
-        networkSettled,
+        screenHasRedrawn: effectiveScreenRedrawn,
+        networkSettled: effectiveNetworkSettled,
         isTimeout,
         timeElapsed,
       });
       resolve("true");
     } else {
       setTimeout(() => {
-        checkCondition(resolve, startTime, timeoutMs);
+        checkCondition(resolve, startTime, timeoutMs, options);
       }, 500);
     }
   }
 
-  function wait(timeoutMs) {
+  function wait(timeoutMs, options = {}) {
+    // Merge current options with any per-call overrides
+    const waitOptions = { ...currentOptions, ...options };
+    
+    // If redraw is disabled, resolve immediately
+    if (!waitOptions.enabled) {
+      return Promise.resolve("true");
+    }
+    
+    // If both are disabled, resolve immediately
+    if (!waitOptions.screenRedraw && !waitOptions.networkMonitor) {
+      return Promise.resolve("true");
+    }
+    
     return new Promise((resolve) => {
       const startTime = Date.now();
-      // Start network monitoring if not already started
-      startNetworkMonitoring();
-      checkCondition(resolve, startTime, timeoutMs);
+      // Start network monitoring if not already started and enabled
+      if (waitOptions.networkMonitor) {
+        startNetworkMonitoring();
+      }
+      checkCondition(resolve, startTime, timeoutMs, waitOptions);
     });
   }
 
@@ -223,7 +311,7 @@ const createRedraw = (emitter, system, sandbox) => {
     stopNetworkMonitoring(networkInterval);
   }
 
-  return { start, wait, cleanup };
+  return { start, wait, cleanup, DEFAULT_OPTIONS: DEFAULT_REDRAW_OPTIONS };
 };
 
-module.exports = { createRedraw };
+module.exports = { createRedraw, DEFAULT_REDRAW_OPTIONS };

package/agent/lib/sandbox.js

@@ -2,7 +2,7 @@ const WebSocket = require("ws");
 const marky = require("marky");
 const { events } = require("../events");
 
-const createSandbox = (emitter, analytics) => {
+const createSandbox = (emitter, analytics, sessionInstance) => {
   class Sandbox {
     constructor() {
       this.socket = null;
@@ -14,6 +14,8 @@ const createSandbox = (emitter, analytics) => {
       this.instance = null;
       this.messageId = 0;
       this.uniqueId = Math.random().toString(36).substring(7);
+      this.os = null; // Store OS value to send with every message
+      this.sessionInstance = sessionInstance; // Store session instance to include in messages
     }
 
     send(message) {
@@ -24,6 +26,24 @@ const createSandbox = (emitter, analytics) => {
         this.messageId++;
         message.requestId = `${this.uniqueId}-${this.messageId}`;
 
+        // If os is set in the message, store it for future messages
+        if (message.os) {
+          this.os = message.os;
+        }
+
+        // Add os to every message if it's been set
+        if (this.os && !message.os) {
+          message.os = this.os;
+        }
+
+        // Add session to every message if available (for interaction tracking)
+        if (this.sessionInstance && !message.session) {
+          const sessionId = this.sessionInstance.get();
+          if (sessionId) {
+            message.session = sessionId;
+          }
+        }
+
         // Start timing for this message
         const timingKey = `sandbox-${message.type}`;
         marky.mark(timingKey);
@@ -51,7 +71,6 @@ const createSandbox = (emitter, analytics) => {
     async auth(apiKey) {
       let reply = await this.send({
         type: "authenticate",
-        os: "linux",
         apiKey,
       });
 
@@ -65,7 +84,6 @@ const createSandbox = (emitter, analytics) => {
     async connect(sandboxId, persist = false) {
       let reply = await this.send({
         type: "connect",
-        os: "linux",
         persist,
         sandboxId,
       });
@@ -73,9 +91,12 @@ const createSandbox = (emitter, analytics) => {
       if (reply.success) {
         this.instanceSocketConnected = true;
         emitter.emit(events.sandbox.connected);
+        // Return the full reply (includes url and sandbox)
+        return reply;
+      } else {
+        // Throw error to trigger fallback to creating new sandbox
+        throw new Error(reply.errorMessage || "Failed to connect to sandbox");
       }
-
-      return reply.sandbox;
     }
 
     async boot(apiRoot) {
@@ -124,7 +145,9 @@ const createSandbox = (emitter, analytics) => {
 
           if (message.error) {
             emitter.emit(events.error.sandbox, message.errorMessage);
-            this.ps[message.requestId].reject(JSON.stringify(message));
+            const error = new Error(message.errorMessage || "Sandbox error");
+            error.responseData = message;
+            this.ps[message.requestId].reject(error);
           } else {
             emitter.emit(events.sandbox.received);
 

package/agent/lib/sdk.js

@@ -123,6 +123,7 @@ const createSDK = (emitter, config, sessionInstance) => {
         ...(token && { Authorization: `Bearer ${token}` }), // Add the authorization bearer token only if token is set
       },
       responseType: typeof onChunk === "function" ? "stream" : "json",
+      timeout: 60000, // 60 second timeout to prevent hanging requests
       data: {
         ...data,
         session: sessionInstance.get(),
@@ -193,6 +194,27 @@ const createSDK = (emitter, config, sessionInstance) => {
 
       return value;
     } catch (error) {
+      // Check if this is an API validation error with detailed problems
+      if (error.response?.data?.problems) {
+        const problems = error.response.data.problems;
+        const errorMessage = error.response.data.message || 'API validation error';
+        const detailedError = new Error(
+          `${errorMessage}\n\nDetails:\n${problems.map(p => `  - ${p}`).join('\n')}`
+        );
+        detailedError.originalError = error;
+        detailedError.problems = problems;
+        
+        // Emit the formatted error
+        emitter.emit(events.error.sdk, {
+          message: detailedError.message,
+          code: error.response?.data?.code || error.code,
+          problems: problems,
+          fullError: error,
+        });
+        
+        throw detailedError;
+      }
+      
       outputError(error);
       throw error; // Re-throw the error so calling code can handle it properly
     }

package/agent/lib/system.js

@@ -8,7 +8,6 @@ const { events } = require("../events.js");
 const createSystem = (emitter, sandbox, config) => {
   const screenshot = async (options) => {
     let { base64 } = await sandbox.send({
-      os: "linux",
       type: "system.screenshot",
     });
 
@@ -113,7 +112,6 @@ const createSystem = (emitter, sandbox, config) => {
   const activeWin = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
-      os: "linux",
       type: "system.get-active-window",
     });
 
@@ -123,7 +121,6 @@ const createSystem = (emitter, sandbox, config) => {
   const getMousePosition = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
-      os: "linux",
       type: "system.get-mouse-position",
     });
 

package/agent/lib/validation.js

@@ -62,33 +62,27 @@ const types = () =>
       },
       // - command: scroll # Scroll up or down. Make sure the correct portion of the page is focused before scrolling.
       //   direction: down # Available directions are: up, down, left, right
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
-      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300 for keyboard and 200 for mouse.
+      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300.
       ScrollCommand: {
         command: '"scroll"',
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
         "amount?": "number",
       },
       // - command: scroll-until-text # Scroll until text is found
       //   text: Sign Up # The text to find on screen. The longer and more unique the better.
       //   direction: down # Available directions are: up, down, left, right
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
       ScrollUntilTextCommand: {
         command: '"scroll-until-text"',
         text: "string",
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
       },
       // - command: scroll-until-image # Scroll until icon or image is found
       //   description: Submit at the bottom of the form
       //   direction: down # Available directions are: up, down, left, rights
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
       ScrollUntilImageCommand: {
         command: '"scroll-until-image"',
         description: "string",
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
       },
       // - command: wait-for-text # Wait until text is seen on screen. Not recommended unless explicitly requested by user.
       //   text: Copyright 2024 # The text to find on screen.

package/debug-locate-response.js

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+
+/**
+ * Debug script to inspect the full locate API response
+ * Run this with: TD_API_KEY=your_key node debug-locate-response.js
+ */
+
+const TestDriverSDK = require("./sdk.js");
+
+async function debugLocateResponse() {
+  const client = new TestDriverSDK(process.env.TD_API_KEY);
+
+  try {
+    console.log("Connecting to sandbox (Linux)...");
+    await client.connect({ headless: true });
+
+    console.log("Opening a test page...");
+    await client.focusApplication("Google Chrome");
+    await client.type("https://example.com");
+    await client.pressKeys(["enter"]);
+
+    // Wait for page to load
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+
+    console.log("\nFinding an element to inspect the response...");
+    const element = await client.find("the heading that says Example Domain");
+
+    console.log("\n=".repeat(60));
+    console.log("FULL LOCATE API RESPONSE:");
+    console.log("=".repeat(60));
+
+    const response = element.getResponse();
+    console.log(JSON.stringify(response, null, 2));
+
+    console.log("\n=".repeat(60));
+    console.log("RESPONSE KEYS:");
+    console.log("=".repeat(60));
+
+    if (response) {
+      Object.keys(response).forEach((key) => {
+        const value = response[key];
+        const type = Array.isArray(value) ? "array" : typeof value;
+        const preview =
+          typeof value === "string" && value.length > 100
+            ? `${value.substring(0, 100)}... (${value.length} chars)`
+            : typeof value === "object"
+              ? JSON.stringify(value)
+              : value;
+
+        console.log(`  ${key} (${type}): ${preview}`);
+      });
+    }
+
+    console.log("\n=".repeat(60));
+    console.log("ELEMENT PROPERTIES:");
+    console.log("=".repeat(60));
+    console.log("  found:", element.found());
+    console.log("  x:", element.x);
+    console.log("  y:", element.y);
+    console.log("  centerX:", element.centerX);
+    console.log("  centerY:", element.centerY);
+    console.log("  width:", element.width);
+    console.log("  height:", element.height);
+    console.log("  confidence:", element.confidence);
+    console.log("  text:", element.text);
+    console.log("  label:", element.label);
+    console.log(
+      "  screenshot:",
+      element.screenshot ? `${element.screenshot.length} chars` : null,
+    );
+    console.log("  boundingBox:", element.boundingBox);
+
+    await client.disconnect();
+  } catch (error) {
+    console.error("Error:", error.message);
+    console.error(error.stack);
+    await client.disconnect();
+    process.exit(1);
+  }
+}
+
+debugLocateResponse();

package/debugger/index.html

@@ -34,10 +34,14 @@
         top: 0;
         left: 0;
         transform-origin: top left;
+        overflow: hidden;
       }
 
+      html,
       body {
         overflow: hidden;
+        width: 100%;
+        height: 100%;
       }
 
       *::-webkit-scrollbar {
@@ -66,6 +70,7 @@
 
       .overlay {
         position: relative;
+        overflow: hidden;
       }
 
       .screenshot {
@@ -179,6 +184,7 @@
         width: 100%;
         height: 100%;
         z-index: 1;
+        overflow: hidden;
       }
 
       /* Loading screen styles */
@@ -354,6 +360,10 @@
 
       const iframe = document.querySelector("#vm-iframe");
 
+      // Detect if OS is Linux
+      const isLinux = parsedData.os === "linux";
+      const topBarOffset = isLinux ? 14 : 0;
+
       // set overlay width and height to match the given resolution
       const overlayWidth = parsedData.resolution[0];
       const overlayHeight = parsedData.resolution[1];
@@ -361,7 +371,8 @@
       iframe.style.display = "block";
       iframe.src = parsedData.url;
       iframe.style.width = overlayWidth + "px";
-      iframe.style.height = overlayHeight + "px";
+      // Increase iframe height by 14px for Linux to account for top bar
+      iframe.style.height = overlayHeight + topBarOffset + "px";
 
       // Calculate scale factor to fit within window if needed
       const windowWidth = window.innerWidth;
@@ -513,7 +524,7 @@
           const boxElement = document.createElement("div");
           boxElement.className = "bounding-box";
           boxElement.style.left = toCss(box.x);
-          boxElement.style.top = toCss(box.y);
+          boxElement.style.top = toCss(box.y + topBarOffset);
           boxElement.style.width = toCss(box.width);
           boxElement.style.height = toCss(box.height);
           effects.appendChild(boxElement);
@@ -541,14 +552,14 @@
       // Mouse event handlers
       addEventHandler(events.mouseMove, (event, { x, y } = {}) => {
         mouse.style.marginLeft = toCss(x);
-        mouse.style.marginTop = toCss(y);
+        mouse.style.marginTop = toCss(y + topBarOffset);
       });
 
       addEventHandler(
         events.mouseClick,
         (event, { x, y, click = "single" } = {}) => {
           mouse.style.marginLeft = toCss(x);
-          mouse.style.marginTop = toCss(y);
+          mouse.style.marginTop = toCss(y + topBarOffset);
           // Reset class so animation can restart
           mouse.setAttribute("class", "mouse");
           // Force reflow