testdriverai 6.2.1 → 7.0.0

package/agent/lib/commands.js

@@ -44,11 +44,12 @@ const createCommands = (
   config,
   sessionInstance,
   getCurrentFilePath,
+  redrawThreshold = 0.1,
 ) => {
   // Create SDK instance with emitter, config, and session
   const sdk = createSDK(emitter, config, sessionInstance);
   // Create redraw instance with the system
-  const redraw = createRedraw(emitter, system, sandbox);
+  const redraw = createRedraw(emitter, system, sandbox, redrawThreshold);
 
   // Helper method to resolve file paths relative to the current file
   const resolveRelativePath = (relativePath) => {
@@ -73,6 +74,7 @@ const createCommands = (
     return Math.round(ms / 1000);
   };
   const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
   const findImageOnScreen = async (
     relativePath,
     haystack,
@@ -170,103 +172,64 @@ const createCommands = (
     return result;
   };
 
-  const assert = async (
-    assertion,
-    shouldThrow = false,
-    async = false,
-    invert = false,
-  ) => {
-    if (async) {
-      shouldThrow = true;
-    }
-
+  const assert = async (assertion, shouldThrow = false) => {
     const handleAssertResponse = (response) => {
       emitter.emit(events.log.log, response);
 
       let valid = response.indexOf("The task passed") > -1;
 
-      if (invert) {
-        valid = !valid;
-      }
-
       if (valid) {
         return true;
       } else {
         if (shouldThrow) {
-          // Is fatal, othewise it just changes the assertion to be true
-          throw new MatchError(
-            `AI Assertion failed ${invert && "(Inverted)"}`,
-            true,
-          );
+          // Is fatal, otherwise it just changes the assertion to be true
+          const errorMessage = `AI Assertion failed: ${assertion}\n${response}`;
+          throw new MatchError(errorMessage, true);
         } else {
           return false;
         }
       }
     };
 
-    emitter.emit(events.log.narration, `thinking...`);
+    // Log asserting action
+    const { formatter } = require("../../sdk-log-formatter.js");
+    const assertingMessage = formatter.formatAsserting(assertion);
+    emitter.emit(events.log.log, assertingMessage);
 
-    if (async) {
-      await sdk
-        .req("assert", {
-          expect: assertion,
-          image: await system.captureScreenBase64(),
-        })
-        .then((response) => {
-          return handleAssertResponse(response.data);
-        });
+    emitter.emit(events.log.narration, `thinking...`);
 
-      return true;
-    } else {
-      let response = await sdk.req("assert", {
-        expect: assertion,
-        image: await system.captureScreenBase64(),
-      });
-      return handleAssertResponse(response.data);
-    }
+    let response = await sdk.req("assert", {
+      expect: assertion,
+      image: await system.captureScreenBase64(),
+    });
+    return handleAssertResponse(response.data);
   };
-  const scroll = async (direction = "down", amount = 300, method = "mouse") => {
+  const scroll = async (direction = "down", amount = 300) => {
+    emitter.emit(
+      events.log.narration,
+      theme.dim(`scrolling ${direction} ${amount}px...`),
+    );
+
     await redraw.start();
 
     amount = parseInt(amount, 10);
 
-    // if direction is down, amount should be negative
-    if (direction === "down") {
-      amount = -Math.abs(amount);
-    } else if (direction === "up") {
-      amount = Math.abs(amount);
-    }
-
     const before = await system.captureScreenBase64();
     switch (direction) {
       case "up":
-        if (method === "mouse") {
-          await sandbox.send({
-            os: "linux",
-            type: "scroll",
-            amount,
-            direction,
-          });
-        } else {
-          await sandbox.send({ os: "linux", type: "press", keys: ["pageup"] });
-        }
+        await sandbox.send({
+          type: "scroll",
+          amount,
+          direction,
+        });
         await redraw.wait(2500);
         break;
       case "down":
-        if (method === "mouse") {
-          await sandbox.send({
-            os: "linux",
-            type: "scroll",
-            amount,
-            direction,
-          });
-        } else {
-          await sandbox.send({
-            os: "linux",
-            type: "press",
-            keys: ["pagedown"],
-          });
-        }
+        await sandbox.send({
+          type: "scroll",
+          amount,
+          direction,
+        });
         await redraw.wait(2500);
         break;
       case "left":
@@ -312,7 +275,7 @@ const createCommands = (
     x = parseInt(x);
     y = parseInt(y);
 
-    await sandbox.send({ os: "linux", type: "moveMouse", x, y });
+    await sandbox.send({ type: "moveMouse", x, y });
 
     emitter.emit(events.mouseMove, { x, y });
 
@@ -320,18 +283,17 @@ const createCommands = (
 
     if (action !== "hover") {
       if (action === "click" || action === "left-click") {
-        await sandbox.send({ os: "linux", type: "leftClick" });
+        await sandbox.send({ type: "leftClick" });
       } else if (action === "right-click") {
-        await sandbox.send({ os: "linux", type: "rightClick" });
+        await sandbox.send({ type: "rightClick" });
       } else if (action === "middle-click") {
-        await sandbox.send({ os: "linux", type: "middleClick" });
+        await sandbox.send({ type: "middleClick" });
       } else if (action === "double-click") {
-        await sandbox.send({ os: "linux", type: "doubleClick" });
-      } else if (action === "drag-start") {
-        await sandbox.send({ os: "linux", type: "mousePress", button: "left" });
-      } else if (action === "drag-end") {
+        await sandbox.send({ type: "doubleClick" });
+      } else if (action === "mouseDown") {
+        await sandbox.send({ type: "mousePress", button: "left" });
+      } else if (action === "mouseUp") {
         await sandbox.send({
-          os: "linux",
           type: "mouseRelease",
           button: "left",
         });
@@ -346,12 +308,14 @@ const createCommands = (
   };
 
   const hover = async (x, y) => {
+    emitter.emit(events.log.narration, theme.dim(`hovering at ${x}, ${y}...`));
+
     await redraw.start();
 
     x = parseInt(x);
     y = parseInt(y);
 
-    await sandbox.send({ os: "linux", type: "moveMouse", x, y });
+    await sandbox.send({ type: "moveMouse", x, y });
 
     await redraw.wait(2500);
 
@@ -370,9 +334,15 @@ const createCommands = (
       text,
       description = null,
       action = "click",
-      method = "turbo",
       timeout = 5000, // we pass this to the subsequent wait-for-text block
     ) => {
+      emitter.emit(
+        events.log.narration,
+        theme.dim(
+          `searching for "${text}"${description ? ` (${description})` : ""}...`,
+        ),
+      );
+
       text = text ? text.toString() : null;
 
       // wait for the text to appear on screen
@@ -382,56 +352,61 @@ const createCommands = (
 
       emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
-      let response = await sdk.req(
-        "hover/text",
-        {
-          needle: text,
-          method,
-          image: await system.captureScreenBase64(),
-          intent: action,
-          description,
-          displayMultiple: 1,
-        },
-        (chunk) => {
-          if (chunk.type === "closeMatches") {
-            emitter.emit(events.matches.show, chunk.data);
-          }
-        },
-      );
+      // Combine text and description into element parameter
+      let element = text;
+      if (description) {
+        element = `"${text}" with description ${description}`;
+      }
 
-      if (!response.data) {
+      let response = await sdk.req("find", {
+        element,
+        image: await system.captureScreenBase64(),
+      });
+
+      if (!response || !response.coordinates) {
         throw new MatchError("No text on screen matches description");
+      }
+
+      // Perform the action using the located coordinates
+      if (action === "hover") {
+        await commands.hover(response.coordinates.x, response.coordinates.y);
       } else {
-        return response.data;
+        await click(response.coordinates.x, response.coordinates.y, action);
       }
+
+      return response;
     },
     // uses our api to find all images on screen
     "hover-image": async (description, action = "click") => {
-      // take a screenshot
-      emitter.emit(events.log.narration, theme.dim("thinking..."), true);
-
-      let response = await sdk.req(
-        "hover/image",
-        {
-          needle: description,
-          image: await system.captureScreenBase64(),
-          intent: action,
-          displayMultiple: 1,
-        },
-        (chunk) => {
-          if (chunk.type === "closeMatches") {
-            emitter.emit(events.matches.show, chunk.data);
-          }
-        },
+      emitter.emit(
+        events.log.narration,
+        theme.dim(`searching for image: "${description}"...`),
       );
 
-      if (!response?.data) {
+      let response = await sdk.req("find", {
+        element: description,
+        image: await system.captureScreenBase64(),
+      });
+
+      if (!response || !response.coordinates) {
         throw new MatchError("No image or icon on screen matches description");
+      }
+
+      // Perform the action using the located coordinates
+      if (action === "hover") {
+        await commands.hover(response.coordinates.x, response.coordinates.y);
       } else {
-        return response.data;
+        await click(response.coordinates.x, response.coordinates.y, action);
       }
+
+      return response;
     },
     "match-image": async (relativePath, action = "click", invert = false) => {
+      emitter.emit(
+        events.log.narration,
+        theme.dim(`${action} on image template "${relativePath}"...`),
+      );
+
       // Resolve the image path relative to the current file
       const resolvedPath = resolveRelativePath(relativePath);
 
@@ -456,23 +431,32 @@ const createCommands = (
       return true;
     },
     // type a string
-    os: "linux",
+
     type: async (string, delay = 250) => {
+      emitter.emit(events.log.narration, theme.dim(`typing "${string}"...`));
+
       await redraw.start();
 
       string = string.toString();
 
-      await sandbox.send({ os: "linux", type: "write", text: string, delay });
+      await sandbox.send({ type: "write", text: string, delay });
       await redraw.wait(5000);
       return;
     },
     // press keys
     // different than `type`, becasue it can press multiple keys at once
     "press-keys": async (inputKeys) => {
+      emitter.emit(
+        events.log.narration,
+        theme.dim(
+          `pressing keys: ${Array.isArray(inputKeys) ? inputKeys.join(", ") : inputKeys}...`,
+        ),
+      );
+
       await redraw.start();
 
       // finally, press the keys
-      await sandbox.send({ os: "linux", type: "press", keys: inputKeys });
+      await sandbox.send({ type: "press", keys: inputKeys });
 
       await redraw.wait(5000);
 
@@ -480,9 +464,10 @@ const createCommands = (
     },
     // simple delay, usually to let ui render or webpage to load
     wait: async (timeout = 3000) => {
+      emitter.emit(events.log.narration, theme.dim(`waiting ${timeout}ms...`));
       return await delay(timeout);
     },
-    "wait-for-image": async (description, timeout = 10000, invert = false) => {
+    "wait-for-image": async (description, timeout = 10000) => {
       emitter.emit(
         events.log.narration,
         theme.dim(
@@ -499,8 +484,6 @@ const createCommands = (
         passed = await assert(
           `An image matching the description "${description}" appears on screen.`,
           false,
-          false,
-          invert,
         );
 
         durationPassed = new Date().getTime() - startTime;
@@ -531,12 +514,7 @@ const createCommands = (
         );
       }
     },
-    "wait-for-text": async (
-      text,
-      timeout = 5000,
-      method = "turbo",
-      invert = false,
-    ) => {
+    "wait-for-text": async (text, timeout = 5000) => {
       await redraw.start();
 
       emitter.emit(
@@ -551,25 +529,13 @@ const createCommands = (
       let passed = false;
 
       while (durationPassed < timeout && !passed) {
-        const response = await sdk.req(
-          "assert/text",
-          {
-            needle: text,
-            method: method,
-            image: await system.captureScreenBase64(),
-          },
-          (chunk) => {
-            if (chunk.type === "closeMatches") {
-              emitter.emit(events.matches.show, chunk.data);
-            }
-          },
-        );
+        const response = await sdk.req("find", {
+          element: text,
+          image: await system.captureScreenBase64(),
+        });
 
-        passed = response.data;
+        passed = !!(response && response.coordinates);
 
-        if (invert) {
-          passed = !passed;
-        }
         durationPassed = new Date().getTime() - startTime;
 
         if (!passed) {
@@ -597,8 +563,6 @@ const createCommands = (
       text,
       direction = "down",
       maxDistance = 10000,
-      textMatchMethod = "turbo",
-      method = "keyboard",
       invert = false,
     ) => {
       await redraw.start();
@@ -609,44 +573,17 @@ const createCommands = (
         true,
       );
 
-      if (method === "keyboard") {
-        try {
-          await sandbox.send({
-            os: "linux",
-            type: "press",
-            keys: ["f", "ctrl"],
-          });
-          await delay(1000);
-          await sandbox.send({ os: "linux", type: "write", text });
-          await redraw.wait(5000);
-          await sandbox.send({ os: "linux", type: "press", keys: ["escape"] });
-        } catch {
-          throw new MatchError(
-            "Could not find element using browser text search",
-          );
-        }
-      }
-
       let scrollDistance = 0;
       let incrementDistance = 500;
       let passed = false;
 
       while (scrollDistance <= maxDistance && !passed) {
-        const response = await sdk.req(
-          "assert/text",
-          {
-            needle: text,
-            method: textMatchMethod,
-            image: await system.captureScreenBase64(),
-          },
-          (chunk) => {
-            if (chunk.type === "closeMatches") {
-              emitter.emit(events.matches.show, chunk.data);
-            }
-          },
-        );
+        const response = await sdk.req("find", {
+          element: text,
+          image: await system.captureScreenBase64(),
+        });
 
-        passed = response.data;
+        passed = !!(response && response.coordinates);
 
         if (invert) {
           passed = !passed;
@@ -660,7 +597,7 @@ const createCommands = (
             ),
             true,
           );
-          await scroll(direction, incrementDistance, method);
+          await scroll(direction, incrementDistance);
           scrollDistance = scrollDistance + incrementDistance;
         }
       }
@@ -678,7 +615,7 @@ const createCommands = (
       description,
       direction = "down",
       maxDistance = 10000,
-      method = "keyboard",
+      method = "mouse",
       path,
       invert = false,
     ) => {
@@ -750,7 +687,6 @@ const createCommands = (
       await redraw.start();
 
       await sandbox.send({
-        os: "linux",
         type: "commands.focus-application",
         name,
       });
@@ -764,8 +700,8 @@ const createCommands = (
       });
       return result.data;
     },
-    assert: async (assertion, async = false, invert = false) => {
-      let response = await assert(assertion, true, async, invert);
+    assert: async (assertion) => {
+      let response = await assert(assertion, true);
 
       return response;
     },
@@ -777,17 +713,36 @@ const createCommands = (
       let plat = system.platform();
 
       if (language == "pwsh" || language == "sh") {
+        if (language === "pwsh" && sandbox.os === "linux") {
+          emitter.emit(
+            events.log.log,
+            theme.yellow(
+              `⚠️  Warning: You are using 'pwsh' exec command on a Linux sandbox. This may fail. Consider using 'bash' or 'sh' for Linux environments.`,
+            ),
+            true,
+          );
+        }
+
+        if (language === "sh" && sandbox.os === "windows") {
+          emitter.emit(
+            events.log.log,
+            theme.yellow(
+              `⚠️  Warning: You are using 'sh' exec command on a Windows sandbox. This will fail. Automatically switching to 'pwsh' for Windows environments.`,
+            ),
+            true,
+          );
+          // Automatically switch to pwsh for Windows
+          language = "pwsh";
+        }
+
         let result = null;
 
         result = await sandbox.send({
-          os: "linux",
           type: "commands.run",
           command: code,
           timeout,
         });
 
-        console.log("Exec result:", result);
-
         if (result.out && result.out.returncode !== 0) {
           throw new MatchError(
             `Command failed with exit code ${result.out.returncode}: ${result.out.stderr}`,

package/agent/lib/redraw.js

@@ -4,8 +4,12 @@ const { events } = require("../events");
 const theme = require("./theme");
 
 // Factory function that creates redraw functionality with the provided system instance
-const createRedraw = (emitter, system, sandbox) => {
-  const redrawThresholdPercent = 0.1;
+const createRedraw = (
+  emitter,
+  system,
+  sandbox,
+  redrawThresholdPercent = 0.1,
+) => {
   const networkUpdateInterval = 15000;
 
   let lastTxBytes = null;
@@ -69,7 +73,6 @@ const createRedraw = (emitter, system, sandbox) => {
   async function updateNetwork() {
     if (sandbox && sandbox.instanceSocketConnected) {
       let network = await sandbox.send({
-        os: "linux",
         type: "system.network",
       });
       parseNetworkStats(

package/agent/lib/sandbox.js

@@ -14,6 +14,7 @@ const createSandbox = (emitter, analytics) => {
       this.instance = null;
       this.messageId = 0;
       this.uniqueId = Math.random().toString(36).substring(7);
+      this.os = null; // Store OS value to send with every message
     }
 
     send(message) {
@@ -24,6 +25,16 @@ const createSandbox = (emitter, analytics) => {
         this.messageId++;
         message.requestId = `${this.uniqueId}-${this.messageId}`;
 
+        // If os is set in the message, store it for future messages
+        if (message.os) {
+          this.os = message.os;
+        }
+
+        // Add os to every message if it's been set
+        if (this.os && !message.os) {
+          message.os = this.os;
+        }
+
         // Start timing for this message
         const timingKey = `sandbox-${message.type}`;
         marky.mark(timingKey);
@@ -51,7 +62,6 @@ const createSandbox = (emitter, analytics) => {
     async auth(apiKey) {
       let reply = await this.send({
         type: "authenticate",
-        os: "linux",
         apiKey,
       });
 
@@ -65,7 +75,6 @@ const createSandbox = (emitter, analytics) => {
     async connect(sandboxId, persist = false) {
       let reply = await this.send({
         type: "connect",
-        os: "linux",
         persist,
         sandboxId,
       });
@@ -73,9 +82,12 @@ const createSandbox = (emitter, analytics) => {
       if (reply.success) {
         this.instanceSocketConnected = true;
         emitter.emit(events.sandbox.connected);
+        // Return the full reply (includes url and sandbox)
+        return reply;
+      } else {
+        // Throw error to trigger fallback to creating new sandbox
+        throw new Error(reply.errorMessage || "Failed to connect to sandbox");
       }
-
-      return reply.sandbox;
     }
 
     async boot(apiRoot) {
@@ -124,7 +136,9 @@ const createSandbox = (emitter, analytics) => {
 
           if (message.error) {
             emitter.emit(events.error.sandbox, message.errorMessage);
-            this.ps[message.requestId].reject(JSON.stringify(message));
+            const error = new Error(message.errorMessage || "Sandbox error");
+            error.responseData = message;
+            this.ps[message.requestId].reject(error);
           } else {
             emitter.emit(events.sandbox.received);
 

package/agent/lib/sdk.js

@@ -123,6 +123,7 @@ const createSDK = (emitter, config, sessionInstance) => {
         ...(token && { Authorization: `Bearer ${token}` }), // Add the authorization bearer token only if token is set
       },
       responseType: typeof onChunk === "function" ? "stream" : "json",
+      timeout: 60000, // 60 second timeout to prevent hanging requests
       data: {
         ...data,
         session: sessionInstance.get(),

package/agent/lib/system.js

@@ -8,7 +8,6 @@ const { events } = require("../events.js");
 const createSystem = (emitter, sandbox, config) => {
   const screenshot = async (options) => {
     let { base64 } = await sandbox.send({
-      os: "linux",
       type: "system.screenshot",
     });
 
@@ -113,7 +112,6 @@ const createSystem = (emitter, sandbox, config) => {
   const activeWin = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
-      os: "linux",
       type: "system.get-active-window",
     });
 
@@ -123,7 +121,6 @@ const createSystem = (emitter, sandbox, config) => {
   const getMousePosition = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
-      os: "linux",
       type: "system.get-mouse-position",
     });