testdriverai 6.1.10 → 6.2.0

package/agent/lib/commander.js

@@ -80,21 +80,21 @@ commands:
       // this will actually interpret the command and execute it
       switch (object.command) {
         case "type":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `typing ${object.text}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.type(object.text, object.delay);
           break;
         case "press-keys":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
             events.log.narration,
-            `pressing keys: ${Array.isArray(object.keys) ? object.keys.join(", ") : object.keys}`,
+            `pressing keys ${object.keys.join(",")}`,
           );
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["press-keys"](object.keys);
           break;
         case "scroll":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `scrolling ${object.direction}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.scroll(
             object.direction,
             object.amount,
@@ -102,21 +102,21 @@ commands:
           );
           break;
         case "wait":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
             events.log.narration,
             `waiting ${object.timeout} seconds`,
           );
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands.wait(object.timeout);
           break;
         case "click":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `${object.action}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["click"](object.x, object.y, object.action);
           break;
         case "hover":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `moving mouse`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover"](object.x, object.y);
           break;
         case "drag":
@@ -124,25 +124,24 @@ commands:
           response = await commands["drag"](object.x, object.y);
           break;
         case "hover-text":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
             events.log.narration,
             `searching for ${object.description}`,
           );
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover-text"](
             object.text,
             object.description,
             object.action,
             object.method,
-            object.timeout,
           );
           break;
         case "hover-image":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
             events.log.narration,
             `searching for image of ${object.description}`,
           );
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover-image"](
             object.description,
             object.action,
@@ -154,38 +153,32 @@ commands:
             events.log.narration,
             `${object.action} image ${object.path}`,
           );
-          response = await commands["match-image"](
-            object.path,
-            object.action,
-            object.invert,
-          );
+          response = await commands["match-image"](object.path, object.action);
           break;
         case "wait-for-image":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(
             events.log.narration,
             `waiting for ${object.description}`,
           );
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["wait-for-image"](
             object.description,
             object.timeout,
-            object.invert,
           );
           break;
         case "wait-for-text":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `waiting for ${object.text}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           copy.text = "*****";
           response = await commands["wait-for-text"](
             object.text,
             object.timeout,
             object.method,
-            object.invert,
           );
           break;
         case "scroll-until-text":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `scrolling until ${object.text}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           copy.text = "*****";
           response = await commands["scroll-until-text"](
             object.text,
@@ -193,26 +186,24 @@ commands:
             object.distance,
             object.textMatchMethod,
             object.method,
-            object.invert,
           );
           break;
         case "scroll-until-image": {
           const needle = object.description || object.path;
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `scrolling until ${needle}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["scroll-until-image"](
             object.description,
             object.direction,
             object.distance,
             object.method,
             object.path,
-            object.invert,
           );
           break;
         }
         case "focus-application":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `focusing ${object.name}`);
+          emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["focus-application"](object.name);
           break;
         case "remember": {
@@ -223,16 +214,12 @@ commands:
           break;
         }
         case "assert":
-          emitter.emit(events.log.log, generator.jsonToManual(object));
           emitter.emit(events.log.narration, `asserting ${object.expect}`);
-          response = await commands.assert(
-            object.expect,
-            object.async,
-            object.invert,
-          );
-
+          emitter.emit(events.log.log, generator.jsonToManual(object));
+          response = await commands.assert(object.expect, object.async);
           break;
         case "exec":
+          emitter.emit(events.log.narration, `exec`);
           emitter.emit(
             events.log.log,
             generator.jsonToManual({

package/agent/lib/commands.js

@@ -170,34 +170,20 @@ const createCommands = (
     return result;
   };
 
-  const assert = async (
-    assertion,
-    shouldThrow = false,
-    async = false,
-    invert = false,
-  ) => {
+  const assert = async (assertion, shouldThrow = false, async = false) => {
     if (async) {
       shouldThrow = true;
     }
 
     const handleAssertResponse = (response) => {
-      emitter.emit(events.log.log, response);
-
-      let valid = response.indexOf("The task passed") > -1;
-
-      if (invert) {
-        valid = !valid;
-      }
+      emitter.emit(events.log.markdown.static, response);
 
-      if (valid) {
+      if (response.indexOf("The task passed") > -1) {
         return true;
       } else {
         if (shouldThrow) {
           // Is fatal, othewise it just changes the assertion to be true
-          throw new MatchError(
-            `AI Assertion failed ${invert && "(Inverted)"}`,
-            true,
-          );
+          throw new MatchError(`AI Assertion failed`, true);
         } else {
           return false;
         }
@@ -241,17 +227,31 @@ const createCommands = (
     switch (direction) {
       case "up":
         if (method === "mouse") {
-          await sandbox.send({ type: "scroll", amount, direction });
+          await sandbox.send({
+            os: "linux",
+            type: "scroll",
+            amount,
+            direction,
+          });
         } else {
-          await sandbox.send({ type: "press", keys: ["pageup"] });
+          await sandbox.send({ os: "linux", type: "press", keys: ["pageup"] });
         }
         await redraw.wait(2500);
         break;
       case "down":
         if (method === "mouse") {
-          await sandbox.send({ type: "scroll", amount, direction });
+          await sandbox.send({
+            os: "linux",
+            type: "scroll",
+            amount,
+            direction,
+          });
         } else {
-          await sandbox.send({ type: "press", keys: ["pagedown"] });
+          await sandbox.send({
+            os: "linux",
+            type: "press",
+            keys: ["pagedown"],
+          });
         }
         await redraw.wait(2500);
         break;
@@ -298,7 +298,7 @@ const createCommands = (
     x = parseInt(x);
     y = parseInt(y);
 
-    await sandbox.send({ type: "moveMouse", x, y });
+    await sandbox.send({ os: "linux", type: "moveMouse", x, y });
 
     emitter.emit(events.mouseMove, { x, y });
 
@@ -306,17 +306,21 @@ const createCommands = (
 
     if (action !== "hover") {
       if (action === "click" || action === "left-click") {
-        await sandbox.send({ type: "leftClick" });
+        await sandbox.send({ os: "linux", type: "leftClick" });
       } else if (action === "right-click") {
-        await sandbox.send({ type: "rightClick" });
+        await sandbox.send({ os: "linux", type: "rightClick" });
       } else if (action === "middle-click") {
-        await sandbox.send({ type: "middleClick" });
+        await sandbox.send({ os: "linux", type: "middleClick" });
       } else if (action === "double-click") {
-        await sandbox.send({ type: "doubleClick" });
+        await sandbox.send({ os: "linux", type: "doubleClick" });
       } else if (action === "drag-start") {
-        await sandbox.send({ type: "mousePress", button: "left" });
+        await sandbox.send({ os: "linux", type: "mousePress", button: "left" });
       } else if (action === "drag-end") {
-        await sandbox.send({ type: "mouseRelease", button: "left" });
+        await sandbox.send({
+          os: "linux",
+          type: "mouseRelease",
+          button: "left",
+        });
       }
 
       emitter.emit(events.mouseClick, { x, y, button, click, double });
@@ -333,7 +337,7 @@ const createCommands = (
     x = parseInt(x);
     y = parseInt(y);
 
-    await sandbox.send({ type: "moveMouse", x, y });
+    await sandbox.send({ os: "linux", type: "moveMouse", x, y });
 
     await redraw.wait(2500);
 
@@ -353,12 +357,11 @@ const createCommands = (
       description = null,
       action = "click",
       method = "turbo",
-      timeout = 5000, // we pass this to the subsequent wait-for-text block
     ) => {
       text = text ? text.toString() : null;
 
       // wait for the text to appear on screen
-      await commands["wait-for-text"](text, timeout);
+      await commands["wait-for-text"](text, 5000);
 
       description = description ? description.toString() : null;
 
@@ -413,7 +416,7 @@ const createCommands = (
         return response.data;
       }
     },
-    "match-image": async (relativePath, action = "click", invert = false) => {
+    "match-image": async (relativePath, action = "click") => {
       // Resolve the image path relative to the current file
       const resolvedPath = resolveRelativePath(relativePath);
 
@@ -421,10 +424,6 @@ const createCommands = (
 
       let result = await findImageOnScreen(resolvedPath, image);
 
-      if (invert) {
-        result = !result;
-      }
-
       if (!result) {
         throw new CommandError(`Image not found: ${resolvedPath}`);
       } else {
@@ -438,12 +437,13 @@ const createCommands = (
       return true;
     },
     // type a string
+    os: "linux",
     type: async (string, delay = 250) => {
       await redraw.start();
 
       string = string.toString();
 
-      await sandbox.send({ type: "write", text: string, delay });
+      await sandbox.send({ os: "linux", type: "write", text: string, delay });
       await redraw.wait(5000);
       return;
     },
@@ -453,7 +453,7 @@ const createCommands = (
       await redraw.start();
 
       // finally, press the keys
-      await sandbox.send({ type: "press", keys: inputKeys });
+      await sandbox.send({ os: "linux", type: "press", keys: inputKeys });
 
       await redraw.wait(5000);
 
@@ -463,7 +463,7 @@ const createCommands = (
     wait: async (timeout = 3000) => {
       return await delay(timeout);
     },
-    "wait-for-image": async (description, timeout = 10000, invert = false) => {
+    "wait-for-image": async (description, timeout = 10000) => {
       emitter.emit(
         events.log.narration,
         theme.dim(
@@ -481,7 +481,6 @@ const createCommands = (
           `An image matching the description "${description}" appears on screen.`,
           false,
           false,
-          invert,
         );
 
         durationPassed = new Date().getTime() - startTime;
@@ -512,12 +511,7 @@ const createCommands = (
         );
       }
     },
-    "wait-for-text": async (
-      text,
-      timeout = 5000,
-      method = "turbo",
-      invert = false,
-    ) => {
+    "wait-for-text": async (text, timeout = 5000, method = "turbo") => {
       await redraw.start();
 
       emitter.emit(
@@ -547,12 +541,7 @@ const createCommands = (
         );
 
         passed = response.data;
-
-        if (invert) {
-          passed = !passed;
-        }
         durationPassed = new Date().getTime() - startTime;
-
         if (!passed) {
           emitter.emit(
             events.log.narration,
@@ -580,7 +569,6 @@ const createCommands = (
       maxDistance = 10000,
       textMatchMethod = "turbo",
       method = "keyboard",
-      invert = false,
     ) => {
       await redraw.start();
 
@@ -592,11 +580,15 @@ const createCommands = (
 
       if (method === "keyboard") {
         try {
-          await sandbox.send({ type: "press", keys: ["f", "ctrl"] });
+          await sandbox.send({
+            os: "linux",
+            type: "press",
+            keys: ["f", "ctrl"],
+          });
           await delay(1000);
-          await sandbox.send({ type: "write", text });
+          await sandbox.send({ os: "linux", type: "write", text });
           await redraw.wait(5000);
-          await sandbox.send({ type: "press", keys: ["escape"] });
+          await sandbox.send({ os: "linux", type: "press", keys: ["escape"] });
         } catch {
           throw new MatchError(
             "Could not find element using browser text search",
@@ -624,11 +616,6 @@ const createCommands = (
         );
 
         passed = response.data;
-
-        if (invert) {
-          passed = !passed;
-        }
-
         if (!passed) {
           emitter.emit(
             events.log.narration,
@@ -657,7 +644,6 @@ const createCommands = (
       maxDistance = 10000,
       method = "keyboard",
       path,
-      invert = false,
     ) => {
       const needle = description || path;
 
@@ -687,7 +673,6 @@ const createCommands = (
             `An image matching the description "${description}" appears on screen.`,
             false,
             false,
-            invert,
           );
         }
 
@@ -727,6 +712,7 @@ const createCommands = (
       await redraw.start();
 
       await sandbox.send({
+        os: "linux",
         type: "commands.focus-application",
         name,
       });
@@ -740,40 +726,41 @@ const createCommands = (
       });
       return result.data;
     },
-    assert: async (assertion, async = false, invert = false) => {
-      let response = await assert(assertion, true, async, invert);
-
-      return response;
+    assert: async (assertion, async = false) => {
+      return await assert(assertion, true, async);
     },
-    exec: async (language, code, timeout, silent = false) => {
+    exec: async (language = "pwsh", code, timeout, silent = false) => {
       emitter.emit(events.log.narration, theme.dim(`calling exec...`), true);
 
       emitter.emit(events.log.log, code);
 
       let plat = system.platform();
 
-      if (language == "pwsh") {
+      if (language == "pwsh" || language == "sh") {
         let result = null;
 
         result = await sandbox.send({
+          os: "linux",
           type: "commands.run",
           command: code,
           timeout,
         });
 
+        console.log("Exec result:", result);
+
         if (result.out && result.out.returncode !== 0) {
           throw new MatchError(
             `Command failed with exit code ${result.out.returncode}: ${result.out.stderr}`,
           );
         } else {
-          if (!silent && result.out?.stdout) {
-            emitter.emit(events.log.log, theme.dim(`stdout:`), true);
+          if (!silent) {
+            emitter.emit(events.log.log, theme.dim(`Command stdout:`), true);
             emitter.emit(events.log.log, `${result.out.stdout}`, true);
-          }
 
-          if (!silent && result.out.stderr) {
-            emitter.emit(events.log.log, theme.dim(`stderr:`), true);
-            emitter.emit(events.log.log, `${result.out.stderr}`, true);
+            if (result.out.stderr) {
+              emitter.emit(events.log.log, theme.dim(`Command stderr:`), true);
+              emitter.emit(events.log.log, `${result.out.stderr}`, true);
+            }
           }
 
           return result.out?.stdout?.trim();
@@ -805,12 +792,7 @@ const createCommands = (
         try {
           await script.runInNewContext(context);
         } catch (e) {
-          // Log the error to the emitter instead of console.error to maintain consistency
-          emitter.emit(
-            events.log.debug,
-            `JavaScript execution error: ${e.message}`,
-          );
-          // Wait a tick to allow any promise rejections to be handled
+          console.error(e);
           throw new CommandError(`Error running script: ${e.message}`);
         }
 

package/agent/lib/debugger-server.js

@@ -65,6 +65,11 @@ function createDebuggerServer(config = {}) {
 
       ws.on("close", () => {
         clients.delete(ws);
+
+        // If no clients connected, we can optionally shut down
+        if (clients.size === 0) {
+          console.log("No clients connected, keeping server alive");
+        }
       });
 
       ws.on("error", (error) => {

package/agent/lib/generator.js

@@ -1,6 +1,6 @@
 // parses markdown content to find code blocks, and then extracts yaml from those code blocks
 const yaml = require("js-yaml");
-const pkg = require("../../package.json");
+const package = require("../../package.json");
 const session = require("./session");
 const theme = require("./theme");
 // do the actual parsing
@@ -56,7 +56,7 @@ const jsonToManual = function (json, colors = true) {
 const dumpToYML = async function (inputArray, sessionInstance = null) {
   // use yml dump to convert json to yml
   let yml = await yaml.dump({
-    version: pkg.version,
+    version: package.version,
     session: sessionInstance ? sessionInstance.get() : session.get(),
     steps: inputArray,
   });

package/agent/lib/redraw.js

@@ -69,6 +69,7 @@ const createRedraw = (emitter, system, sandbox) => {
   async function updateNetwork() {
     if (sandbox && sandbox.instanceSocketConnected) {
       let network = await sandbox.send({
+        os: "linux",
         type: "system.network",
       });
       parseNetworkStats(

package/agent/lib/sandbox.js

@@ -51,6 +51,7 @@ const createSandbox = (emitter, analytics) => {
     async auth(apiKey) {
       let reply = await this.send({
         type: "authenticate",
+        os: "linux",
         apiKey,
       });
 
@@ -64,6 +65,7 @@ const createSandbox = (emitter, analytics) => {
     async connect(sandboxId, persist = false) {
       let reply = await this.send({
         type: "connect",
+        os: "linux",
         persist,
         sandboxId,
       });

package/agent/lib/sdk.js

@@ -95,7 +95,7 @@ const createSDK = (emitter, config, sessionInstance) => {
         return token;
       } catch (error) {
         outputError(error);
-        throw error; // Re-throw the error so calling code can handle it properly
+        return;
       }
     }
   };
@@ -194,7 +194,6 @@ const createSDK = (emitter, config, sessionInstance) => {
       return value;
     } catch (error) {
       outputError(error);
-      throw error; // Re-throw the error so calling code can handle it properly
     }
   };
 

package/agent/lib/source-mapper.js

@@ -263,7 +263,7 @@ class SourceMapper {
     let description = `${fileName}:${(sourcePosition.step.startLine || 0) + 1}`;
 
     if (sourcePosition.command) {
-      description += `:${(sourcePosition.command.startLine || 0) + 1}`;
+      description += `:${(sourcePosition.command.startLine || 0) + 1} (${sourcePosition.command.command || "unknown command"})`;
     } else {
       description += ` (step ${sourcePosition.step.stepIndex + 1})`;
     }

package/agent/lib/system.js

@@ -7,7 +7,10 @@ const { events } = require("../events.js");
 
 const createSystem = (emitter, sandbox, config) => {
   const screenshot = async (options) => {
-    let { base64 } = await sandbox.send({ type: "system.screenshot" });
+    let { base64 } = await sandbox.send({
+      os: "linux",
+      type: "system.screenshot",
+    });
 
     if (!base64) {
       console.error("Failed to take screenshot");
@@ -110,6 +113,7 @@ const createSystem = (emitter, sandbox, config) => {
   const activeWin = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
+      os: "linux",
       type: "system.get-active-window",
     });
 
@@ -119,6 +123,7 @@ const createSystem = (emitter, sandbox, config) => {
   const getMousePosition = async () => {
     // Get Mouse Position from command line
     let result = await sandbox.send({
+      os: "linux",
       type: "system.get-mouse-position",
     });