testdriverai 4.1.26 → 4.1.28

package/agent.js

@@ -215,11 +215,21 @@ const exit = async (failed = true) => {
   });
 };
 
+const dieOnFatal = async (error) => {
+  console.log("");
+  log.log("info", chalk.red("Fatal Error") + `\n${error.message}`);
+  await summarize(error.message);
+  return await exit(true);
+}
+
 // creates a new "thread" in which the AI is given an error
 // and responds. notice `actOnMarkdown` which will continue
 // the thread until there are no more codeblocks to execute
 const haveAIResolveError = async (error, markdown, depth = 0, undo = true) => {
-  
+
+  if (thisCommand == "run" || error.fatal) {
+    return await dieOnFatal(error);
+  }
 
   let eMessage = error.message ? error.message : error;
 
@@ -350,19 +360,12 @@ const runCommand = async (command, depth) => {
       return await actOnMarkdown(response, depth);
     }
   } catch (error) {
-    if (thisCommand == "run") {
-      console.log("");
-      log.log("info", chalk.red("Fatal Error") + `\n${error.message}`);
-      await summarize(error.message);
-      return await exit(true);
-    } else {
-      return await haveAIResolveError(
-        error,
-        yaml.dump({ commands: [yml] }),
-        depth,
-        true,
-      );
-    }
+    return await haveAIResolveError(
+      error,
+      yaml.dump({ commands: [yml] }),
+      depth,
+      true,
+    );
   }
 };
 

package/lib/commander.js

@@ -12,7 +12,6 @@ const sdk = require("./sdk");
 // object is a json representation of the individual yml command
 // the process turns markdown -> yml -> json -> js function execution
 const run = async (object, depth) => {
-
   log("debug", { object, depth });
 
   // success returns null
@@ -63,7 +62,11 @@ commands:
         speak(`scrolling ${object.direction}`);
         log("info", generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
-        response = await commands.scroll(object.direction, object.amount);
+        response = await commands.scroll(
+          object.direction,
+          object.amount,
+          object.method,
+        );
         break;
       case "wait":
         speak(`waiting ${object.timeout} seconds`);
@@ -143,6 +146,7 @@ commands:
           object.text,
           object.direction,
           object.distance,
+          object.textMatchMethod,
           object.method,
         );
         break;
@@ -154,6 +158,7 @@ commands:
           object.description,
           object.direction,
           object.distance,
+          object.method,
         );
         break;
       case "focus-application":
@@ -179,10 +184,10 @@ commands:
   }
 
   let timing = marky.stop(object.command);
-  
+
   await Promise.all([
-    sdk.req('ran', { command: object.command, data: object }),
-    analytics.track("command", { data: object, depth, timing })
+    sdk.req("ran", { command: object.command, data: object }),
+    analytics.track("command", { data: object, depth, timing }),
   ]);
 
   return response;

package/lib/commands.js

@@ -129,7 +129,7 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
       return true;
     } else {
       if (shouldThrow) {
-        throw new AiError(`AI Assertion failed`);
+        throw new AiError(`AI Assertion failed`, true);
       } else {
         return false;
       }
@@ -160,18 +160,33 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
     return handleAssertResponse(response.data);
   }
 };
-const scroll = async (direction = "down", amount = 300) => {
+const scroll = async (direction = "down", amount = 300, method = "keyboard") => {
   await redraw.start();
 
   amount = parseInt(amount);
 
+  if (method === "mouse") {
+    // after experimenting, 200 is a good default for mouse, mostly as mouse will be called only when the user asks for it
+    // and that happens when keyboard scrolling cannot do things when there's a pop up that needs to be scrolled over and 
+    // pop ups are usually smaller and needs a smaller amount of scrolling
+    amount = 200;
+  }
+
   switch (direction) {
     case "up":
-      await robot.keyTap("pageup");
+      if (method === "mouse") {
+        await robot.scrollMouse(0, amount );
+      } else {
+        await robot.keyTap("pageup");
+      }
       await redraw.wait(2500);
       break;
     case "down":
-      await robot.keyTap("pagedown");
+      if (method === "mouse") {
+        await robot.scrollMouse(0, amount * -1);
+      } else {
+        await robot.keyTap("pagedown");
+      }
       await redraw.wait(2500);
       break;
     case "left":
@@ -321,7 +336,7 @@ let commands = {
     mdStream.end();
 
     if (!response?.data) {
-      throw new AiError("No text on screen matches description", true);
+      throw new AiError("No image or icon on screen matches description");
     } else {
       return response.data;
     }
@@ -335,7 +350,7 @@ let commands = {
     let result = await findImageOnScreen(relativePath, image);
 
     if (!result) {
-      throw new AiError(`Image not found: ${relativePath}`);
+      throw new AiError(`Image not found: ${relativePath}`, true);
     } else {
       if (action === "click") {
         await click(result.centerX, result.centerY, action);
@@ -521,29 +536,38 @@ let commands = {
     text,
     direction = "down",
     maxDistance = 1200,
-    method = "turbo",
+    textMatchMethod = "turbo",
+    method = "keyboard"
   ) => {
     await redraw.start();
 
     log("info", chalk.dim(`scrolling for text: "${text}"...`), true);
 
-    try {
-      // use robot to press CMD+F
-      await robot.keyTap("f", commandOrControl);
-      // type the text
-      await robot.typeString(text);
-      await redraw.wait(5000);
-      await robot.keyTap("escape");
-    } catch (e) {
-      console.log(e);
-      throw new AiError(
-        "Could not find element using browser text search",
-        true,
-      );
+    if (method === "keyboard") {
+      try {
+        // use robot to press CMD+F
+        await robot.keyTap("f", commandOrControl);
+        // keyTap will release the normal keys, but will not release modifier keys
+        // so we need to release the modifier keys manually
+        robot.keyToggle(commandOrControl, "up");
+        // type the text
+        await robot.typeString(text);
+        await redraw.wait(5000);
+        await robot.keyTap("escape");
+      } catch (e) {
+        console.log(e);
+        throw new AiError(
+          "Could not find element using browser text search",
+          true,
+        );
+      }
     }
 
     let scrollDistance = 0;
     let incrementDistance = 300;
+    if (method === "mouse") {
+      incrementDistance = 200;
+    }
     let passed = false;
 
     while (scrollDistance <= maxDistance && !passed) {
@@ -551,7 +575,7 @@ let commands = {
         "assert/text",
         {
           needle: text,
-          method: method,
+          method: textMatchMethod,
           image: await captureScreenBase64(),
         },
         (chunk) => {
@@ -570,7 +594,7 @@ let commands = {
           ),
           true,
         );
-        await scroll(direction);
+        await scroll(direction, incrementDistance, method);
         scrollDistance = scrollDistance + incrementDistance;
       }
     }
@@ -589,6 +613,7 @@ let commands = {
     description,
     direction = "down",
     maxDistance = 10000,
+    method = "keyboard",
   ) => {
     log(
       "info",
@@ -613,7 +638,7 @@ let commands = {
           chalk.dim(`scrolling ${direction} ${incrementDistance} pixels...`),
           true,
         );
-        await scroll(direction);
+        await scroll(direction, incrementDistance, method);
         scrollDistance = scrollDistance + incrementDistance;
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.1.26",
+  "version": "4.1.28",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {