testdriverai 5.2.0 → 5.2.2

package/electron/overlay.js

@@ -75,7 +75,7 @@ app.whenReady().then(() => {
   });
 
   // open developer tools
-  window.webContents.openDevTools();
+  // window.webContents.openDevTools();
 
   ipc.serve(() => {
     for (const event of eventsArray) {

package/lib/commands.js

@@ -1,5 +1,6 @@
 // the actual commands to interact with the system
 const sdk = require("./sdk");
+const vm = require('vm');
 const chalk = require("chalk");
 const {
   captureScreenBase64,
@@ -7,10 +8,9 @@ const {
   platform,
   activeWin,
 } = require("./system");
-const keymap = require("./keymap");
+
 const { focusApplication } = require("./focus-application");
 const fs = require("fs").promises; // Using the promises version for async operations
-const robot = require("robotjs");
 const { findTemplateImage } = require("./subimage/index");
 const { cwd } = require("node:process");
 const path = require("path");
@@ -18,14 +18,24 @@ const Jimp = require("jimp");
 const os = require("os");
 const cliProgress = require("cli-progress");
 const redraw = require("./redraw");
+const sandbox = require("./sandbox.js");
+const config = require("./config.js");
+let robot;
+
+let keymap;
+if (config.TD_VM) {
+  keymap = require("./keymaps/sandbox.js");
+} else {
+  robot = require("robotjs");
+  keymap = require("./keymaps/robot.js");
+}
+
 const {
   logger,
   prettyMarkdown,
   createMarkdownStreamLogger,
 } = require("./logger");
 const { emitter, events } = require("./events.js");
-const util = require('util');
-const exec = util.promisify(require('child_process').exec);
 
 const niceSeconds = (ms) => {
   return Math.round(ms / 1000);
@@ -177,26 +187,26 @@ const scroll = async (direction = "down", amount = 300, method = "mouse") => {
   switch (direction) {
     case "up":
       if (method === "mouse") {
-        await robot.scrollMouse(0, amount );
+        config.TD_VM ? await sandbox.send({type: "scroll", amount }) : await robot.scrollMouse(0, amount );
       } else {
-        await robot.keyTap("pageup");
+        config.TD_VM ? await sandbox.send({type: "press", keys: ["pageup"]}) : await robot.keyTap("pageup");
       }
       await redraw.wait(2500);
       break;
     case "down":
       if (method === "mouse") {
-        await robot.scrollMouse(0, amount * -1);
+        config.TD_VM ? await sandbox.send({type: "scroll", amount: amount * -1}) : await robot.scrollMouse(0, amount * -1);
       } else {
-        await robot.keyTap("pagedown");
+        config.TD_VM ? await sandbox.send({type: "press", keys: ["pagedown"] }) : await robot.keyTap("pagedown"); 
       }
       await redraw.wait(2500);
       break;
     case "left":
-      await robot.scrollMouse(amount * -1, 0);
+      config.TD_VM ? console.log('Not Supported') : await robot.scrollMouse(amount * -1, 0);
       await redraw.wait(2500);
       break;
     case "right":
-      await robot.scrollMouse(amount, 0);
+      config.TD_VM ? console.log('Not Supported') : await robot.scrollMouse(amount, 0);
       await redraw.wait(2500);
       break;
     default:
@@ -219,22 +229,22 @@ const click = async (x, y, action = "click") => {
     double = true;
   }
 
-  logger.debug(chalk.dim(`${action} ${button} clicking at ${x}, ${y}...`), true);
+  logger.debug(chalk.dim(`${click} ${button} clicking at ${x}, ${y}...`), true);
 
   x = parseInt(x);
   y = parseInt(y);
 
-  robot.moveMouseSmooth(x, y, 0.1);
-  
+  config.TD_VM ? await sandbox.send({type: "moveMouse", x, y }) : await robot.moveMouseSmooth(x, y, 0.1);
+  emitter.emit(events.mouseMove, { x, y });
+
   await delay(1000); // wait for the mouse to move
 
-  if (process.platform === "darwin" && action === "right-click") {
+  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
     robot.keyToggle('control', 'down', 'control');
     await delay(250);
   }
 
   if (action !== "hover") {
-    
     if (config.TD_VM) {
       if (action === "click" || action === "left-click") {
         await sandbox.send({type: "leftClick" })
@@ -245,9 +255,9 @@ const click = async (x, y, action = "click") => {
       } else if (action === "double-click") {
         await sandbox.send({type: "doubleClick" })
       }else if (action === "drag-start") {
-        await sandbox.send({type: "mousePressLeft" })
+        await sandbox.send({type: "mousePress", button: "left" })
       }else if (action === "drag-end") {
-        await sandbox.send({type: "doubleClick" })
+        await sandbox.send({type: "mouseRelease", button: "left" })
       }
     } else {
 
@@ -262,10 +272,9 @@ const click = async (x, y, action = "click") => {
     }
 
     emitter.emit(events.mouseClick, { x, y, button, click });
-
   }
 
-  if (process.platform === "darwin" && action === "right-click") {
+  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
     await delay(250);
     robot.keyToggle('control', 'up', 'control');
   }
@@ -280,7 +289,7 @@ const hover = async (x, y) => {
   x = parseInt(x);
   y = parseInt(y);
 
-  await robot.moveMouseSmooth(x, y, 0.1);
+  await sandbox.send({type: "moveMouse", x, y });
 
   await redraw.wait(2500);
 
@@ -391,7 +400,13 @@ let commands = {
   // type a string
   type: async (string, delay = 500) => {
     await redraw.start();
+    
     string = string.toString();
+
+    if (config.TD_VM) {
+      await sandbox.send({type: "write", text: string });
+    } else {
+
     // there is a bug in robotjs that causes repeated characters to only be typed once
     // so we need to check for repeated characters and type them slowly if so
     const hasRepeatedChars = /(.)\1/.test(string);
@@ -399,7 +414,8 @@ let commands = {
       await robot.typeStringDelayed(string, delay);
     else
       await robot.typeString(string);
-    await redraw.wait(5000);
+    }
+    // await redraw.wait(5000);
     return;
   },
   // press keys
@@ -422,7 +438,7 @@ let commands = {
         key = "control";
       }
 
-      if (modifierKeys.includes(key)) {
+      if (!config.TD_VM && modifierKeys.includes(key)) {
         modifierKeysPressed.push(key);
       } else {
         keysPressed.push(key);
@@ -438,32 +454,40 @@ let commands = {
     });
 
     // only one key can be pressed at a time
-    if (keysPressed.length > 1) {
+    if (!config.TD_VM && keysPressed.length > 1) {
       throw new AiError(
         "Only one key can be pressed at a time. However, multiple modifier keys can be pressed at the same time.",
       );
     }
 
     // make sure modifier keys are valid, multiple are allowed
-    let modsToPress = modifierKeysPressed.map((key) => {
-      if (keymap[key] === undefined) {
-        logger.error(`Modifier key not found: ${key}`);
-        throw new AiError(`Modifier key not found: ${key}`);
-      } else {
-        return keymap[key];
-      }
-    });
+    let modsToPress = [];
+    
+    if (!config.TD_VM) {
+      modifierKeysPressed.map((key) => {
+        if (keymap[key] === undefined) {
+          logger.error(`Modifier key not found: ${key}`);
+          throw new AiError(`Modifier key not found: ${key}`);
+        } else {
+          return keymap[key];
+        }
+      }); 
+      robot.keyTap(keysPressed[0], modsToPress);
+    } else {
+      await sandbox.send({type: "press", keys: keysPressed });
+    }
 
     // finally, press the keys
-    robot.keyTap(keysPressed[0], modsToPress);
 
     await redraw.wait(5000);
 
     // keyTap will release the normal keys, but will not release modifier keys
     // so we need to release the modifier keys manually
-    modsToPress.forEach((key) => {
-      robot.keyToggle(key, "up");
-    });
+    if (!config.TD_VM) {
+      modsToPress.forEach((key) => {
+        robot.keyToggle(key, "up");
+      });
+    }
 
     return;
   },
@@ -573,15 +597,18 @@ let commands = {
 
     if (method === "keyboard") {
       try {
-        // use robot to press CMD+F
-        await robot.keyTap("f", commandOrControl);
-        // keyTap will release the normal keys, but will not release modifier keys
-        // so we need to release the modifier keys manually
-        robot.keyToggle(commandOrControl, "up");
-        // type the text
-        await robot.typeStringDelayed(text, 500);
-        await redraw.wait(5000);
-        await robot.keyTap("escape");
+
+        if (!config.TD_VM) {
+          await robot.keyTap("f", commandOrControl);;
+          robot.keyToggle(commandOrControl, "up");
+          await robot.typeStringDelayed(text, 500);
+        } else {
+          await sandbox.send({type: "press", keys: ["f", commandOrControl] })
+          await sandbox.send({type: "write", text });
+          await redraw.wait(5000);
+          await sandbox.send({type: "press", keys: ["escape"] });
+        }
+
       } catch (e) {
         logger.error("%s", e);
         throw new AiError(
@@ -691,25 +718,26 @@ let commands = {
   assert: async (assertion, async = false) => {
     return await assert(assertion, true, async);
   },
-  exec: async (cli_command, use_stderr = false, silent = false) => {
+  exec: async (nodejs_code) => {
 
-    let args = {};
-    if (silent) {
-      args = { stdio: [] };
-    }
+    // must be assigned to `result`
+    // do not overwrite `result`
+    // must install locally via npm install
 
-    const { stdout, stderr } = await exec(cli_command, args);
+    const context = vm.createContext({ require, console, fs, process });
+    const script = new vm.Script(`
+      (async () => {
+        ${nodejs_code}
+      })();
+    `);
 
-    if (!silent) {
-      logger.info(chalk.dim(stdout), true);
-    }
+    await script.runInContext(context);
 
-    if (use_stderr) {
-      return stderr;
-    } else {
-      return stdout;
-    }
+    // wait for context.result to resolve
+    const stepResult = await context.result;
 
+    return stepResult;
+    
   }
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "5.2.0",
+  "version": "5.2.2",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {