testdriverai 5.1.1 → 5.2.0

package/agent.js

@@ -31,7 +31,7 @@ const sanitizeFilename = require("sanitize-filename");
 const macScreenPerms = require("mac-screen-capture-permissions");
 
 // local modules
-const wss = require("./lib/websockets.js");
+const websocketserver = require("./lib/websockets.js");
 const speak = require("./lib/speak.js");
 const analytics = require("./lib/analytics.js");
 const log = require("./lib/logger.js");
@@ -63,6 +63,7 @@ let checkCount = 0;
 let checkLimit = 7;
 let lastScreenshot = null;
 let rl;
+let wss; 
 
 // list of prompts that the user has given us
 let tasks = [];
@@ -837,7 +838,7 @@ const firstPrompt = async () => {
 
   rl.on("line", handleInput);
 
-  wss.addEventListener("input", async (message) => {
+  config.TD_VM && wss.addEventListener("input", async (message) => {
     handleInput(message.data);
   });
 
@@ -1081,7 +1082,7 @@ ${yaml.dump(step)}
 };
 
 const promptUser = () => {
-  wss.sendToClients("done");
+  config.TD_VM && wss.sendToClients("done");
   emitter.emit(events.interactive, true);
   rl.prompt(true);
 };
@@ -1144,6 +1145,9 @@ const embed = async (file, depth) => {
 
 const buildEnv = async () => {
   let win = await system.activeWin();
+  if (config.TD_VM) {
+    wss = websocketserver.create();
+  }
   setTerminalApp(win);
   await ensureMacScreenPerms();
   await makeSandbox();

package/electron/overlay.html

@@ -93,9 +93,9 @@
     }
 
     .box {
-      border: 2px solid #B0CF34;
+      border: 1px solid #B0CF34;
       position: absolute;
-
+      border-radius: 5px;
       animation-duration: 5s;
       animation-delay: 0s;
       animation-timing-function: cubic-bezier(0.26, 0.53, 0.74, 1.48);
@@ -111,6 +111,29 @@
       position: absolute;
     }
 
+    #mouse {
+      margin-left: -100px;
+      margin-top: -100px;
+      width: 50px;
+      height: 50px;
+      opacity: 50%;
+      position: absolute;
+      transform: translate(-50%, -50%);
+      border-radius: 70%;
+      background: #FFD700;
+    }
+
+    #mouse #dot {
+      width: 7px;
+      height: 7px;
+      position: absolute;
+      top: 50%;
+      left: 50%;
+      transform: translate(-50%, -50%);
+      border-radius: 50%;
+      background-color: black;
+    }
+
     #pointer {
       width: 25px;
       height: 25px;
@@ -213,6 +236,7 @@
   <div id="main" class="container">
     <div id="boxes">
       <div id="pointer"></div>
+      <div id="mouse"><div id="dot"></div></div>
     </div>
     <div id="terminal-wrapper">
       <img src="td.png" alt="td" style="position: absolute; top: 20; right: 20; height: 40px; z-index: 9999; background-color: black;">
@@ -226,6 +250,7 @@
     const { ipcRenderer } = require("electron");
     const { events } = require("../lib/events.js");
 
+    const mouse = document.querySelector("#mouse");
     const pointer = document.querySelector("#pointer");
     const container = document.querySelector("#main");
     const screenshotElement = document.querySelector("#screenshot");
@@ -277,9 +302,20 @@
         container.style.opacity = 1
       }, 2000)
     });
+    ipcRenderer.on(events.mouseMove,
+      (event, { x, y } = {}) => {
+        
+        console.log("mouseMove", x, y)
+
+        mouse.style.marginLeft = toCss(x);
+        mouse.style.marginTop = toCss(y);
+
+      },
+    );
 
     ipcRenderer.on(events.mouseClick,
       (event, { x, y, click = "single" } = {}) => {
+
         pointer.style.marginLeft = toCss(x);
         pointer.style.marginTop = toCss(y);
         pointer.setAttribute("class", "");

package/electron/overlay.js

@@ -75,7 +75,7 @@ app.whenReady().then(() => {
   });
 
   // open developer tools
-  // window.webContents.openDevTools();
+  window.webContents.openDevTools();
 
   ipc.serve(() => {
     for (const event of eventsArray) {

package/lib/commands.js

@@ -1,6 +1,5 @@
 // the actual commands to interact with the system
 const sdk = require("./sdk");
-const vm = require('vm');
 const chalk = require("chalk");
 const {
   captureScreenBase64,
@@ -8,9 +7,10 @@ const {
   platform,
   activeWin,
 } = require("./system");
-
+const keymap = require("./keymap");
 const { focusApplication } = require("./focus-application");
 const fs = require("fs").promises; // Using the promises version for async operations
+const robot = require("robotjs");
 const { findTemplateImage } = require("./subimage/index");
 const { cwd } = require("node:process");
 const path = require("path");
@@ -18,24 +18,14 @@ const Jimp = require("jimp");
 const os = require("os");
 const cliProgress = require("cli-progress");
 const redraw = require("./redraw");
-const sandbox = require("./sandbox.js");
-const config = require("./config.js");
-let robot;
-
-let keymap;
-if (config.TD_VM) {
-  keymap = require("./keymaps/sandbox.js");
-} else {
-  robot = require("robotjs");
-  keymap = require("./keymaps/robot.js");
-}
-
 const {
   logger,
   prettyMarkdown,
   createMarkdownStreamLogger,
 } = require("./logger");
 const { emitter, events } = require("./events.js");
+const util = require('util');
+const exec = util.promisify(require('child_process').exec);
 
 const niceSeconds = (ms) => {
   return Math.round(ms / 1000);
@@ -187,26 +177,26 @@ const scroll = async (direction = "down", amount = 300, method = "mouse") => {
   switch (direction) {
     case "up":
       if (method === "mouse") {
-        config.TD_VM ? await sandbox.send({type: "scroll", amount }) : await robot.scrollMouse(0, amount );
+        await robot.scrollMouse(0, amount );
       } else {
-        config.TD_VM ? await sandbox.send({type: "press", keys: ["pageup"]}) : await robot.keyTap("pageup");
+        await robot.keyTap("pageup");
       }
       await redraw.wait(2500);
       break;
     case "down":
       if (method === "mouse") {
-        config.TD_VM ? await sandbox.send({type: "scroll", amount: amount * -1}) : await robot.scrollMouse(0, amount * -1);
+        await robot.scrollMouse(0, amount * -1);
       } else {
-        config.TD_VM ? await sandbox.send({type: "press", keys: ["pagedown"] }) : await robot.keyTap("pagedown"); 
+        await robot.keyTap("pagedown");
       }
       await redraw.wait(2500);
       break;
     case "left":
-      config.TD_VM ? console.log('Not Supported') : await robot.scrollMouse(amount * -1, 0);
+      await robot.scrollMouse(amount * -1, 0);
       await redraw.wait(2500);
       break;
     case "right":
-      config.TD_VM ? console.log('Not Supported') : await robot.scrollMouse(amount, 0);
+      await robot.scrollMouse(amount, 0);
       await redraw.wait(2500);
       break;
     default:
@@ -229,20 +219,22 @@ const click = async (x, y, action = "click") => {
     double = true;
   }
 
-  logger.debug(chalk.dim(`${click} ${button} clicking at ${x}, ${y}...`), true);
+  logger.debug(chalk.dim(`${action} ${button} clicking at ${x}, ${y}...`), true);
 
   x = parseInt(x);
   y = parseInt(y);
 
-  config.TD_VM ? await sandbox.send({type: "moveMouse", x, y }) : await robot.moveMouseSmooth(x, y, 0.1);
+  robot.moveMouseSmooth(x, y, 0.1);
+  
   await delay(1000); // wait for the mouse to move
 
-  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
+  if (process.platform === "darwin" && action === "right-click") {
     robot.keyToggle('control', 'down', 'control');
     await delay(250);
   }
 
   if (action !== "hover") {
+    
     if (config.TD_VM) {
       if (action === "click" || action === "left-click") {
         await sandbox.send({type: "leftClick" })
@@ -252,14 +244,28 @@ const click = async (x, y, action = "click") => {
         await sandbox.send({type: "middleClick" })
       } else if (action === "double-click") {
         await sandbox.send({type: "doubleClick" })
+      }else if (action === "drag-start") {
+        await sandbox.send({type: "mousePressLeft" })
+      }else if (action === "drag-end") {
+        await sandbox.send({type: "doubleClick" })
       }
     } else {
-      robot.mouseClick(button, double);
+
+      if (action === "drag-start") {
+        robot.mouseToggle("down", button);
+      } else if( action === "drag-end") {
+        robot.mouseToggle("up", button);
+      } else {
+        robot.mouseClick(button, double);
+      }
+
     }
+
     emitter.emit(events.mouseClick, { x, y, button, click });
+
   }
 
-  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
+  if (process.platform === "darwin" && action === "right-click") {
     await delay(250);
     robot.keyToggle('control', 'up', 'control');
   }
@@ -274,7 +280,7 @@ const hover = async (x, y) => {
   x = parseInt(x);
   y = parseInt(y);
 
-  await sandbox.send({type: "moveMouse", x, y });
+  await robot.moveMouseSmooth(x, y, 0.1);
 
   await redraw.wait(2500);
 
@@ -385,13 +391,7 @@ let commands = {
   // type a string
   type: async (string, delay = 500) => {
     await redraw.start();
-    
     string = string.toString();
-
-    if (config.TD_VM) {
-      await sandbox.send({type: "write", text: string });
-    } else {
-
     // there is a bug in robotjs that causes repeated characters to only be typed once
     // so we need to check for repeated characters and type them slowly if so
     const hasRepeatedChars = /(.)\1/.test(string);
@@ -399,8 +399,7 @@ let commands = {
       await robot.typeStringDelayed(string, delay);
     else
       await robot.typeString(string);
-    }
-    // await redraw.wait(5000);
+    await redraw.wait(5000);
     return;
   },
   // press keys
@@ -423,7 +422,7 @@ let commands = {
         key = "control";
       }
 
-      if (!config.TD_VM && modifierKeys.includes(key)) {
+      if (modifierKeys.includes(key)) {
         modifierKeysPressed.push(key);
       } else {
         keysPressed.push(key);
@@ -439,40 +438,32 @@ let commands = {
     });
 
     // only one key can be pressed at a time
-    if (!config.TD_VM && keysPressed.length > 1) {
+    if (keysPressed.length > 1) {
       throw new AiError(
         "Only one key can be pressed at a time. However, multiple modifier keys can be pressed at the same time.",
       );
     }
 
     // make sure modifier keys are valid, multiple are allowed
-    let modsToPress = [];
-    
-    if (!config.TD_VM) {
-      modifierKeysPressed.map((key) => {
-        if (keymap[key] === undefined) {
-          logger.error(`Modifier key not found: ${key}`);
-          throw new AiError(`Modifier key not found: ${key}`);
-        } else {
-          return keymap[key];
-        }
-      }); 
-      robot.keyTap(keysPressed[0], modsToPress);
-    } else {
-      await sandbox.send({type: "press", keys: keysPressed });
-    }
+    let modsToPress = modifierKeysPressed.map((key) => {
+      if (keymap[key] === undefined) {
+        logger.error(`Modifier key not found: ${key}`);
+        throw new AiError(`Modifier key not found: ${key}`);
+      } else {
+        return keymap[key];
+      }
+    });
 
     // finally, press the keys
+    robot.keyTap(keysPressed[0], modsToPress);
 
     await redraw.wait(5000);
 
     // keyTap will release the normal keys, but will not release modifier keys
     // so we need to release the modifier keys manually
-    if (!config.TD_VM) {
-      modsToPress.forEach((key) => {
-        robot.keyToggle(key, "up");
-      });
-    }
+    modsToPress.forEach((key) => {
+      robot.keyToggle(key, "up");
+    });
 
     return;
   },
@@ -582,18 +573,15 @@ let commands = {
 
     if (method === "keyboard") {
       try {
-
-        if (!config.TD_VM) {
-          await robot.keyTap("f", commandOrControl);;
-          robot.keyToggle(commandOrControl, "up");
-          await robot.typeStringDelayed(text, 500);
-        } else {
-          await sandbox.send({type: "press", keys: ["f", commandOrControl] })
-          await sandbox.send({type: "write", text });
-          await redraw.wait(5000);
-          await sandbox.send({type: "press", keys: ["escape"] });
-        }
-
+        // use robot to press CMD+F
+        await robot.keyTap("f", commandOrControl);
+        // keyTap will release the normal keys, but will not release modifier keys
+        // so we need to release the modifier keys manually
+        robot.keyToggle(commandOrControl, "up");
+        // type the text
+        await robot.typeStringDelayed(text, 500);
+        await redraw.wait(5000);
+        await robot.keyTap("escape");
       } catch (e) {
         logger.error("%s", e);
         throw new AiError(
@@ -703,26 +691,25 @@ let commands = {
   assert: async (assertion, async = false) => {
     return await assert(assertion, true, async);
   },
-  exec: async (nodejs_code) => {
+  exec: async (cli_command, use_stderr = false, silent = false) => {
 
-    // must be assigned to `result`
-    // do not overwrite `result`
-    // must install locally via npm install
+    let args = {};
+    if (silent) {
+      args = { stdio: [] };
+    }
 
-    const context = vm.createContext({ require, console, fs, process });
-    const script = new vm.Script(`
-      (async () => {
-        ${nodejs_code}
-      })();
-    `);
+    const { stdout, stderr } = await exec(cli_command, args);
 
-    await script.runInContext(context);
+    if (!silent) {
+      logger.info(chalk.dim(stdout), true);
+    }
 
-    // wait for context.result to resolve
-    const stepResult = await context.result;
+    if (use_stderr) {
+      return stderr;
+    } else {
+      return stdout;
+    }
 
-    return stepResult;
-    
   }
 };
 

package/lib/events.js

@@ -5,6 +5,7 @@ const emitter = new EventEmitter();
 const events = {
   showWindow: "show-window",
   mouseClick: "mouse-click",
+  mouseMove: "mouse-move",
   screenCapture: {
     start: "screen-capture:start",
     end: "screen-capture:end",

package/lib/init.js

@@ -78,7 +78,7 @@ module.exports = async () => {
     {
       type: "confirm",
       name: "TD_VM",
-      message: "Use Testdriver Sandbox? (Recommended)",
+      message: "Use TestDriver Sandbox Runners? (Recommended)",
       initial: true,
     },
     {

package/lib/logger.js

@@ -10,6 +10,7 @@ const shouldLog =
 // responsible for rendering ai markdown output
 const { marked } = require("marked");
 const { markedTerminal } = require("marked-terminal");
+const { config } = require("dotenv");
 
 const { printf } = winston.format;
 
@@ -84,7 +85,9 @@ const createMarkdownStreamLogger = () => {
         return;
       }        
       
-      websockets.sendToClients("output", chunk);
+      if (config.TD_VM) {
+        websockets.sendToClients("output", chunk);
+      }
 
       const previousConsoleOutput = markedParsePartial(buffer, 0, -1);
 

package/lib/overlay.js

@@ -29,7 +29,6 @@ try {
   logger.error("Failed to locate Electron CLI or start process:", error);
 }
 
-
 module.exports.electronProcessPromise = new Promise((resolve) => {
   ipc.connectTo("testdriverai_overlay");
   ipc.of.testdriverai_overlay.on("connect", () => {

package/lib/speak.js

@@ -6,7 +6,6 @@ const websockets = require("./websockets");
 module.exports = (message) => {
   if (config["TD_SPEAK"]) {
     say.stop();
-    // websockets.sendToClients("output", message);  
     if (process.platform === "darwin") {
       say.speak(message, "Fred", 1.2);
     } else {

package/lib/websockets.js

@@ -1,7 +1,10 @@
 const WebSocket = require('ws');
+const config = require('./config');
 
+let instance;
 class WebSocketServerSingleton {
   constructor() {
+
     if (!WebSocketServerSingleton.instance) {
       this.wss = new WebSocket.Server({ port: 8080 }, () => {
       });
@@ -72,7 +75,11 @@ class WebSocketServerSingleton {
   }
 }
 
-const instance = new WebSocketServerSingleton();
-Object.freeze(instance);
 
-module.exports = instance;
+module.exports = {
+  create: () => {
+    instance = new WebSocketServerSingleton();
+    Object.freeze(instance);  
+    return instance;
+  }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "5.1.1",
+  "version": "5.2.0",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {