testdriverai 5.2.2 → 5.3.1

package/electron/overlay.js

@@ -1,25 +1,56 @@
-const ipc = require("node-ipc").default;
-const { app, screen, BrowserWindow } = require("electron");
+const { default: nodeIPC } = require("node-ipc");
+const { app: electronApp, remote, screen, BrowserWindow, Tray, Menu } = require("electron");
 const { eventsArray } = require("../lib/events.js");
 const config = require("../lib/config.js");
+const path = require('path');
 
-ipc.config.id = "testdriverai_overlay";
-ipc.config.retry = 1500;
+let tray = null;
+
+const app = electronApp || remote;
+if (!app) {
+  console.log("No app");
+  process.exit(1);
+}
+
+// Seems like the direct process id is not the electron process id
+// so we use the parent process id
+const rendererId = process.env.TD_OVERLAY_ID ?? process.ppid;
+
+const ipc = new nodeIPC.IPC();
+ipc.config.id = `testdriverai_overlay_${rendererId}`;
+ipc.config.retry = 0;
 ipc.config.silent = true;
 
 app.whenReady().then(() => {
+
+  // Path to tray icon (must be .ico on Windows, .png on Mac/Linux)
+  const iconPath = path.join(__dirname, 'tray.png');
+
+  tray = new Tray(iconPath);
+
+  const contextMenu = Menu.buildFromTemplate([
+    {
+      label: 'Quit',
+      click: () => {
+        app.quit();
+      },
+    },
+  ]);
+
+  tray.setToolTip('TestDriver.ai');
+  tray.setContextMenu(contextMenu);
+
   app.dock?.hide();
 
   let windowOptions;
 
   if (config.TD_VM) {
-
     windowOptions = {
-      width: 1024,
-      height: 768,
+      width: config.TD_VM_RESOLUTION[0],
+      height: config.TD_VM_RESOLUTION[1],
       closable: true,
       resizable: true,
-      alwaysOnTop: true,
+      // alwaysOnTop: true,
       show: false,
       webPreferences: {
         nodeIntegration: true,
@@ -27,9 +58,7 @@ app.whenReady().then(() => {
       },
       autoHideMenuBar: true,
     };
-
   } else {
-
     windowOptions = {
       ...screen.getPrimaryDisplay().bounds,
       closable: true,
@@ -49,10 +78,9 @@ app.whenReady().then(() => {
       autoHideMenuBar: true,
     };
 
-    if (process.platform !== 'darwin') {
+    if (process.platform !== "darwin") {
       windowOptions.fullscreen = true;
     }
-
   }
 
   const window = new BrowserWindow(windowOptions);
@@ -64,13 +92,13 @@ app.whenReady().then(() => {
       visibleOnFullScreen: true,
     });
   } else {
-    window.setContentSize(1024, 768);
+    window.setContentSize(config.TD_VM_RESOLUTION[0], config.TD_VM_RESOLUTION[1]);
     window.setBackgroundColor('#000')
   }
-  
+
   window.loadFile("overlay.html");
 
-  window.once('ready-to-show', () => {
+  window.once("ready-to-show", () => {
     // window.showInactive();
   });
 
@@ -78,6 +106,7 @@ app.whenReady().then(() => {
   // window.webContents.openDevTools();
 
   ipc.serve(() => {
+    console.log("Serving IPC");
     for (const event of eventsArray) {
       ipc.server.on(event, (data) => {
         if (event === "show-window") {
@@ -88,8 +117,39 @@ app.whenReady().then(() => {
       });
     }
   });
+
+  // We do this because node-ipc doesn't prevent new servers from using the same id
+  // so we need to timeout if no clients connect after 5 minutes to avoid keeping older
+  // overlay processes alive
+  const timeout = setTimeout(
+    () => {
+      console.log("No connected clients for 5 minutes");
+      process.exit(0);
+    },
+    1000 * 60 * 5,
+  );
+
+  ipc.server.on("connect", () => {
+    console.log("Client connected");
+    clearTimeout(timeout);
+  });
+
   ipc.server.on("socket.disconnected", function () {
+    // We exit because we want the renderer process to be single use
+    // and not stay alive if the cli gets disconnected
+    console.log("Client disconnected");
     process.exit();
   });
+
+  ipc.server.on("error", () => {
+    console.log("Server error");
+    process.exit(1);
+  });
+
+  ipc.server.on("destroy", () => {
+    console.log("Server destroyed");
+    process.exit(1);
+  });
+
   ipc.server.start();
 });

package/index.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 const config = require("./lib/config.js");
-const system = require("./lib/system.js");
+// We need to initialize the IPC server quickly
+require("./lib/ipc.js");
 const { emitter, events } = require("./lib/events.js");
 const { logger } = require("./lib/logger.js");
 
@@ -8,42 +9,82 @@ if (process.argv[2] === "--help" || process.argv[2] === "-h") {
   console.log("Command: testdriverai [init, run, edit] [yaml filepath]");
   process.exit(0);
 }
+if (process.argv[2] === "--renderer") {
+  const {
+    // connectToOverlay,
+    createOverlayProcess,
+  } = require("./lib/overlay.js");
+  // const id = config.TD_OVERLAY_ID;
+  const id = process.argv[3] ?? config.TD_OVERLAY_ID;
+  if (!id) {
+    logger.error("Renderer ID is not set");
+    process.exit(1);
+  }
+  (async () => {
+    try {
+      if (!id) {
+        throw new Error("Renderer ID is not set");
+      }
+      const electronProcess = await createOverlayProcess({
+        id,
+        detached: true,
+      });
+      logger.info(`Started renderer, process ID: ${electronProcess.pid}`);
+      process.exit(0);
+    } catch (err) {
+      logger.error("%s", err);
+      process.exit(1);
+    }
+  })();
+} else {
+  (async () => {
+    if (!config.TD_OVERLAY) {
+      let agent = require("./agent.js");
+      await agent.start();
+    } else {
+      // Intercept all stdout and stderr calls (works with console as well)
+      const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+      process.stdout.write = (...args) => {
+        const [data, encoding] = args;
+        emitter.emit(
+          events.terminal.stdout,
+          data.toString(typeof encoding === "string" ? encoding : undefined),
+        );
+        originalStdoutWrite(...args);
+      };
 
-(async () => {
+      const originalStderrWrite = process.stderr.write.bind(process.stderr);
+      process.stderr.write = (...args) => {
+        const [data, encoding] = args;
+        emitter.emit(
+          events.terminal.stderr,
+          data.toString(typeof encoding === "string" ? encoding : undefined),
+        );
+        originalStderrWrite(...args);
+      };
 
-  if (!config.TD_OVERLAY) {
-    let agent = require("./agent.js");
-    agent.start();
-  } else {
-    // Intercept all stdout and stderr calls (works with console as well)
-    const originalStdoutWrite = process.stdout.write.bind(process.stdout);
-    process.stdout.write = (...args) => {
-      const [data, encoding] = args;
-      emitter.emit(
-        events.terminal.stdout,
-        data.toString(typeof encoding === "string" ? encoding : undefined),
-      );
-      originalStdoutWrite(...args);
-    };
+      const {
+        connectToOverlay,
+        createOverlayProcess,
+      } = require("./lib/overlay.js");
 
-    const originalStderrWrite = process.stderr.write.bind(process.stderr);
-    process.stderr.write = (...args) => {
-      const [data, encoding] = args;
-      emitter.emit(
-        events.terminal.stderr,
-        data.toString(typeof encoding === "string" ? encoding : undefined),
-      );
-      originalStderrWrite(...args);
-    };
+      try {
+        let id = config.TD_OVERLAY_ID;
+        if (!id) {
+          const electronProcess = await createOverlayProcess();
+          electronProcess.on("exit", (code) => {
+            logger.info(`Renderer process exited with code ${code}`);
+            process.exit(code);
+          });
+          id = electronProcess.pid;
+        }
 
-    require("./lib/overlay.js")
-      .electronProcessPromise.then(async () => {
-        let agent = require("./agent.js");
-        agent.start();
-      })
-      .catch((err) => {
+        await connectToOverlay(id);
+        await require("./agent.js").start();
+      } catch (err) {
         logger.error("%s", err);
         process.exit(1);
-      });
-  }
-})();
+      }
+    }
+  })();
+}

package/lib/commander.js

@@ -9,6 +9,7 @@ const analytics = require("./analytics");
 const marky = require("marky");
 const sdk = require("./sdk");
 const outputs = require("./outputs");
+const {server} = require("./ipc");
 
 // replace all occurances of ${OUTPUT.ls} with outputs.get("ls") in every possible property of the `object` 
 // this is a recursive function that will go through all the properties of the object
@@ -69,18 +70,21 @@ commands:
     switch (object.command) {
       case "type":
         speak(`typing ${object.text}`);
+        server.broadcast("status", `typing ${object.text}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands.type(object.text, object.delay);
         break;
       case "press-keys":
         speak(`pressing keys ${object.keys.join(",")}`);
+        server.broadcast("status", `pressing keys ${object.keys.join(",")}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["press-keys"](object.keys);
         break;
       case "scroll":
         speak(`scrolling ${object.direction}`);
+        server.broadcast("status", `scrolling ${object.direction}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands.scroll(
@@ -91,12 +95,14 @@ commands:
         break;
       case "wait":
         speak(`waiting ${object.timeout} seconds`);
+        server.broadcast("status", `waiting ${object.timeout} seconds`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands.wait(object.timeout);
         break;
       case "click":
         speak(`${object.action}`);
+        server.broadcast("status", `${object.action}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["click"](
@@ -107,12 +113,14 @@ commands:
         break;
       case "hover":
         speak(`moving mouse`);
+        server.broadcast("status", `moving mouse`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["hover"](object.x, object.y);
         break;
       case "hover-text":
         speak(`searching for ${object.description}`);
+        server.broadcast("status", `searching for ${object.description}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["hover-text"](
@@ -124,6 +132,7 @@ commands:
         break;
       case "hover-image":
         speak(`searching for image of ${object.description}`);
+        server.broadcast("status", `searching for image of ${object.description}`);
         logger.info(generator.jsonToManual(object));
         response = await commands["hover-image"](
           object.description,
@@ -132,6 +141,7 @@ commands:
         break;
       case "match-image":
         logger.info(generator.jsonToManual(object));
+        server.broadcast("status", `${object.action} image ${object.path}`);
         notify(generator.jsonToManual(object, false));
         response = await commands["match-image"](
           object.path,
@@ -140,6 +150,7 @@ commands:
         break;
       case "wait-for-image":
         speak(`waiting for ${object.description}`);
+        server.broadcast("status", `waiting for ${object.description}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["wait-for-image"](
@@ -149,6 +160,7 @@ commands:
         break;
       case "wait-for-text":
         speak(`waiting for ${object.text}`);
+        server.broadcast("status", `waiting for ${object.text}`);
         logger.info(generator.jsonToManual(object));
         copy.text = "*****";
         notify(generator.jsonToManual(copy, false));
@@ -160,6 +172,7 @@ commands:
         break;
       case "scroll-until-text":
         speak(`scrolling until ${object.text}`);
+        server.broadcast("status", `scrolling until ${object.text}`);
         logger.info(generator.jsonToManual(object));
         copy.text = "*****";
         notify(generator.jsonToManual(copy, false));
@@ -173,6 +186,7 @@ commands:
         break;
       case "scroll-until-image":
         speak(`scrolling until ${object.description}`);
+        server.broadcast("status", `scrolling until ${object.description}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["scroll-until-image"](
@@ -184,6 +198,7 @@ commands:
         break;
       case "focus-application":
         speak(`focusing ${object.name}`);
+        server.broadcast("status", `focusing ${object.name}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands["focus-application"](object.name);
@@ -195,16 +210,22 @@ commands:
         break;
       case "assert":
         speak(`asserting ${object.expect}`);
+        server.broadcast("status", `asserting ${object.expect}`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
         response = await commands.assert(object.expect, object.async);
         break;
       case "exec":
+        
         speak(`exec`);
+        server.broadcast("status", `exec`);
         logger.info(generator.jsonToManual(object));
         notify(generator.jsonToManual(object, false));
-        response = await commands.exec(object.js);
+
+        response = await commands.exec(object.lang, object.mac, object.windows, object.linux);
+        
         outputs.set(object.output, response);
+        
         break;
       default:
         throw new Error(`Command not found: ${object.command}`);

package/lib/commands.js

@@ -2,6 +2,7 @@
 const sdk = require("./sdk");
 const vm = require('vm');
 const chalk = require("chalk");
+const server = require("./ipc").server;
 const {
   captureScreenBase64,
   captureScreenPNG,
@@ -20,6 +21,8 @@ const cliProgress = require("cli-progress");
 const redraw = require("./redraw");
 const sandbox = require("./sandbox.js");
 const config = require("./config.js");
+const util = require('util');
+const exec = util.promisify(require('child_process').exec);
 let robot;
 
 let keymap;
@@ -49,7 +52,7 @@ class AiError extends Error {
   }
 }
 
-const commandOrControl = process.platform === "darwin" ? "command" : "control";
+const commandOrControl = platform() === "darwin" ? "command" : "control";
 
 const findImageOnScreen = async (relativePath, haystack, restrictToWindow) => {
   // move the file from filePath to `testdriver/screenshots`
@@ -154,6 +157,7 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
   // take a screenshot
   logger.info("");
   logger.info(chalk.dim("thinking..."), true);
+  server.broadcast("status", `thinking...`);
   logger.info("");
 
   if (async) {
@@ -222,7 +226,7 @@ const click = async (x, y, action = "click") => {
   let button = 'left';
   let double = false;
 
-  if (action === "right-click" && process.platform !== "darwin" ) {
+  if (action === "right-click" && platform !== "darwin" ) {
     button = "right";
   }
   if (action === "double-click") {
@@ -239,7 +243,7 @@ const click = async (x, y, action = "click") => {
 
   await delay(1000); // wait for the mouse to move
 
-  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
+  if (!config.TD_VM && platform()== "darwin" && action === "right-click") {
     robot.keyToggle('control', 'down', 'control');
     await delay(250);
   }
@@ -274,7 +278,7 @@ const click = async (x, y, action = "click") => {
     emitter.emit(events.mouseClick, { x, y, button, click });
   }
 
-  if (!config.TD_VM && process.platform === "darwin" && action === "right-click") {
+  if (!config.TD_VM && platform()== "darwin" && action === "right-click") {
     await delay(250);
     robot.keyToggle('control', 'up', 'control');
   }
@@ -434,7 +438,7 @@ let commands = {
     // remove any modifier keys from the inputKeys array and put them in a new array
     inputKeys.forEach((key) => {
       // change command to control on windows
-      if (key === "command" && process.platform === "win32") {
+      if (key === "command" && platform()== "win32") {
         key = "control";
       }
 
@@ -718,25 +722,89 @@ let commands = {
   assert: async (assertion, async = false) => {
     return await assert(assertion, true, async);
   },
-  exec: async (nodejs_code) => {
+  exec: async (language, mac_code, windows_code, linux_code) => {
 
-    // must be assigned to `result`
-    // do not overwrite `result`
-    // must install locally via npm install
+    logger.info(chalk.dim(`calling exec...`), true,);
 
-    const context = vm.createContext({ require, console, fs, process });
-    const script = new vm.Script(`
-      (async () => {
-        ${nodejs_code}
-      })();
-    `);
+    let plat = platform();
+    
+    const scriptCode = plat == "linux" ? linux_code : plat == "windows" ? windows_code : plat == "mac" ? mac_code : (() => { throw new AiError(`Unsupported plat: ${plat}`); })();
+
+    if (!scriptCode) {
+      logger.warn(`No code provided for ${plat}`);
+      return;
+    }
+
+    if (language == 'shell') {
+
+      logger.info(chalk.dim(`running in shell...`), true,);
+
+      if (plat == "linux") {
+
+        if (config.TD_VM) {
+
+          logger.info(chalk.dim(`sending value of \`linux\` to vm...`), true,);
+
+          return await sandbox.send({
+            type: "commands.run",
+            command: linux_code,
+          });
+          
+        } else {
+
+          if (!linux_code) {
+            throw new AiError(`No code provided for linux`, true);
+          }
 
-    await script.runInContext(context);
+          logger.info(chalk.dim(`running value of \`${plat}\` on this machine...`), true);
+          return await exec(mac_code, { cwd: cwd() });
+        }
+      } else if (plat == "windows") {
+        logger.info(chalk.dim(`running value of \`${plat}\` on this machine...`), true);
+        return await exec(windows_code, { cwd: cwd() });
+      } else if (plat == "mac") {
+        logger.info(chalk.dim(`running value of \`${plat}\` on this machine...`), true);
+        return await exec(mac_code, { cwd: cwd() });
+        
+      }
+
+    } else if (language == 'js') {
+
+      logger.info(chalk.dim(`running js...`), true,);
+  
+      // must be assigned to `result`
+      // do not overwrite `result`
+      // must install locally via npm install
+
+      if (config.TD_VM) {
+        logger.info(chalk.dim(`running value of \`linux\` on vm...`), true);
+        return await sandbox.send({
+          type: "js.run",
+          js: linux_code,
+        });
+      } else {
+
+        logger.info(chalk.dim(`running value of \`${plat}\` in local JS vm...`), true);
 
-    // wait for context.result to resolve
-    const stepResult = await context.result;
+        const context = vm.createContext({ require, console, fs, process });
 
-    return stepResult;
+        const script = new vm.Script(`
+          (async () => {
+            ${scriptCode}
+          })();
+        `);
+
+        await script.runInContext(context);
+
+        // wait for context.result to resolve
+        const stepResult = await context.result;
+
+        return stepResult;
+      }
+
+    } else {
+      throw new AiError(`Language not supported: ${language}`);
+    }
     
   }
 };

package/lib/config.js

@@ -32,11 +32,20 @@ const config = {
   TD_PROFILE: false,
   TD_OVERLAY: true,
   TD_SECRET: null,
-  TD_VM: false
+  TD_VM: false,
+  TD_OVERLAY_ID: null,
+  TD_VM_RESOLUTION: [1024, 768]
 };
 
 // Find all env vars starting with TD_
 for (let key in process.env) {
+  if (key == "TD_VM_RESOLUTION") {
+    config[key] = process.env[key]
+      .split("x")
+      .map((x) => parseInt(x.trim()));
+    continue;
+  }
+
   if (key.startsWith("TD_")) {
     config[key] = parseValue(process.env[key]);
   }

package/lib/focus-application.js

@@ -12,20 +12,20 @@ if (!config.TD_VM) {
   robot = require("robotjs");
 }
 
-async function focusVSCode() {
+// async function focusVSCode() {
 
-  try {
+//   try {
     
-    if (platform() == "mac") {
-      return await execSync('open -a "Visual Studio Code"');
-    } else {
-      return await execSync('"C:\\Program Files\\Microsoft VS Code\\Code.exe"');
-    }
-  } catch (error) {
-    logger.error(error);
-  }
+//     if (platform() == "mac") {
+//       return await execSync('open -a "Visual Studio Code"');
+//     } else {
+//       return await execSync('"C:\\Program Files\\Microsoft VS Code\\Code.exe"');
+//     }
+//   } catch (error) {
+//     logger.error(error);
+//   }
 
-}
+// }
 
 // apple script that focuses on a window
 const appleScriptSetFrontmost = (windowName) => `
@@ -52,19 +52,26 @@ const runPwsh = (appName, method) => {
 };
 
 async function focusApplication(appName) {
-  if (!config.TD_VM) {
-    try {
-      if (platform() == "mac") {
-        return await execSync(`osascript -e '${appleScriptSetFrontmost(appName)}'`);
-      } else if (platform() == "linux") {
-        // TODO: This needs fixing
-        return;
-      } else if (platform() == "windows") {
-        return runPwsh(appName, "Focus");
-      }
-    } catch (error) {
-      logger.error(error);
+  if (config.TD_VM) {
+    let result = await sandbox.send({type: "commands.run", command: `/home/user/scripts/control_window.sh "${appName}" Focus`});
+    if (result.type == "error") {
+      logger.error(result.error.result.stdout);
     }
+
+    return;
+  }
+
+  try {
+    if (platform() == "mac") {
+      return await execSync(`osascript -e '${appleScriptSetFrontmost(appName)}'`);
+    } else if (platform() == "linux") {
+      // TODO: This needs fixing
+      return;
+    } else if (platform() == "windows") {
+      return runPwsh(appName, "Focus");
+    }
+  } catch (error) {
+    logger.error(error);
   }
 }