testdriverai 5.2.2 → 5.3.0

package/lib/system.js

@@ -19,15 +19,14 @@ if (!config.TD_VM) {
 }
 
 const screenshot = async (options) => {
-
   if (config.TD_VM) {
-    let {base64} = await sandbox.send({type: 'screenshot'});
-    let image = Buffer.from(base64, 'base64');
+    let { base64 } = await sandbox.send({ type: "screenshot" });
+    let image = Buffer.from(base64, "base64");
     fs.writeFileSync(options.filename, image);
   } else {
     return await scshotdesk({ filename: options.filename, format: "png" });
   }
-}
+};
 
 let primaryDisplay = null;
 
@@ -35,26 +34,20 @@ let primaryDisplay = null;
 // this is the only display we ever target, because fuck it
 // the vm only has one and most people only have one
 const getPrimaryDisplay = async () => {
-  
   if (!primaryDisplay) {
-
     // calculate scaling resolution
     let graphics = await si.graphics();
-    primaryDisplay = graphics.displays.find(
-      (display) => display.main == true,
-    );
-
+    primaryDisplay = graphics.displays.find((display) => display.main == true);
   }
-  
+
   return primaryDisplay;
-  
 };
 
 const getSystemInformationOsInfo = async () => {
   if (config.TD_VM) {
     return {
-      os: 'linux'
-    }
+      os: "linux",
+    };
   } else {
     return await si.osInfo();
   }
@@ -68,7 +61,6 @@ const tmpFilename = () => {
 
 const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
   try {
-
     if (!silent) {
       emitter.emit(events.screenCapture.start, {
         scale,
@@ -82,35 +74,32 @@ const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
 
     await screenshot({ filename: step1, format: "png" });
 
+    // Location of cursor image
+    const cursorPath = path.join(__dirname, "resources", "cursor.png");
+
     let sharpInstance;
-    if (config.TD_VM) {
+    const mousePos = await getMousePosition();
 
+    if (config.TD_VM) {
       // resize to 1:1 px ratio
       sharpInstance = sharp(step1).resize(
-        Math.floor(1024 * scale),
-        Math.floor(768 * scale),
+        Math.floor(config.TD_VM_RESOLUTION[0] * scale),
+        Math.floor(config.TD_VM_RESOLUTION[1] * scale),
       );
-
     } else {
-
       const primaryDisplay = await getPrimaryDisplay();
-      
+
       sharpInstance = sharp(step1).resize(
         Math.floor(primaryDisplay.currentResX * scale),
         Math.floor(primaryDisplay.currentResY * scale),
       );
+    }
 
-      // Fetch the mouse position
-      const mousePos = robot.getMousePos();
-
-      // Location of cursor image
-      const cursorPath = path.join(__dirname, "resources", "cursor.png");
-
-      if (mouse) {
-        // composite the mouse image ontop
-        sharpInstance.composite([{ input: cursorPath, left: mousePos.x, top: mousePos.y }]);
-      }
-
+    if (mouse) {
+      // composite the mouse image ontop
+      sharpInstance.composite([
+        { input: cursorPath, left: mousePos.x, top: mousePos.y },
+      ]);
     }
 
     await sharpInstance.toFile(step2);
@@ -134,7 +123,11 @@ const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
 };
 
 // our handy screenshot function
-const captureScreenBase64 = async (scale = 1, silent = false, mouse = false) => {
+const captureScreenBase64 = async (
+  scale = 1,
+  silent = false,
+  mouse = false,
+) => {
   let step2 = await captureAndResize(scale, silent, mouse);
   return fs.readFileSync(step2, "base64");
 };
@@ -145,6 +138,11 @@ const captureScreenPNG = async (scale = 1, silent = false, mouse = false) => {
 
 const platform = () => {
   let platform = process.platform;
+
+  if (config.TD_VM) {
+    platform = "linux";
+  }
+
   if (platform === "darwin") {
     platform = "mac";
   } else if (platform === "win32") {
@@ -161,7 +159,10 @@ const platform = () => {
 let activeWindowFn = null;
 const initializeActiveWindow = async () => {
   if (!activeWindowFn && !config.TD_VM) {
-    const { activeWindow } = await import('get-windows');
+    // For some reason, the import fails in certain situations
+    const activeWindow = await import("get-windows")
+      .then(({ activeWindow }) => activeWindow)
+      .catch(() => null);
     activeWindowFn = activeWindow;
   }
   return activeWindowFn;
@@ -169,18 +170,34 @@ const initializeActiveWindow = async () => {
 
 // this is the focused window
 const activeWin = async () => {
-  
-  if (config.TD_VM) { 
+  if (config.TD_VM) {
     return "error getting active window, proceed normally";
   } else {
     const activeWindow = await initializeActiveWindow();
-    return await activeWindow();
+    return await activeWindow?.();
   }
 };
 
 const getMousePosition = async () => {
-  // @todo vm does expose mouse position, add it back
-  return !config.TD_VM ? await robot.getMousePos() : null;
+  if (config.TD_VM) {
+    // Get Mouse Position from command line
+    let response = await sandbox.send({
+      type: "commands.run",
+      command:
+        'eval $(xdotool getmouselocation --shell) && echo "{\\"x\\":$X,\\"y\\":$Y}"',
+    });
+
+    if (response.out.stderr) {
+      return {
+        x: 0,
+        y: 0,
+      };
+    }
+
+    return JSON.parse(response.out.stdout);
+  } else {
+    return await robot.getMousePos();
+  }
 };
 
 module.exports = {

package/lib/upload-secrets.js

@@ -0,0 +1,65 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+const dotenv = require('dotenv');
+const prompts = require('prompts');
+
+module.exports = async () => {
+  
+// 1. Check for GitHub CLI
+try {
+  execSync('gh --version', { stdio: 'ignore' });
+} catch (err) {
+  console.log(err)
+  console.error('❌ GitHub CLI (gh) is not installed.\n');
+  console.log('👉 Install it from https://cli.github.com/');
+  console.log('   macOS:    brew install gh');
+  console.log('   Windows:  winget install GitHub.cli');
+  console.log('   Ubuntu:   sudo apt install gh\n');
+  process.exit(1);
+}
+
+// 2. Load and parse .env file
+const envPath = '.env';
+if (!fs.existsSync(envPath)) {
+  console.error('❌ .env file not found.');
+  process.exit(1);
+}
+
+const env = dotenv.parse(fs.readFileSync(envPath));
+const tdSecrets = Object.entries(env).filter(([key]) => key.startsWith('TD_'));
+
+if (tdSecrets.length === 0) {
+  console.log('ℹ️ No secrets found in .env that start with TD_');
+  process.exit(0);
+}
+
+// 3. Prompt user for confirmation
+(async () => {
+  console.log('\n🔐 The following TD_ secrets will be uploaded:');
+  tdSecrets.forEach(([key]) => console.log(`- ${key}`));
+
+  const response = await prompts({
+    type: 'confirm',
+    name: 'confirm',
+    message: '\nAre you sure you want to upload these secrets to the current GitHub repo?',
+    initial: false,
+  });
+
+  if (!response.confirm) {
+    console.log('❌ Upload cancelled.');
+    process.exit(0);
+  }
+
+  // 4. Upload secrets using GitHub CLI
+  for (const [key, value] of tdSecrets) {
+    try {
+      const cmd = `gh secret set ${key} --body "${value}"`;
+      execSync(cmd, { stdio: 'inherit' });
+    } catch (err) {
+      console.error(`❌ Failed to upload ${key}: ${err.message}`);
+    }
+  }
+
+})();
+
+}

package/lib/validation.js

@@ -0,0 +1,175 @@
+const types = () =>
+  import("arktype").then(({ scope }) =>
+    scope({
+      // - command: press-keys # Types a keyboard combination. Repeat the command to repeat the keypress.
+      //   keys: [command, space]
+      PressKeysCommand: {
+        command: '"press-keys"',
+        keys: "string[]",
+      },
+      // - command: hover-text # Hovers text matching the \`description\`. The text must be visible. This will also handle clicking or right clicking on the text if required.
+      //   text: Sign Up # The text to find on screen. The longer and more unique the better.
+      //   description: registration in the top right of the header # Describe the element so it can be identified in the future. Do not include the text itself here. Make sure to include the unique traits of this element.
+      //   action: click # What to do when text is found. Available actions are: click, right-click, double-click, hover
+      //   method: ai # Optional. Only try this if text match is not working.
+      HoverTextCommand: {
+        command: '"hover-text"',
+        text: "string",
+        description: "string",
+        action: '"click" | "right-click" | "double-click" | "hover"',
+        "method?": '"ai"',
+      },
+      // - command: type # Types the string into the active application. You must focus the correct field before typing.
+      //   text: Hello World
+      TypeCommand: {
+        command: '"type"',
+        text: "string",
+      },
+      // - command: wait # Waits a number of miliseconds before continuing.
+      //   timeout: 5000
+      WaitCommand: {
+        command: '"wait"',
+        timeout: "number",
+      },
+      // - command: hover-image # Hovers an icon, button, or image matching \`description\`. This will also handle handle clicking or right clicking on the icon or image if required.
+      //   description: search icon in the webpage content # Describe the icon or image and what it represents. Describe the element so it can be identified in the future. Do not include the image or icon itself here. Make sure to include the unique traits of this element.
+      //   action: click # What to do when text is found. Available actions are: click, right-click, double-click, hover
+      HoverImageCommand: {
+        command: '"hover-image"',
+        description: "string",
+        action: '"click" | "right-click" | "double-click" | "hover"',
+      },
+      // - command: focus-application # Focus an application by name.
+      //   name: Google Chrome # The name of the application to focus.
+      FocusApplicationCommand: {
+        command: '"focus-application"',
+        name: "string",
+      },
+      // - command: remember # Remember a string value without needing to interact with the desktop.
+      //   description: My dog's name # The key of the memory value to store.
+      //   value: Roofus # The value of the memory to store
+      RememberCommand: {
+        command: '"remember"',
+        description: "string",
+        value: "string",
+      },
+      // - command: get-email-url # Retrieves the URL from a sign-up confirmation email in the background.
+      //   # This retrieves an email confirmation URL without opening an email client. Do not view the screen, just run this command when dealing with emails
+      //   username: testdriver # The username of the email address to check.
+      GetEmailUrlCommand: {
+        command: '"get-email-url"',
+        username: "string",
+      },
+      // - command: scroll # Scroll up or down. Make sure the correct portion of the page is focused before scrolling.
+      //   direction: down # Available directions are: up, down, left, right
+      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
+      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300 for keyboard and 200 for mouse.
+      ScrollCommand: {
+        command: '"scroll"',
+        direction: '"up" | "down" | "left" | "right"',
+        "method?": '"keyboard" | "mouse"',
+        "amount?": "number",
+      },
+      // - command: scroll-until-text # Scroll until text is found
+      //   text: Sign Up # The text to find on screen. The longer and more unique the better.
+      //   direction: down # Available directions are: up, down, left, right
+      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
+      ScrollUntilTextCommand: {
+        command: '"scroll-until-text"',
+        text: "string",
+        direction: '"up" | "down" | "left" | "right"',
+        "method?": '"keyboard" | "mouse"',
+      },
+      // - command: scroll-until-image # Scroll until icon or image is found
+      //   description: Submit at the bottom of the form
+      //   direction: down # Available directions are: up, down, left, rights
+      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
+      ScrollUntilImageCommand: {
+        command: '"scroll-until-image"',
+        description: "string",
+        direction: '"up" | "down" | "left" | "right"',
+        "method?": '"keyboard" | "mouse"',
+      },
+      // - command: wait-for-text # Wait until text is seen on screen. Not recommended unless explicitly requested by user.
+      //   text: Copyright 2024 # The text to find on screen.
+      WaitForTextCommand: {
+        command: '"wait-for-text"',
+        text: "string",
+      },
+      // - command: wait-for-image # Wait until icon or image is seen on screen. Not recommended unless explicitly requested by user.
+      //   description: trash icon
+      WaitForImageCommand: {
+        command: '"wait-for-image"',
+        description: "string",
+      },
+      // - command: assert # Assert that a condition is true. This is used to validate that a task was successful. Only use this when the user asks to "assert", "check," or "make sure" of something.
+      //   expect: the video is playing # The condition to check. This should be a string that describes what you see on screen.
+      AssertCommand: {
+        command: '"assert"',
+        expect: "string",
+      },
+      // - command: if # Conditional block. If the condition is true, run the commands in the block. Otherwise, run the commands in the else block. Only use this if the user explicitly asks for a condition.
+      //   condition: the active window is "Google Chrome"
+      //   then:
+      //     - command: hover-text
+      //       text: Search Google or type a URL
+      //       description: main google search
+      //       action: click
+      //     - command: type
+      //       text: monster trucks
+      //       description: search for monster trucks
+      //   else:
+      //     - command: focus-application
+      //       name: Google Chrome
+      IfCommand: {
+        command: '"if"',
+        condition: "string",
+        // The unknown[] instead of Command[] is because of a weird error, although it does work
+        // on the other typescript repo
+        then: "unknown[]",
+        "else?": "unknown[]",
+      },
+
+      Command:
+        "PressKeysCommand | HoverTextCommand | TypeCommand | WaitCommand | HoverImageCommand | FocusApplicationCommand | RememberCommand | GetEmailUrlCommand | ScrollCommand | ScrollUntilTextCommand | ScrollUntilImageCommand | WaitForTextCommand | WaitForImageCommand | AssertCommand | IfCommand",
+
+      CommandList: "Command[]",
+
+      Step: {
+        prompt: "string",
+        commands: "Command[]",
+      },
+
+      StepList: "Step[]",
+
+      File: {
+        version: "string",
+        session: "string | undefined",
+        steps: "Step[]",
+      },
+    }).export(),
+  );
+
+/**
+ *
+ * @param {unknown} data
+ */
+const getType = async (data) => {
+  const { type } = await import("arktype");
+  const t = await types();
+
+  /**
+   * @type {["File", "Step", "StepList", "Command", "CommandList"]}
+   */
+  const options = ["File", "Step", "StepList", "Command", "CommandList"];
+
+  for (const option of options) {
+    const result = t[option](data);
+    if (!(result instanceof type.errors)) {
+      return option;
+    }
+  }
+  return null;
+};
+
+module.exports = { getType };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "5.2.2",
+  "version": "5.3.0",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {
@@ -18,6 +18,7 @@
   "dependencies": {
     "@e2b/desktop": "^1.6.0",
     "@electerm/strip-ansi": "^1.0.0",
+    "arktype": "^2.1.19",
     "axios": "^1.7.7",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",

package/postinstall.js

@@ -1,7 +1,5 @@
 let platform = require("os").platform();
 let exec = require("child_process").exec;
-const { execSync } = require("child_process");
-const readline = require("readline");
 
 if (platform !== "darwin") {
   console.log("TestDriver Setup: Skipping codesign becasue not on Mac");
@@ -20,25 +18,3 @@ exec(signScript, (error, stdout, stderr) => {
   console.log(`stdout: ${stdout}`);
   console.error(`stderr: ${stderr}`);
 });
-
-const rl = readline.createInterface({
-  input: process.stdin,
-  output: process.stdout
-});
-
-rl.question("Do you want to install Vue support? (y/n) ", (answer) => {
-  if (answer.toLowerCase() === "y") {
-    console.log("Installing Vue dependencies...");
-    execSync("npm install -g vue vue-router", { stdio: "inherit" });
-  }
-
-  rl.question("Do you want to install React support? (y/n) ", (answer2) => {
-    if (answer2.toLowerCase() === "y") {
-      console.log("Installing React dependencies...");
-      execSync("npm install -g react react-dom", { stdio: "inherit" });
-    }
-
-    console.log("Setup complete!");
-    rl.close();
-  });
-});

package/lib/websockets.js

@@ -1,85 +0,0 @@
-const WebSocket = require('ws');
-const config = require('./config');
-
-let instance;
-class WebSocketServerSingleton {
-  constructor() {
-
-    if (!WebSocketServerSingleton.instance) {
-      this.wss = new WebSocket.Server({ port: 8080 }, () => {
-      });
-
-      this.clients = new Set();
-      this.eventListeners = new Map();
-
-      this.wss.on('error', (error) => {
-        console.error('WebSocket server error:', error);
-      });
-
-      this.wss.on('connection', (ws) => {
-
-        console.log('Client connected');
-        this.clients.add(ws);
-
-        ws.on('message', (message) => {
-          try {
-            const parsedMessage = JSON.parse(message);
-            if (parsedMessage.event) {
-              this.triggerEvent(parsedMessage.event, parsedMessage);
-            } else {
-              console.error('Parsed message does not contain an event property');
-            }
-          } catch (error) {
-            console.error('Error parsing message:', error);
-          }
-        });
-
-        ws.on('close', () => {
-          this.clients.delete(ws);
-          console.log('Client disconnected');
-        });
-
-        ws.on('error', (error) => {
-          console.error('WebSocket error:', error);
-        });
-      });
-
-      WebSocketServerSingleton.instance = this;
-    }
-
-    return WebSocketServerSingleton.instance;
-  }
-
-  addEventListener(event, listener) {
-    if (!this.eventListeners.has(event)) {
-      this.eventListeners.set(event, []);
-    }
-    this.eventListeners.get(event).push(listener);
-  }
-
-  triggerEvent(event, data) {
-    if (this.eventListeners.has(event)) {
-      for (const listener of this.eventListeners.get(event)) {
-        listener(data);
-      }
-    }
-  }
-
-  sendToClients(event, message) {
-    const data = JSON.stringify({ event, message });
-    for (const client of this.clients) {
-      if (client.readyState === WebSocket.OPEN) {
-        client.send(data);
-      }
-    }
-  }
-}
-
-
-module.exports = {
-  create: () => {
-    instance = new WebSocketServerSingleton();
-    Object.freeze(instance);  
-    return instance;
-  }
-};

package/test.md

@@ -1,8 +0,0 @@
-# Easy Math
-
-2 + 2 = 4 // this test will pass
-2 + 2 = 5 // this test will fail
-
-# Harder Math
-
-230230 + 5819123 = 6049353

package/test.yml

@@ -1,18 +0,0 @@
-version: 4.1.40
-steps:
-  - prompt: Enter Password n stuff
-    commands:
-      - command: focus-application
-        name: Google Chrome
-      - command: hover-text
-        text: Username 
-        description: username field
-        action: click
-      - command: type
-        text: ${TD_PASSWORD}
-      - command: hover-text
-        text: Password
-        description: Password field
-        action: click
-      - command: type
-        text: ${TD_PASSWORD}