testdriverai 5.2.1 → 5.3.0

package/.github/workflows/test-install.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         runner: [macos-latest, windows-latest]
-        node-version: ['18', '20']
+        node-version: ['18', '20', '22']
 
     steps:
       - name: Checkout repository

package/README.md

@@ -4,17 +4,9 @@
 
 Automate and scale QA with computer-use agents.
 
-[Docs](https://docs.testdriver.ai) | [Website](https://testdriver.ai) | [GitHub Action](https://github.com/marketplace/actions/testdriver-ai) | [Join our Discord](https://discord.gg/a8Cq739VWn)
+[Docs](https://docs.testdriver.ai) | [Website](https://testdriver.ai) | [GitHub Action](https://github.com/marketplace/actions/testdriver-ai) | [Join our Discord](https://discord.com/invite/cWDFW8DzPm)
 
----
-
-
-
-https://github.com/user-attachments/assets/4719e834-652a-43ba-8b8c-24ea6f357ae3
-
-
-
-# Install via NPM
+---https://github.com/user-attachments/assets/4719e834-652a-43ba-8b8c-24ea6f357ae3# Install via NPM
 
 ```sh
 npm install testdriverai -g
@@ -36,7 +28,9 @@ TestDriver isn't like any test framework you've used before. TestDriver is an OS
 - **Less Maintenance:** Tests don't break when code changes
 - **More Power:** TestDriver can test any application and control any OS setting
 
-### Demo
+### Demo (Playing Balatro Desktop)
+
+https://github.com/user-attachments/assets/7cb9ee5a-0d05-4ff0-a4fa-084bcee12e98
 
 # Examples
 

package/agent.js

@@ -31,7 +31,7 @@ const sanitizeFilename = require("sanitize-filename");
 const macScreenPerms = require("mac-screen-capture-permissions");
 
 // local modules
-const websocketserver = require("./lib/websockets.js");
+const { server } = require("./lib/ipc.js");
 const speak = require("./lib/speak.js");
 const analytics = require("./lib/analytics.js");
 const log = require("./lib/logger.js");
@@ -44,6 +44,7 @@ const commands = require("./lib/commands.js");
 const init = require("./lib/init.js");
 const config = require("./lib/config.js");
 const sandbox = require("./lib/sandbox.js");
+const uploadSecrets = require("./lib/upload-secrets.js");
 
 const { showTerminal, hideTerminal } = require("./lib/focus-application.js");
 const isValidVersion = require("./lib/valid-version.js");
@@ -63,14 +64,17 @@ let checkCount = 0;
 let checkLimit = 7;
 let lastScreenshot = null;
 let rl;
-let wss; 
 
 // list of prompts that the user has given us
 let tasks = [];
 
-let isInteractive = false;
+let isInteractive = true;
 emitter.on(events.interactive, (data) => {
   isInteractive = data;
+  server.broadcast(events.interactive, data);
+});
+emitter.on(events.vm.show, ({ url }) => {
+  server.broadcast(events.vm.show, url);
 });
 
 // get args from terminal
@@ -78,6 +82,8 @@ const args = process.argv.slice(2);
 
 const commandHistoryFile = path.join(os.homedir(), ".testdriver_history");
 
+let workingDir = process.cwd();
+
 let getArgs = () => {
   let command = 0;
   let file = 1;
@@ -95,6 +101,8 @@ let getArgs = () => {
 
   if (args[command] == "init") {
     args[command] = "init";
+  } else if (args[command] == "upload-secrets") {
+    args[command] = "upload-secrets";
   } else if (args[command] !== "run" && !args[file]) {
     args[file] = args[command];
     args[command] = "edit";
@@ -104,7 +112,7 @@ let getArgs = () => {
 
   if (!args[file]) {
     // make testdriver directory if it doesn't exist
-    let testdriverFolder = path.join(process.cwd(), "testdriver");
+    let testdriverFolder = path.join(workingDir, "testdriver");
     if (!fs.existsSync(testdriverFolder)) {
       fs.mkdirSync(testdriverFolder);
     }
@@ -114,7 +122,7 @@ let getArgs = () => {
 
   // turn args[file] into local path
   if (args[file]) {
-    args[file] = path.join(process.cwd(), args[file]);
+    args[file] = path.join(workingDir, args[file]);
     if (!args[file].endsWith(".yaml")) {
       args[file] += ".yaml";
     }
@@ -125,7 +133,7 @@ let getArgs = () => {
 
 let a = getArgs();
 
-const thisFile = a.file;
+let thisFile = a.file;
 const thisCommand = a.command;
 
 logger.info(chalk.green(`Howdy! I'm TestDriver v${package.version}`));
@@ -153,7 +161,7 @@ function fileCompleter(line) {
     partial = line.slice(lastSepIndex + 1);
   }
   try {
-    const dirPath = path.resolve(process.cwd(), dir);
+    const dirPath = path.resolve(workingDir, dir);
 
     let files = fs.readdirSync(dirPath);
     files = files.map((file) => {
@@ -171,10 +179,10 @@ function fileCompleter(line) {
 }
 
 function completer(line) {
-  let completions = "/summarize /save /run /quit /assert /undo /manual /yml".split(
+  let completions = "/summarize /save /run /quit /assert /undo /manual /yml /js /exec".split(
     " ",
   );
-  if (line.startsWith("/run ")) {
+  if (line.startsWith("/run ") || line.startsWith("/explore ")) {
     return fileCompleter(line);
   } else {
     completions.concat(tasks);
@@ -270,7 +278,7 @@ const haveAIResolveError = async (error, markdown, depth = 0, undo = true) => {
 
   speak("thinking...");
   notify("thinking...");
-
+  server.broadcast("status", `thinking...`);
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
@@ -310,6 +318,7 @@ const check = async () => {
 
   logger.info("");
   logger.info(chalk.dim("checking..."), "testdriver");
+  server.broadcast("status", `checking...`);
   logger.info("");
 
   let thisScreenshot = await system.captureScreenBase64(1, false, true);
@@ -345,6 +354,8 @@ const check = async () => {
 const runCommand = async (command, depth) => {
   let yml = await yaml.dump(command);
 
+  await save({});
+
   logger.debug(`running command: \n\n${yml}`);
 
   try {
@@ -467,7 +478,7 @@ const loadYML = async (file) => {
   } catch (e) {
     logger.error(e);
     logger.error(`File not found: ${file}`);
-    logger.error(`Current directory: ${process.cwd()}`);
+    logger.error(`Current directory: ${workingDir}`);
 
     await summarize("File not found");
     await exit(true);
@@ -513,6 +524,7 @@ const assert = async (expect) => {
 
   speak("thinking...");
   notify("thinking...");
+  server.broadcast("status", `thinking...`);
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
@@ -539,6 +551,7 @@ const exploratoryLoop = async (currentTask, dry = false, validateAndLoop = false
 
   speak("thinking...");
   notify("thinking...");
+  server.broadcast("status", `thinking...`);
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
@@ -561,11 +574,14 @@ const exploratoryLoop = async (currentTask, dry = false, validateAndLoop = false
   );
   mdStream.end();
 
-  await aiExecute(message.data, validateAndLoop, dry);
-
-  logger.debug("showing prompt from exploratoryLoop response check");
-
+  if (message) {
+    await aiExecute(message.data, validateAndLoop, dry);
+    logger.debug("showing prompt from exploratoryLoop response check");
+  }
+  
   await save({ silent: false });
+
+  return;
 };
 
 const generate = async (type, count, baseYaml, skipYaml = false) => {
@@ -573,7 +589,7 @@ const generate = async (type, count, baseYaml, skipYaml = false) => {
 
   speak("thinking...");
   notify("thinking...");
-
+  server.broadcast("status", `thinking...`);
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
@@ -604,29 +620,34 @@ const generate = async (type, count, baseYaml, skipYaml = false) => {
 
   // for each testPrompt
   for (const testPrompt of testPrompts) {
+
     // with the contents of the testPrompt
     let fileName =
-      sanitizeFilename(testPrompt.headings[0])
+      sanitizeFilename(testPrompt.name)
         .trim()
         .replace(/ /g, "-")
         .replace(/['"`]/g, "")
         .replace(/[^a-zA-Z0-9-]/g, "") // remove any non-alphanumeric chars except hyphens
-        .toLowerCase() + ".md";
-    let path1 = path.join(process.cwd(), "testdriver", "generate", fileName);
+        .toLowerCase() + ".yaml";
+    let path1 = path.join(workingDir, "testdriver", "generate", fileName);
 
     // create generate directory if it doesn't exist
-    if (!fs.existsSync(path.join(process.cwd(), "testdriver", "generate"))) {
-      fs.mkdirSync(path.join(process.cwd(), "testdriver", "generate"));
+    if (!fs.existsSync(path.join(workingDir, "testdriver", "generate"))) {
+      fs.mkdirSync(path.join(workingDir, "testdriver", "generate"));
     }
 
-    let list = testPrompt.listsOrdered[0];
+    let list = testPrompt.steps;
 
     if (baseYaml && fs.existsSync(baseYaml)) {
-      list.unshift(`/run ${baseYaml} --embed`);
+      list.unshift({step: {
+        command: "run",
+        file: baseYaml,
+      }});
     }
-    let contents = list
-      .map((item, index) => `${index + 1}. ${item}`)
-      .join("\n");
+    let contents = yaml.dump({ 
+      version: package.version,
+      steps: list
+    });
     fs.writeFileSync(path1, contents);
   }
 
@@ -712,21 +733,9 @@ const ensureMacScreenPerms = async () => {
   }
 }
 
-const newSession = async () => {
-  // should be start of new session
-  const sessionRes = await sdk.req("session/start", {
-    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
-    mousePosition: await system.getMousePosition(),
-    activeWindow: await system.activeWin(),
-  });
-
-  session.set(sessionRes.data.id);
-};
-
 // simple function to backfill the chat history with a prompt and
 // then call `promptUser()` to get the user input
 const firstPrompt = async () => {
-  await newSession();
 
   // readline is what allows us to get user input
   rl = readline.createInterface({
@@ -748,6 +757,7 @@ const firstPrompt = async () => {
   // this is how we parse user input
   // notice that the AI is only called if the input is not a command
   const handleInput = async (input) => {
+
     if (!isInteractive) return;
     if (!input.trim().length) return promptUser();
 
@@ -790,37 +800,16 @@ const firstPrompt = async () => {
       await manualInput(commands.slice(1).join(" "));
     } else if (input.indexOf("/run") == 0) {
       const file = commands[1];
+      thisFile = file;
       const flags = commands.slice(2);
       let shouldSave = flags.includes("--save") ? true : false;
       let shouldExit = flags.includes("--exit") ? true : false;
-      let shouldEmbed = flags.includes("--embed") ? true : false;
 
-      if (shouldEmbed && (shouldSave || shouldExit)) {
-        await dieOnFatal({
-          message:
-            "Cannot embed AND save or exit. Please either use --embed or  use any combination of --save and --exit.",
-        });
-      }
+      await run(file, shouldSave, shouldExit);
 
-      if (shouldEmbed) {
-        const relativePath = path.relative(
-          process.cwd(),
-          path.resolve(process.cwd(), file),
-        );
-
-        executionHistory.push({
-          prompt: `/run ${relativePath}`,
-          commands: [
-            {
-              command: "run",
-              file: relativePath,
-            },
-          ],
-        });
-        await embed(file, 0);
-      } else {
-        await run(file, shouldSave, shouldExit);
-      }
+    } else if (input.indexOf("/explore") == 0) {
+      const file = commands[1];
+      await run(file, true, true);
     } else if (input.indexOf("/generate") == 0) {
       const skipYaml = commands[4] === "--skip-yaml";
       await generate(commands[1], commands[2], commands[3], skipYaml);
@@ -828,6 +817,26 @@ const firstPrompt = async () => {
       await exploratoryLoop(input.replace('/dry', ''), true, false);
     } else if (input.indexOf("/yaml") == 0) {
       await runRawYML(commands[1]);
+    } else if (input.indexOf("/js") == 0) {
+      let result = await commander.run({
+        command: "exec",
+        js: commands.slice(1).join(" "),
+      });
+      if (result.out) {
+        logger.info(result.out.stdout);
+      } else if (result.error) {
+        logger.error(result.error.result.stdout);
+      }
+    } else if (input.indexOf("/exec") == 0) {
+      let result = await commander.run({
+        command: "exec",
+        cli: commands.slice(1).join(" "),
+      });
+      if (result.out) {
+        logger.info(result.out.stdout);
+      } else if (result.error) {
+        logger.error(result.error.result.stdout);
+      }
     } else {
       await exploratoryLoop(input, false, true);
     }
@@ -837,10 +846,7 @@ const firstPrompt = async () => {
   };
 
   rl.on("line", handleInput);
-
-  config.TD_VM && wss.addEventListener("input", async (message) => {
-    handleInput(message.data);
-  });
+  server.on("input", handleInput);
 
   // if file exists, load it
   if (fs.existsSync(thisFile)) {
@@ -851,15 +857,8 @@ const firstPrompt = async () => {
       fs.readFileSync(thisFile, "utf-8"),
     );
 
-    if (!object?.steps) {
-      analytics.track("load invalid yaml");
-      logger.error("Invalid YAML. No steps found.");
-      logger.info("Invalid YAML: " + thisFile);
-      return await exit(true);
-    }
-
     // push each step to executionHistory from { commands: {steps: [ { commands: [Array] } ] } }
-    object.steps.forEach((step) => {
+    object.steps?.forEach((step) => {
       executionHistory.push(step);
     });
 
@@ -973,7 +972,6 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
   }
 
   if (!executionHistory.length) {
-    console.log('no exeuction history, not saving')
     return;
   }
 
@@ -1001,6 +999,8 @@ ${regression}
       logger.info(chalk.dim(`saved as ${fileName}`));
     }
   }
+
+  return;
 };
 
 let runRawYML = async (yml) => {
@@ -1022,14 +1022,10 @@ let runRawYML = async (yml) => {
 // it parses the markdown file and executes the codeblocks exactly as if they were
 // generated by the AI in a single prompt
 let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
-  await newSession();
 
   setTerminalWindowTransparency(true);
   emitter.emit(events.interactive, false);
 
-  // get the current wowrking directory where this file is being executed
-  let cwd = process.cwd();
-
   logger.info(chalk.cyan(`running ${file}...`));
 
   let ymlObj = await loadYML(file);
@@ -1050,12 +1046,24 @@ let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
 
   for (const step of ymlObj.steps) {
     logger.info(``, null);
-    logger.info(chalk.yellow(`${step.prompt || "no prompt"}`), null);
+    logger.info(chalk.yellow(`> ${step.prompt || "no prompt"}`), null);
 
-    executionHistory.push({
-      prompt: step.prompt,
-      commands: [], // run will overwrite the commands
-    });
+    if (!step.commands && !step.prompt) {
+      logger.info(chalk.red("No commands or prompt found"));
+      return await exit(true);
+    } else if (!step.commands) {
+      logger.info(chalk.yellow("No commands found, running exploratory"));
+      await exploratoryLoop(step.prompt, false, true);
+    }
+
+    if (shouldSave) {
+
+      executionHistory.push({
+        prompt: step.prompt,
+        commands: [], // run will overwrite the commands
+      });
+
+    }
 
     let markdown = `\`\`\`yaml
 ${yaml.dump(step)}
@@ -1073,7 +1081,6 @@ ${yaml.dump(step)}
   }
 
   setTerminalWindowTransparency(false);
-  emitter.emit(events.interactive, true);
 
   if (shouldExit) {
     await summarize();
@@ -1082,7 +1089,6 @@ ${yaml.dump(step)}
 };
 
 const promptUser = () => {
-  config.TD_VM && wss.sendToClients("done");
   emitter.emit(events.interactive, true);
   rl.prompt(true);
 };
@@ -1122,7 +1128,7 @@ const embed = async (file, depth) => {
   logger.info(`${file} (start)`);
 
   // get the current wowrking directory where this file is being executed
-  let cwd = process.cwd();
+  let cwd = workingDir;
 
   // if the file is not an absolute path, we will try to resolve it
   if (!path.isAbsolute(file)) {
@@ -1145,13 +1151,12 @@ const embed = async (file, depth) => {
 
 const buildEnv = async () => {
   let win = await system.activeWin();
-  if (config.TD_VM) {
-    wss = websocketserver.create();
-  }
   setTerminalApp(win);
   await ensureMacScreenPerms();
   await makeSandbox();
-}
+  await newSession();
+  await runPrerun();
+};
 
 const start = async () => {
   // logger.info(await  system.getPrimaryDisplay());
@@ -1168,18 +1173,21 @@ const start = async () => {
 
   if (thisCommand !== "run") {
     speak("Howdy! I am TestDriver version " + package.version);
+  }
+
+  if (thisCommand !== "init" || thisCommand !== "upload-secrets") {
 
     if (!config.TD_VM) {
       logger.info(
-        chalk.red("Warning!" ) +
-          chalk.dim("Local mode sends screenshots of the desktop to our API."),
+        chalk.red("Warning! " ) +
+          chalk.dim("Local mode sends screenshots of the desktop to our API. Set `TD_VM=true` to run in a secure VM."),
       );
       logger.info(
         chalk.dim("https://docs.testdriver.ai/security-and-privacy/agent"),
       );
       logger.info("");
     }
-    
+
   }
 
   analytics.track("command", { command: thisCommand, file: thisFile });
@@ -1194,6 +1202,8 @@ const start = async () => {
   } else if (thisCommand == "init") {
     await init();
     process.exit(0);
+  } else if (thisCommand == "upload-secrets") {
+    await uploadSecrets();
   }
 };
 
@@ -1203,19 +1213,25 @@ const makeSandbox = async () => {
           
     try {
 
-      logger.info(chalk.gray(`- creating linux sandbox...`));
+      logger.info(chalk.gray(`- creating sandbox...`));
+      server.broadcast("status", `Creating new sandbox...`);
       await sandbox.boot();
       logger.info(chalk.gray(`- authenticating...`));
+      server.broadcast("status", `Authenticating...`);
       await sandbox.send({type: 'authenticate', apiKey: config.TD_API_KEY, secret: config.TD_SECRET} );
-      logger.info(chalk.gray(`- setting up...`));
-      await sandbox.send({type: 'create', resolution: [1024, 768]});
+      logger.info(chalk.gray(`- configuring...`));
+      server.broadcast("status", `Configuring...`);
+      await sandbox.send({type: 'create', resolution: config.TD_VM_RESOLUTION});
       logger.info(chalk.gray(`- starting stream...`));
+      server.broadcast("status", `Starting stream...`);
       await sandbox.send({type: 'stream.start'});
       let {url} = await sandbox.send({type: 'stream.getUrl'});
       logger.info(chalk.gray(`- rendering...`));
+      server.broadcast("status", `Rendering...`);
       await sandbox.send({type: 'ready'});
       emitter.emit(events.vm.show, {url});
       logger.info(chalk.gray(`- booting...`));
+      server.broadcast("status", `Starting...`);
       await new Promise(resolve => setTimeout(resolve, 3000)); 
       logger.info(chalk.green(``));
       logger.info(chalk.green(`sandbox runner ready!`));
@@ -1233,6 +1249,26 @@ const makeSandbox = async () => {
 
 }
 
+
+
+const newSession = async () => {
+  // should be start of new session
+  const sessionRes = await sdk.req("session/start", {
+    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
+    mousePosition: await system.getMousePosition(),
+    activeWindow: await system.activeWin(),
+  });
+
+  session.set(sessionRes.data.id);
+};
+
+const runPrerun = async () => {
+  const prerunFile = path.join(workingDir, "testdriver", "lifecycle", "prerun.yaml");
+  if (fs.existsSync(prerunFile)) {
+    await run(prerunFile, false, false);
+  }
+}
+
 process.on("uncaughtException", async (err) => {
   analytics.track("uncaughtException", { err });
   logger.error("Uncaught Exception: %s", err);

package/docs/30x30.mdx

@@ -0,0 +1,84 @@
+# 30x30 Promotion: 30 Tests in 30 Days Free!
+
+## Overview
+We understand that QA coverage is critical and often needed immediately. That’s why we’re offering our **30x30 Promotion**: our team will build and deploy **30 custom tests in 30 days** for free during your trial! Unlike other QA services that take months and cost hundreds of thousands of dollars, TestDriver gets you up and running in just 30 days.
+
+This is a **limited-time offer**, so don’t miss your chance! [**Schedule an onboarding call now**](#).
+
+---
+
+## Pricing
+We’re confident you’ll love TestDriver, which is why we’re offering this risk-free trial. After the trial, plans start at just **$995/month**, which includes:
+- **12,000 runner minutes per month**: Enough to run your 30 tests **2–5 times per day**.
+- Full access to our enterprise test dashboards.
+- Seamless integration with GitHub Actions.
+
+For more details, see the [Promotion Details](#promotion-details) and [Contract Details](#contract-details).
+
+---
+
+## Why Choose TestDriver?
+
+### Quickly Deploy AI QA Tests
+- Safeguard your most important user flows with AI-generated tests.
+- Increase your total coverage with minimal effort.
+
+### Test Flows Never Possible Before
+- Our powerful computer-use agent can test desktop apps, Chrome extensions, mobile apps, and websites.
+
+### Spend Less Time on Maintenance
+- TestDriver tests automatically repair themselves, reducing the time spent on maintenance.
+
+---
+
+## Promotion Details
+Here’s what’s included in the **30x30 Promotion**:
+
+- **Test Coverage**: Test any publicly available desktop app, Chrome extension, mobile app, or website.
+- **AI-Generated Tests**: Choose from **250 AI-generated tests** within the first 7 days.
+- **Daily Test Runs**: Run tests **3–5 times every day**.
+- **AI Quality Reports**: Receive detailed reports delivered directly to your email.
+- **Self-Healing Tests**: AI automatically fixes and maintains your tests.
+- **GitHub Integration**: Seamlessly integrate with GitHub Actions.
+- **Enterprise Dashboards**: Gain full access to our enterprise-grade test dashboards.
+
+For more details, see the [Contract Details](#contract-details).
+
+---
+
+## How Does It Work?
+
+1. **Book an Onboarding Call**: Share the specific flows you want to test.
+2. **Generate a Test Suite**: We’ll explore your app and generate hundreds of tests using our AI.
+3. **Deploy Tests**: Your new tests will be deployed to our GitHub Actions.
+4. **Custom Test Creation**: Our support team will work with you to create tests for any features TestDriver might have missed.
+5. **Engineer Training**: We’ll train your engineers on best practices for creating and maintaining TestDriver tests.
+
+---
+
+## Onboarding Process
+
+| Service               | Timeline       | Description                                                                 |
+|-----------------------|----------------|-----------------------------------------------------------------------------|
+| **Custom Onboarding** | First 7 Days   | Get set up quickly with custom workflows and Prerun Scripts developed by the TestDriver team. |
+| **AI Test Generation**| First 7 Days   | Instant coverage: We’ll generate hundreds of tests for you to choose from.  |
+| **Custom Test Creation** | First 30 Days | Our team will create tests for any features TestDriver might have missed.   |
+| **Training**          | First 30 Days  | Our support team will train your engineers on best practices.               |
+| **Test Execution**    | Recurring      | Seamless deployment: Tests are executed on a schedule or via GitHub Actions.|
+
+---
+
+## Contract Details
+
+- **Free Trial**: Get 30 custom tests free during a 30-day trial when subscribing to the $995/month "30x30" plan.
+- **Ownership**: The tests are yours! You can modify, duplicate, or distribute them however you wish.
+- **Payment**: A payment method is required to begin the trial.
+- **Renewal**: The contract renews monthly.
+- **Cancellation**: Cancel anytime, for any reason.
+- **Credits**: Unused credits do not roll over and expire at the end of each month.
+- **Additional Usage**: Additional usage is billed at standard rates.
+
+---
+
+## Ready to Get Started?
+Don’t wait—this is a limited-time offer! [**Schedule your onboarding call now**](#) and let us help you achieve QA coverage in just 30 days.