testdriverai 4.2.20 → 5.0.0-beta.2

package/.github/dependabot.yml

@@ -1,7 +1,7 @@
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yaml-file
 
 version: 2
 updates:

package/.github/workflows/test_interp.yml

@@ -17,7 +17,7 @@ jobs:
           branch: main
           key: ${{secrets.TESTDRIVER_API_KEY}}
           prompt: |
-            1. /run test.yml
+            1. /run test.yaml
           prerun: |
             Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "${{ env.WEBSITE_URL }}"
 

package/.github/workflows/testdriver.yml

@@ -0,0 +1,28 @@
+name: TestDriver.ai
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: "TestDriver"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: testdriverai/action@main
+        with:
+          key: ${{secrets.TESTDRIVER_API_KEY}}
+          prompt: |
+            1. /run testdriver/testdriver.yml
+          prerun: |
+            cd $env:TEMP
+            npm init -y
+            npm install dashcam-chrome
+            Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "${{ env.WEBSITE_URL }}"
+            exit
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FORCE_COLOR: "3"
+          WEBSITE_URL: "https://example.com" # Define the website URL here

package/README.md

@@ -60,7 +60,7 @@ npm install testdriverai -g
 Let's show TestDriver what we want to test. Run the following command:
 
 ```sh
-testdriverai .testdriver/test.yml
+testdriverai .testdriver/test.yaml
 ```
 
 ## Reset the test state
@@ -131,7 +131,7 @@ If something didn't work, you can use `/undo` to remove all of the test steps ad
 Now it's time to make sure the test plan works before we deploy it. Use testdriver run to run the test file you just created with /save .
 
 ```sh
-testdriverai run testdriver/test.yml
+testdriverai run testdriver/test.yaml
 ```
 
 Make sure to reset the test state!
@@ -147,3 +147,4 @@ gh pr create --web
 ```
 
 Your test will run on every commit and the results will be posted as a Dashcam.io video within your GitHub summary! Learn more about deploying on CI [here](https://docs.testdriver.ai/continuous-integration/overview).
+# vscode

package/agent.js

@@ -31,6 +31,7 @@ const sanitizeFilename = require("sanitize-filename");
 const macScreenPerms = require("mac-screen-capture-permissions");
 
 // local modules
+const wss = require("./lib/websockets.js");
 const speak = require("./lib/speak.js");
 const analytics = require("./lib/analytics.js");
 const log = require("./lib/logger.js");
@@ -42,6 +43,7 @@ const sdk = require("./lib/sdk.js");
 const commands = require("./lib/commands.js");
 const init = require("./lib/init.js");
 const config = require("./lib/config.js");
+const sandbox = require("./lib/sandbox.js");
 
 const { showTerminal, hideTerminal } = require("./lib/focus-application.js");
 const isValidVersion = require("./lib/valid-version.js");
@@ -106,14 +108,14 @@ let getArgs = () => {
       fs.mkdirSync(testdriverFolder);
     }
 
-    args[file] = "testdriver/testdriver.yml";
+    args[file] = "testdriver/testdriver.yaml";
   }
 
   // turn args[file] into local path
   if (args[file]) {
-    args[file] = `${process.cwd()}/${args[file]}`;
-    if (!args[file].endsWith(".yml")) {
-      args[file] += ".yml";
+    args[file] = path.join(process.cwd(), args[file]);
+    if (!args[file].endsWith(".yaml")) {
+      args[file] += ".yaml";
     }
   }
 
@@ -167,7 +169,7 @@ function fileCompleter(line) {
 }
 
 function completer(line) {
-  let completions = "/summarize /save /run /quit /assert /undo /manual".split(
+  let completions = "/summarize /save /run /quit /assert /undo /manual /yml".split(
     " ",
   );
   if (line.startsWith("/run ")) {
@@ -266,6 +268,7 @@ const haveAIResolveError = async (error, markdown, depth = 0, undo = true) => {
 
   speak("thinking...");
   notify("thinking...");
+
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
@@ -374,14 +377,16 @@ const runCommand = async (command, depth) => {
 let lastCommand = new Date().getTime();
 let csv = [["command,time"]];
 
-const executeCommands = async (commands, depth, pushToHistory = false) => {
+const executeCommands = async (commands, depth, pushToHistory = false, dry = false) => {
   if (commands?.length) {
     for (const command of commands) {
       if (pushToHistory) {
         executionHistory[executionHistory.length - 1]?.commands.push(command);
       }
 
-      await runCommand(command, depth);
+      if (!dry) {
+        await runCommand(command, depth);
+      }
 
       let timeToComplete = (new Date().getTime() - lastCommand) / 1000;
       // logger.info(timeToComplete, 'seconds')
@@ -394,7 +399,7 @@ const executeCommands = async (commands, depth, pushToHistory = false) => {
 
 // note that commands are run in a recursive loop, so that the AI can respond to the output of the commands
 // like `click-image` and `click-text` for example
-const executeCodeBlocks = async (codeblocks, depth, pushToHistory = false) => {
+const executeCodeBlocks = async (codeblocks, depth, pushToHistory = false, dry = false) => {
   depth = depth + 1;
 
   logger.debug("%j", { message: "execute code blocks", depth });
@@ -412,20 +417,20 @@ const executeCodeBlocks = async (codeblocks, depth, pushToHistory = false) => {
       );
     }
 
-    await executeCommands(commands, depth, pushToHistory);
+    await executeCommands(commands, depth, pushToHistory, dry);
   }
 };
 
 // this is the main function that interacts with the ai, runs commands, and checks the results
 // notice that depth is 0 here. when this function resolves, the task is considered complete
 // notice the call to `check()` which validates the prompt is complete
-const aiExecute = async (message, validateAndLoop = false) => {
+const aiExecute = async (message, validateAndLoop = false, dry = false) => {
   executionHistory.push({ prompt: lastPrompt, commands: [] });
 
   logger.debug("kicking off exploratory loop");
-
+  
   // kick everything off
-  await actOnMarkdown(message, 0, true);
+  await actOnMarkdown(message, 0, true, dry);
 
   if (validateAndLoop) {
     logger.debug("exploratory loop resolved, check your work");
@@ -509,7 +514,7 @@ const assert = async (expect) => {
   logger.info(chalk.dim("thinking..."), true);
   logger.info("");
 
-  let response = `\`\`\`yml
+  let response = `\`\`\`yaml
 commands:
   - command: assert
     expect: ${expect}
@@ -522,11 +527,11 @@ commands:
 
 // this function responds to the result of `promptUser()` which is the user input
 // it kicks off the exploratory loop, which is the main function that interacts with the AI
-const humanInput = async (currentTask, validateAndLoop = false) => {
+const exploratoryLoop = async (currentTask, dry = false, validateAndLoop = false) => {
   lastPrompt = currentTask;
   checkCount = 0;
 
-  logger.debug("humanInput called");
+  logger.debug("exploratoryLoop called");
 
   tasks.push(currentTask);
 
@@ -554,11 +559,11 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
   );
   mdStream.end();
 
-  await aiExecute(message.data, validateAndLoop);
+  await aiExecute(message.data, validateAndLoop, dry);
 
-  logger.debug("showing prompt from humanInput response check");
+  logger.debug("showing prompt from exploratoryLoop response check");
 
-  await save({ silent: true });
+  await save({ silent: false });
 };
 
 const generate = async (type, count, baseYaml, skipYaml = false) => {
@@ -663,7 +668,7 @@ ${yml}
 };
 
 // this function is responsible for starting the recursive process of executing codeblocks
-const actOnMarkdown = async (content, depth, pushToHistory = false) => {
+const actOnMarkdown = async (content, depth, pushToHistory = false, dry = false) => {
   logger.debug("%j", {
     message: "actOnMarkdown called",
     depth,
@@ -678,7 +683,7 @@ const actOnMarkdown = async (content, depth, pushToHistory = false) => {
   }
 
   if (codeblocks.length) {
-    let executions = await executeCodeBlocks(codeblocks, depth, pushToHistory);
+    let executions = await executeCodeBlocks(codeblocks, depth, pushToHistory, dry);
     return executions;
   } else {
     return true;
@@ -720,7 +725,7 @@ const firstPrompt = async () => {
 
   // this is how we parse user input
   // notice that the AI is only called if the input is not a command
-  rl.on("line", async (input) => {
+  const handleInput = async (input) => {
     if (!isInteractive) return;
     if (!input.trim().length) return promptUser();
 
@@ -789,12 +794,22 @@ const firstPrompt = async () => {
     } else if (input.indexOf("/generate") == 0) {
       const skipYaml = commands[4] === "--skip-yaml";
       await generate(commands[1], commands[2], commands[3], skipYaml);
+    } else if (input.indexOf("/dry") == 0) {
+      await exploratoryLoop(input.replace('/dry', ''), true, false);
+    } else if (input.indexOf("/yaml") == 0) {
+      await runRawYML(commands[1]);
     } else {
-      await humanInput(input, true);
+      await exploratoryLoop(input, false, true);
     }
 
     setTerminalWindowTransparency(false);
     promptUser();
+  };
+
+  rl.on("line", handleInput);
+
+  wss.addEventListener("input", async (message) => {
+    handleInput(message.data);
   });
 
   // if file exists, load it
@@ -923,11 +938,12 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
   analytics.track("save", { silent });
 
   if (!silent) {
-    logger.info(chalk.dim("saving..."), true);
+    logger.info(chalk.dim(`saving as ${filepath}...`), true);
     logger.info("");
   }
 
   if (!executionHistory.length) {
+    console.log('no exeuction history, not saving')
     return;
   }
 
@@ -936,6 +952,7 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
   try {
     fs.writeFileSync(filepath, regression);
   } catch (e) {
+    console.log(e)
     logger.error(e.message);
     logger.error("%s", e);
   }
@@ -956,15 +973,33 @@ ${regression}
   }
 };
 
+let runRawYML = async (yml) => {
+
+  const tmp = require("tmp");
+  let tmpobj = tmp.fileSync();
+
+  let decoded = decodeURIComponent(yml);
+
+  // saved the yml to a temp file using tmp
+  // and run it with run()
+  fs.writeFileSync(tmpobj.name, await generator.rawToFormatted(decoded));
+  
+  await run(tmpobj.name, false, true);
+
+}
+
 // this will load a regression test from a file location
 // it parses the markdown file and executes the codeblocks exactly as if they were
 // generated by the AI in a single prompt
-let run = async (file, shouldSave = false, shouldExit = true) => {
+let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
   await newSession();
 
   setTerminalWindowTransparency(true);
   emitter.emit(events.interactive, false);
 
+  // get the current wowrking directory where this file is being executed
+  let cwd = process.cwd();
+
   logger.info(chalk.cyan(`running ${file}...`));
 
   let ymlObj = await loadYML(file);
@@ -1017,6 +1052,7 @@ ${yaml.dump(step)}
 };
 
 const promptUser = () => {
+  wss.sendToClients("done");
   emitter.emit(events.interactive, true);
   rl.prompt(true);
 };
@@ -1122,9 +1158,11 @@ const start = async () => {
   analytics.track("command", { command: thisCommand, file: thisFile });
 
   if (thisCommand == "edit") {
+    await makeSandbox();
     firstPrompt();
   } else if (thisCommand == "run") {
     errorLimit = 100;
+    await makeSandbox();
     run(thisFile);
   } else if (thisCommand == "init") {
     await init();
@@ -1132,6 +1170,34 @@ const start = async () => {
   }
 };
 
+const makeSandbox = async () => {
+
+  if (config.TD_VM) {
+          
+    logger.info(chalk.gray(`- creating linux sandbox...`));
+    await sandbox.boot();
+    logger.info(chalk.gray(`- authenticating...`));
+    await sandbox.send({type: 'authenticate', apiKey: config.TD_API_KEY });
+    logger.info(chalk.gray(`- setting up...`));
+    await sandbox.send({type: 'create', resolution: [1024, 768]});
+    logger.info(chalk.gray(`- starting stream...`));
+    await sandbox.send({type: 'stream.start'});
+    let {url} = await sandbox.send({type: 'stream.getUrl'});
+    logger.info(chalk.gray(`- rendering...`));
+    await sandbox.send({type: 'ready'});
+    logger.info(chalk.gray(`- booting...`));
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    logger.info(chalk.green(``));
+    logger.info(chalk.green(`sandbox runner ready!`));
+    emitter.emit(events.vm.show, {url}); 
+
+  }
+
+  emitter.emit(events.interactive, false);
+  emitter.emit(events.showWindow)
+
+}
+
 process.on("uncaughtException", async (err) => {
   analytics.track("uncaughtException", { err });
   logger.error("Uncaught Exception: %s", err);

package/electron/overlay.html

@@ -2,12 +2,20 @@
 <html>
 
 <head>
+  <link rel="preconnect" href="https://fonts.googleapis.com">
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+  <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;1,100;1,200;1,300;1,400;1,500;1,600;1,700&display=swap" rel="stylesheet">
+  <title>TestDriver</title>
   <style>
     * {
       margin: 0;
       padding: 0;
       overflow: hidden;
       box-sizing: border-box;
+      font-family: "IBM Plex Mono", monospace;
+      font-weight: 400;
+      font-style: normal;
+      font-size: 14px;
     }
 
     @keyframes animate-glow {
@@ -64,11 +72,14 @@
       position: relative;
     }
 
+    #main {
+      pointer-events: none;
+    }
+
     .screenshot {
       position: absolute;
       inset: 0;
-      border: 8px dashed #B0CF34;
-      border-radius: 20px;
+      z-index: 1;
       opacity: 0;
       animation-duration: 0.3s;
       animation-delay: 0;
@@ -77,6 +88,7 @@
       animation-name: animate-screenshot;
       animation-timing-function: ease;
       animation-fill-mode: forwards;
+      background-color: white;
     }
 
     .box {
@@ -90,6 +102,7 @@
       animation-name: animate-glow;
       animation-timing-function: ease;
       animation-fill-mode: forwards;
+      border-radius: 5px;
     }
 
     .container {
@@ -150,6 +163,7 @@
     }
 
     #terminal-wrapper {
+      pointer-events: none;
       position: absolute;
       left: calc(100vw - 700px);
       top: 0px;
@@ -159,7 +173,7 @@
       opacity: 0;
       z-index: -1;
       overflow-y: auto;
-      padding: 4px;
+      padding: 20px;
     }
 
     #terminal {
@@ -169,11 +183,22 @@
     }
 
     #boxes {
+      pointer-events: none;
       position: absolute;
       top: 0;
       left: 0;
       right: 0;
       bottom: 0;
+      z-index: 2;
+    }
+
+    #vm-iframe {
+      position: absolute;
+      top: 0;
+      left: 0;
+      width: 100%;
+      height: 100%;
+      border: none;
     }
   </style>
   <link rel="stylesheet" href="terminal/xterm.css" />
@@ -182,15 +207,19 @@
 </head>
 
 <body>
+  
   <div id="screenshot"></div>
   <div id="main" class="container">
     <div id="boxes">
       <div id="pointer"></div>
     </div>
     <div id="terminal-wrapper">
+      <img src="td.png" alt="td" style="position: absolute; top: 20; right: 20; height: 40px; z-index: 9999; background-color: black;">
       <div id="terminal"></div>
     </div>
   </div>
+  <iframe id="vm-iframe" frameborder="0"></iframe>
+  
   <script>
 
     const { ipcRenderer } = require("electron");
@@ -226,7 +255,6 @@
       });
     }
 
-
     ipcRenderer.on(events.screenCapture.start, (event, data) => {
       if (data?.silent) return
       // Hide everything whlie the app takes the screenshot
@@ -279,7 +307,7 @@
       cursorInactiveStyle: "none",
       scrollback: 9999999,
       allowProposedApi: true,
-      fontSize: 18
+      fontSize: 14
     });
 
     const fitAddon = new FitAddon.FitAddon();
@@ -287,6 +315,11 @@
     terminal.open(terminalElement);
     fitAddon.fit();
 
+    ipcRenderer.on(events.vm.show, (event, data) => {
+      const iframe = document.querySelector("#vm-iframe");
+      iframe.src = data.url;
+    });
+
     ipcRenderer.on(events.terminal.stdout, (event, data) =>
       terminal.write(data),
     );

package/electron/overlay.js

@@ -1,6 +1,7 @@
 const ipc = require("node-ipc").default;
 const { app, screen, BrowserWindow } = require("electron");
 const { eventsArray } = require("../lib/events.js");
+const config = require("../lib/config.js");
 
 ipc.config.id = "testdriverai_overlay";
 ipc.config.retry = 1500;
@@ -9,47 +10,79 @@ ipc.config.silent = true;
 app.whenReady().then(() => {
   app.dock?.hide();
 
-  const windowOptions = {
-    ...screen.getPrimaryDisplay().bounds,
-    enableLargerThanScreen: true,
-    frame: false,
-    show: false,
-    closable: false,
-    resizable: false,
-    focusable: false,
-    fullscreenable: true,
-    transparent: true,
-    alwaysOnTop: true,
-    skipTaskbar: true,
-    webPreferences: {
-      nodeIntegration: true,
-      contextIsolation: false,
-    },
-    autoHideMenuBar: true,
-  };
+  let windowOptions;
+
+  if (config.TD_VM) {
+
+    windowOptions = {
+      width: 1030,
+      height: 800,
+      closable: true,
+      resizable: true,
+      show: false,
+      alwaysOnTop: true,
+      webPreferences: {
+        nodeIntegration: true,
+        contextIsolation: false,
+      },
+      autoHideMenuBar: true,
+    };
+
+  } else {
+
+    windowOptions = {
+      ...screen.getPrimaryDisplay().bounds,
+      closable: true,
+      resizable: true,
+      alwaysOnTop: true,
+      enableLargerThanScreen: true,
+      frame: false,
+      show: false,
+      focusable: false,
+      fullscreenable: true,
+      transparent: true,
+      skipTaskbar: true,
+      webPreferences: {
+        nodeIntegration: true,
+        contextIsolation: false,
+      },
+      autoHideMenuBar: true,
+    };
+
+  }
 
   if (process.platform !== 'darwin') {
     windowOptions.fullscreen = true;
   }
 
   const window = new BrowserWindow(windowOptions);
-  window.setIgnoreMouseEvents(true);
-  window.setAlwaysOnTop(true, "screen-saver");
-  window.setVisibleOnAllWorkspaces(true, {
-    visibleOnFullScreen: true,
-  });
+
+  if (!config.TD_VM) {
+    window.setIgnoreMouseEvents(true);
+    window.setAlwaysOnTop(true, "screen-saver");
+    window.setVisibleOnAllWorkspaces(true, {
+      visibleOnFullScreen: true,
+    });
+  } else {
+    window.setBackgroundColor('#000')
+  }
+  
   window.loadFile("overlay.html");
 
   window.once('ready-to-show', () => {
-    window.showInactive();
+    // window.showInactive();
   });
 
   // open developer tools
-  // window.webContents.openDevTools();
+  window.webContents.openDevTools();
 
   ipc.serve(() => {
     for (const event of eventsArray) {
       ipc.server.on(event, (data) => {
+        if (event === "show-window") {
+          window.showInactive();
+          return;
+        }
         window?.webContents.send(event, data);
       });
     }

package/index.js

@@ -13,6 +13,7 @@ const { logger } = require("./lib/logger.js");
     agent.setTerminalApp(win);
     agent.start();
   } else {
+
     // Intercept all stdout and stderr calls (works with console as well)
     const originalStdoutWrite = process.stdout.write.bind(process.stdout);
     process.stdout.write = (...args) => {
@@ -35,9 +36,8 @@ const { logger } = require("./lib/logger.js");
     };
   
     require("./lib/overlay.js")
-      .electronProcessPromise.then(() => {
+      .electronProcessPromise.then(async () => {
         let agent = require("./agent.js");
-        agent.setTerminalApp(win);
         agent.start();
       })
       .catch((err) => {

package/lib/commander.js

@@ -25,6 +25,9 @@ const replaceOutputs = (obj) => {
 // object is a json representation of the individual yml command
 // the process turns markdown -> yml -> json -> js function execution
 const run = async (object, depth) => {
+
+try {
+
   logger.debug("%j", { object, depth });
 
   // success returns null
@@ -44,7 +47,7 @@ Our test structure is in YAML format.
 
 Here is a simple example:
 
-\`\`\`yml
+\`\`\`yaml
 commands:
   - command: press-keys
     keys: [command, space]
@@ -216,6 +219,11 @@ commands:
   ]);
 
   return response;
-};
+
+} catch (error) {
+  logger.error(error);
+  throw error;
+}
+}
 
 module.exports = { run };