testdriverai 4.2.0-test.0 → 4.2.0-test.2

package/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "npm" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

package/.github/workflows/test_interp.yml

@@ -20,6 +20,7 @@ jobs:
             1. /run test.yml
           prerun: |
             Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "${{ env.WEBSITE_URL }}"
+          version: test
 
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/agent.js

@@ -202,8 +202,11 @@ if (!commandHistory.length) {
   ];
 }
 
-const exit = async (failed = true) => {
-  await save();
+const exit = async (failed = true, shouldSave = false) => {
+  
+  if (shouldSave) {
+    await save();
+  }
 
   analytics.track("exit", { failed });
 
@@ -641,6 +644,16 @@ const actOnMarkdown = async (content, depth, pushToHistory = false) => {
 // simple function to backfill the chat history with a prompt and
 // then call `promptUser()` to get the user input
 const firstPrompt = async () => {
+
+  // should be start of new session
+  const sessionRes = await sdk.req("session/start", {
+    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
+    mousePosition: await system.getMousePosition(),
+    activeWindow: await system.activeWin(),
+  });
+
+  session.set(sessionRes.data.id);
+
   // readline is what allows us to get user input
   rl = readline.createInterface({
     terminal: true,
@@ -681,7 +694,7 @@ const firstPrompt = async () => {
     if (input.indexOf("/summarize") == 0) {
       await summarize();
     } else if (input.indexOf("/quit") == 0) {
-      await exit();
+      await exit(false, true);
     } else if (input.indexOf("/save") == 0) {
       await save({ filepath: commands[1] });
     } else if (input.indexOf("/undo") == 0) {
@@ -691,7 +704,7 @@ const firstPrompt = async () => {
     } else if (input.indexOf("/manual") == 0) {
       await manualInput(commands.slice(1).join(" "));
     } else if (input.indexOf("/run") == 0) {
-      await run(commands[1], commands[2], commands[3]);
+      await run(commands[1], commands[2] == "true", commands[3] == "true");
     } else if (input.indexOf("/generate") == 0) {
       await generate(commands[1], commands[2]);
     } else {
@@ -705,7 +718,7 @@ const firstPrompt = async () => {
   // if file exists, load it
   if (fs.existsSync(thisFile)) {
     analytics.track("load");
-    let object = await generator.ymlToHistory(
+    let object = await generator.hydrateFromYML(
       fs.readFileSync(thisFile, "utf-8"),
     );
 
@@ -835,7 +848,7 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
   }
 
   // write reply to /tmp/oiResult.log.log
-  let regression = await generator.historyToYml(executionHistory);
+  let regression = await generator.dumpToYML(executionHistory);
   try {
     fs.writeFileSync(filepath, regression);
   } catch (e) {
@@ -862,9 +875,10 @@ ${regression}
 // this will load a regression test from a file location
 // it parses the markdown file and executes the codeblocks exactly as if they were
 // generated by the AI in a single prompt
-let run = async (file, overwrite = false, shouldExit = true) => {
+let run = async (file, shouldSave = false, shouldExit = true) => {
 
   setTerminalWindowTransparency(true);
+  emitter.emit(events.interactive, false);
 
   log.log("info", chalk.cyan(`running ${file}...`));
 
@@ -884,10 +898,10 @@ let run = async (file, overwrite = false, shouldExit = true) => {
     await exit(true);
   }
 
-  console.log(yml)
-  console.log(process.env)
+  let interpolationVars = JSON.parse(process.env["TD_INTERPOLATION_VARS"] || '{}');
+
   // Inject environment variables into any ${VAR} strings
-  yml = parser.interpolate(yml, process.env);
+  yml = parser.interpolate(yml, interpolationVars);
 
   console.log(yml)
   let ymlObj = null;
@@ -937,13 +951,14 @@ ${yaml.dump(step)}
     await actOnMarkdown(markdown, 0, true);
   }
 
-  if (overwrite || overwrite == "true") {
+  if (shouldSave) {
     await save({ filepath: file });
   }
 
   setTerminalWindowTransparency(false);
+  emitter.emit(events.interactive, true);
 
-  if (shouldExit || shouldExit == "true") {
+  if (shouldExit) {
     await summarize();
     await exit(false);
   }
@@ -1059,15 +1074,6 @@ const start = async () => {
     console.log("");
   }
 
-  // should be start of new session
-  const sessionRes = await sdk.req("session/start", {
-    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
-    mousePosition: await system.getMousePosition(),
-    activeWindow: await system.activeWin(),
-  });
-
-  session.set(sessionRes.data);
-
   analytics.track("command", { command: thisCommand, file: thisFile });
 
   if (thisCommand == "edit") {

package/electron/overlay.html

@@ -246,7 +246,7 @@
       screenshotElement.classList.add('screenshot');
       setTimeout(() => {
         container.style.opacity = 1
-      }, 300)
+      }, 2000)
     });
 
     ipcRenderer.on(events.mouseClick,
@@ -305,4 +305,4 @@
   </script>
 </body>
 
-</html>
+</html>

package/lib/commands.js

@@ -160,15 +160,12 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
     return handleAssertResponse(response.data);
   }
 };
-const scroll = async (direction = "down", amount = 300, method = "keyboard") => {
+const scroll = async (direction = "down", amount = 300, method = "mouse") => {
   await redraw.start();
 
   amount = parseInt(amount);
 
   if (method === "mouse") {
-    // after experimenting, 200 is a good default for mouse, mostly as mouse will be called only when the user asks for it
-    // and that happens when keyboard scrolling cannot do things when there's a pop up that needs to be scrolled over and 
-    // pop ups are usually smaller and needs a smaller amount of scrolling
     amount = 200;
   }
 

package/lib/generator.js

@@ -2,6 +2,7 @@
 const yaml = require("js-yaml");
 const chalk = require("chalk");
 const package = require("../package.json");
+const session = require("./session");
 // do the actual parsing
 // this library is very strict
 // note that errors are sent to the AI will it may self-heal
@@ -52,25 +53,30 @@ const jsonToManual = function (json, colors = true) {
   return params;
 };
 
-const historyToYml = async function (inputArray) {
+const dumpToYML = async function (inputArray) {
+
   // use yml dump to convert json to yml
   let yml = await yaml.dump({
     version: package.version,
+    session: session.get(),
     steps: inputArray,
   });
 
   return yml;
 };
 
-const ymlToHistory = async function (yml) {
+const hydrateFromYML = async function (yml) {
   // use yml load to convert yml to json
   let json = await yaml.load(yml);
+
+  session.set(json.session);
+
   return json;
 };
 
 module.exports = {
   manualToYml,
-  historyToYml,
-  ymlToHistory,
+  dumpToYML,
+  hydrateFromYML,
   jsonToManual,
 };

package/lib/sdk.js

@@ -106,7 +106,7 @@ const req = async (path, data, onChunk) => {
     responseType: typeof onChunk === "function" ? "stream" : "json",
     data: {
       ...data,
-      session: session.get()?.id,
+      session: session.get(),
       stream: typeof onChunk === "function",
     },
   };

package/lib/session.js

@@ -5,6 +5,8 @@ module.exports = {
     return session;
   },
   set: (s) => {
-    session = s;
+    if (s) {
+      session = s;
+    }
   },
 };

package/lib/system.js

@@ -2,12 +2,11 @@
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
-const screenshot = require("screenshot-desktop");
 const si = require("systeminformation");
-const activeWindow = require("active-win");
 const robot = require("robotjs");
 const sharp = require("sharp");
 const { emitter, events } = require("./events.js");
+const { Monitor } = require("node-screenshots");
 
 let primaryDisplay = null;
 
@@ -48,7 +47,11 @@ const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
     let step1 = tmpFilename();
     let step2 = tmpFilename();
 
-    await screenshot({ filename: step1, format: "png" });
+    const monitors = Monitor.all();
+    const primaryMonitor = monitors.find(monitor => monitor.isPrimary);
+    const image = await primaryMonitor.captureImage(); // Capture the image asynchronously
+    const buffer = await image.toPng(); // Convert the image to PNG format
+    fs.writeFileSync(step1, buffer); // Save the image to a file
 
     // Fetch the mouse position
     const mousePos = robot.getMousePos();
@@ -111,9 +114,25 @@ const platform = () => {
   return platform;
 };
 
+// Import get-windows using dynamic import for ES module compatibility
+let activeWindowFn = null;
+const initializeActiveWindow = async () => {
+  if (!activeWindowFn) {
+    const { activeWindow } = await import('get-windows');
+    activeWindowFn = activeWindow;
+  }
+  return activeWindowFn;
+};
+
 // this is the focused window
 const activeWin = async () => {
-  return await activeWindow();
+  try {
+    const activeWindow = await initializeActiveWindow();
+    return await activeWindow();
+  } catch (error) {
+    console.error('Error getting active window:', error);
+    return null;
+  }
 };
 
 const getMousePosition = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.2.0-test.0",
+  "version": "4.2.0-test.2",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {
@@ -17,7 +17,6 @@
   "license": "ISC",
   "dependencies": {
     "@electerm/strip-ansi": "^1.0.0",
-    "active-win": "^8.2.1",
     "axios": "^1.7.7",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",
@@ -25,6 +24,7 @@
     "decompress": "^4.2.1",
     "dotenv": "^16.4.5",
     "electron": "^33.0.2",
+    "get-windows": "^9.2.0",
     "jimp": "^0.22.12",
     "js-yaml": "^4.1.0",
     "mac-screen-capture-permissions": "^2.1.0",
@@ -34,14 +34,13 @@
     "marky": "^1.2.5",
     "node-ipc": "^12.0.0",
     "node-notifier": "^10.0.1",
+    "node-screenshots": "0.2.1",
     "odiff-bin": "^3.1.2",
     "prompts": "^2.4.2",
     "remark-parse": "^11.0.0",
-    "rimraf": "^5.0.5",
     "robotjs": "^0.6.0",
     "sanitize-filename": "^1.6.3",
     "say": "^0.16.0",
-    "screenshot-desktop": "^1.15.0",
     "semver": "^7.6.2",
     "sharp": "^0.33.5",
     "systeminformation": "^5.23.5",
@@ -49,16 +48,19 @@
     "uuid": "^10.0.0",
     "winston": "^3.13.0"
   },
+  "overrides": {
+    "glob": "^11.0.1",
+    "rimraf": "^5.0.10"
+  },
   "devDependencies": {
     "@eslint/js": "^9.10.0",
+    "chai": "^5.1.2",
     "esbuild": "0.20.2",
     "esbuild-plugin-fileloc": "^0.0.6",
     "eslint": "^9.10.0",
     "globals": "^15.9.0",
-    "node-addon-api": "^8.0.0",
-    "node-gyp": "^10.1.0",
     "mocha": "^10.8.2",
-    "chai": "^5.1.2",
+    "node-addon-api": "^8.0.0",
     "prettier": "3.3.3"
   },
   "optionalDependencies": {