testdriverai 4.1.12 → 4.1.14

package/agent.js

@@ -72,8 +72,6 @@ const args = process.argv.slice(2);
 
 const commandHistoryFile = path.join(os.homedir(), ".testdriver_history");
 
-const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
-
 let getArgs = () => {
   let command = 0;
   let file = 1;
@@ -256,6 +254,7 @@ const haveAIResolveError = async (error, markdown, depth = 0, undo = false) => {
   speak("thinking...");
   notify("thinking...");
   log.log("info", chalk.dim("thinking..."), true);
+  log.log("info", "");
 
   let response = await sdk.req("error", {
     description: eMessage,
@@ -280,9 +279,9 @@ const check = async () => {
     return await exit(true);
   }
 
-  await delay(3000);
-
+  log.log("info", "");
   log.log("info", chalk.dim("checking..."), "testdriver");
+  log.log("info", "");
 
   let thisScreenshot = await system.captureScreenBase64();
   let images = [lastScreenshot, thisScreenshot];
@@ -452,6 +451,7 @@ const assert = async (expect) => {
   speak("thinking...");
   notify("thinking...");
   log.log("info", chalk.dim("thinking..."), true);
+  log.log("info", "");
 
   let response = `\`\`\`yml
 commands:
@@ -477,7 +477,6 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
   speak("thinking...");
   notify("thinking...");
   log.log("info", chalk.dim("thinking..."), true);
-
   log.log("info", "");
 
   lastScreenshot = await system.captureScreenBase64();
@@ -492,6 +491,7 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
       image: lastScreenshot,
     },
     (chunk) => {
+
       if (chunk.type === "data") {
         mdStream.log(chunk.data);
       }
@@ -513,7 +513,6 @@ const generate = async (type, count) => {
   notify("thinking...");
 
   log.log("info", chalk.dim("thinking..."), true);
-
   log.log("info", "");
 
   let image = await system.captureScreenBase64();
@@ -649,8 +648,7 @@ const firstPrompt = async () => {
     if (!input.trim().length) return promptUser();
 
     emitter.emit(events.interactive, false);
-    await setTerminalApp();
-    // setTerminalWindowTransparency(true);
+    setTerminalWindowTransparency(true);
     errorCounts = {};
 
     // append this to commandHistoryFile
@@ -683,7 +681,7 @@ const firstPrompt = async () => {
       await humanInput(input, true);
     }
 
-    // setTerminalWindowTransparency(false);
+    setTerminalWindowTransparency(false);
     promptUser();
   });
 
@@ -726,6 +724,7 @@ New commands will be appended.
 };
 
 let setTerminalWindowTransparency = async (hide) => {
+
   if (hide) {
     try {
       http
@@ -752,6 +751,7 @@ let setTerminalWindowTransparency = async (hide) => {
     return;
   }
 
+
   try {
     if (hide) {
       if (terminalApp) {
@@ -775,7 +775,6 @@ let summarize = async (error = null) => {
 
   log.log("info", "");
 
-  log.log("info", chalk.cyan("summarizing"));
   log.log("info", chalk.dim("reviewing test..."), true);
 
   // let text = prompts.summarize(tasks, error);
@@ -937,9 +936,9 @@ const promptUser = () => {
   rl.prompt(true);
 };
 
-const setTerminalApp = async () => {
+const setTerminalApp = async (win) => {
+
   if (terminalApp) return;
-  let win = await system.activeWin();
   if (process.platform === "win32") {
     terminalApp = win?.title || "";
   } else {
@@ -999,7 +998,8 @@ const embed = async (file, depth) => {
   log.log("info", `${file} (end)`);
 };
 
-(async () => {
+const start = async () => {
+  
   // console.log(await  system.getPrimaryDisplay());
 
   // @todo add-auth
@@ -1043,7 +1043,6 @@ const embed = async (file, depth) => {
     console.log("");
   }
 
-  await setTerminalApp();
 
   // should be start of new session
   const sessionRes = await sdk.req("session/start", {
@@ -1064,7 +1063,7 @@ const embed = async (file, depth) => {
   } else if (thisCommand == "init") {
     init();
   }
-})();
+};
 
 process.on("uncaughtException", async (err) => {
   analytics.track("uncaughtException", { err });
@@ -1079,3 +1078,8 @@ process.on("unhandledRejection", async (reason, promise) => {
   // Optionally, you might want to exit the process
   await exit(true);
 });
+
+module.exports = {
+  setTerminalApp,
+  start
+};

package/electron/overlay.js

@@ -39,7 +39,10 @@ app.whenReady().then(() => {
     visibleOnFullScreen: true,
   });
   window.loadFile("overlay.html");
-  window.show();
+
+  window.once('ready-to-show', () => {
+    window.showInactive();
+  });
 
   // open developer tools
   // window.webContents.openDevTools();

package/index.js

@@ -1,37 +1,49 @@
 #!/usr/bin/env node
 const config = require("./lib/config.js");
+const system = require("./lib/system.js");
 const { emitter, events } = require("./lib/events.js");
 
-if (!config.TD_OVERLAY) {
-  require("./agent.js");
-} else {
-  // Intercept all stdout and stderr calls (works with console as well)
-  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
-  process.stdout.write = (...args) => {
-    const [data, encoding] = args;
-    emitter.emit(
-      events.terminal.stdout,
-      data.toString(typeof encoding === "string" ? encoding : undefined),
-    );
-    originalStdoutWrite(...args);
-  };
+(async () => {
 
-  const originalStderrWrite = process.stderr.write.bind(process.stderr);
-  process.stderr.write = (...args) => {
-    const [data, encoding] = args;
-    emitter.emit(
-      events.terminal.stderr,
-      data.toString(typeof encoding === "string" ? encoding : undefined),
-    );
-    originalStderrWrite(...args);
-  };
+  let win = await system.activeWin();
 
-  require("./lib/overlay.js")
-    .electronProcessPromise.then(() => {
-      require("./agent.js");
-    })
-    .catch((err) => {
-      console.error(err);
-      process.exit(1);
-    });
-}
+  if (!config.TD_OVERLAY) {
+    let agent = require("./agent.js");
+    agent.setTerminalApp(win);
+    agent.start();
+  } else {
+    // Intercept all stdout and stderr calls (works with console as well)
+    const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+    process.stdout.write = (...args) => {
+      const [data, encoding] = args;
+      emitter.emit(
+        events.terminal.stdout,
+        data.toString(typeof encoding === "string" ? encoding : undefined),
+      );
+      originalStdoutWrite(...args);
+    };
+  
+    const originalStderrWrite = process.stderr.write.bind(process.stderr);
+    process.stderr.write = (...args) => {
+      const [data, encoding] = args;
+      emitter.emit(
+        events.terminal.stderr,
+        data.toString(typeof encoding === "string" ? encoding : undefined),
+      );
+      originalStderrWrite(...args);
+    };
+  
+    require("./lib/overlay.js")
+      .electronProcessPromise.then(() => {
+        let agent = require("./agent.js");
+        agent.setTerminalApp(win);
+        agent.start();
+      })
+      .catch((err) => {
+        console.error(err);
+        process.exit(1);
+      });
+  }
+
+  
+})()

package/lib/commander.js

@@ -7,6 +7,7 @@ const speak = require("./speak");
 const notify = require("./notify");
 const analytics = require("./analytics");
 const marky = require("marky");
+const sdk = require("./sdk");
 
 // object is a json representation of the individual yml command
 // the process turns markdown -> yml -> json -> js function execution
@@ -181,7 +182,11 @@ commands:
   }
 
   let timing = marky.stop(object.command);
-  await analytics.track("command", { data: object, depth, timing });
+  
+  await Promise.all([
+    sdk.req('ran', { command: object.command, data: object }),
+    analytics.track("command", { data: object, depth, timing })
+  ]);
 
   return response;
 };

package/lib/commands.js

@@ -139,6 +139,7 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
   // take a screenshot
   log("info", "");
   log("info", chalk.dim("thinking..."), true);
+  log("info", "");
 
   if (async) {
     await sdk
@@ -200,6 +201,7 @@ const click = async (x, y, button = "left", click = "single") => {
   await delay(1000); // wait for the mouse to move
   robot.mouseClick(button, double);
   emitter.emit(events.mouseClick, { x, y, button, click });
+
   await redraw.wait(5000);
   return;
 };
@@ -211,6 +213,7 @@ const hover = async (x, y) => {
   y = parseInt(y);
 
   await robot.moveMouseSmooth(x, y, 0.1);
+
   await redraw.wait(2500);
 
   return;
@@ -237,6 +240,7 @@ let commands = {
 
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     const mdStream = logger.createMarkdownStreamLogger();
     let response = await sdk.req(
@@ -277,6 +281,7 @@ let commands = {
     // take a screenshot
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     const mdStream = logger.createMarkdownStreamLogger();
     let response = await sdk.req(

package/lib/config.js

@@ -28,7 +28,7 @@ const config = {
   TD_API_ROOT: "https://api.testdriver.ai",
   TD_DEV: parseValue(process.env["DEV"]),
   TD_PROFILE: false,
-  TD_OVERLAY: true,
+  TD_OVERLAY: true
 };
 
 // Find all env vars starting with TD_

package/lib/focus-application.js

@@ -3,20 +3,25 @@ const path = require("path");
 const { execSync } = require("child_process");
 const { platform } = require("./system");
 const scriptPath = path.join(__dirname, "focusWindow.ps1");
+const robot = require("robotjs");
 
 // apple script that focuses on a window
-const appleScriptFocus = (windowName) => `
+const appleScriptSetFrontmost = (windowName) => `
 tell application "System Events" to tell process "${windowName}"
     set frontmost to true
 end tell`;
 
-const appleScriptMin = (windowName) => `
-tell application "System Events" 
-    set value of attribute "AXMinimized" of every window of application process "${windowName}" to true
-end tell`;
+// const appleScriptAXMinimized = (windowName) => `
+// tell application "System Events" 
+//     set value of attribute "AXMinimized" of every window of application process "${windowName}" to true
+// end tell`;
+
+// const appleScriptActivate = (windowName) => `
+// tell application "${windowName}" to activate
+// `;
 
-const appleScriptMax = (windowName) => `
-tell application "${windowName}" to activate
+const appleScriptOpen = (windowName) => `
+open -a "${windowName}"
 `;
 
 const runPwsh = (appName, method) => {
@@ -27,7 +32,7 @@ const runPwsh = (appName, method) => {
 async function focusApplication(appName) {
   try {
     if (platform() == "mac") {
-      return await execSync(`osascript -e '${appleScriptFocus(appName)}'`);
+      return await execSync(`osascript -e '${appleScriptSetFrontmost(appName)}'`);
     } else if (platform() == "linux") {
       // TODO: This needs fixing
       return;
@@ -42,7 +47,7 @@ async function focusApplication(appName) {
 async function hideTerminal(appName) {
   try {
     if (platform() == "mac") {
-      return await execSync(`osascript -e '${appleScriptMin(appName)}'`);
+      robot.keyTap('m', ['command']);
     } else if (platform() == "windows") {
       return runPwsh(appName, "Minimize");
     }
@@ -54,7 +59,7 @@ async function hideTerminal(appName) {
 async function showTerminal(appName) {
   try {
     if (platform() == "mac") {
-      return await execSync(`osascript -e '${appleScriptMax(appName)}'`);
+      return await execSync(appleScriptOpen(appName));
     } else if (platform() == "windows") {
       return runPwsh(appName, "Restore");
     }

package/lib/logger.js

@@ -44,7 +44,7 @@ const markedParsePartial = (markdown, start = 0, end = 0) => {
 
 const createMarkdownStreamLogger = () => {
   let buffer = "";
-  console.log("");
+
   return {
     log: (chunk) => {
       if (typeof chunk !== "string") {
@@ -65,10 +65,11 @@ const createMarkdownStreamLogger = () => {
       }
     },
     end() {
-      const previousConsoleOutput = markedParsePartial(buffer, 0, -1);
 
-      const consoleOutput = markedParsePartial(buffer);
+      const previousConsoleOutput = markedParsePartial(buffer, 0, -1);
+      const consoleOutput = markedParsePartial(buffer, 0, Infinity);
       const diff = consoleOutput.replace(previousConsoleOutput, "");
+
       if (diff) {
         process.stdout.write(diff);
       }

package/lib/overlay.js

@@ -10,7 +10,7 @@ ipc.config.silent = true;
 
 let electronProcess;
 
-console.log("Starting overlay process...");
+console.log("Spawning GUI...");
 
 try {
   // Resolve the path to Electron CLI

package/lib/redraw.js

@@ -2,6 +2,7 @@ const { captureScreenPNG } = require("./system");
 const os = require("os");
 const path = require("path");
 const { compare } = require("odiff-bin");
+const logger = require("./logger");
 
 // network
 const si = require('systeminformation');
@@ -60,16 +61,6 @@ async function updateNetwork() {
       } else {
         networkSettled = false;
       }
-    
-      if (process.env["DEV"]) {
-
-        if (!networkSettled) {
-          console.log(chalk.red(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`));
-        } else {
-          console.log(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`);
-        }
-        
-      }
   
     });
 }
@@ -125,10 +116,10 @@ async function checkCondition(resolve, startTime, timeoutMs) {
   let networkText = networkSettled ? chalk.green(`y`) : chalk.dim(`${Math.trunc((diffRxBytes + diffTxBytes) / networkUpdateInterval)}b/s`);
   let timeoutText = isTimeout ?  chalk.green(`y`) : chalk.dim(`${Math.floor((timeElapsed)/1000)}/${(timeoutMs / 1000)}s`);
 
-  console.log(`   `, chalk.dim('redraw='), redrawText, chalk.dim('network='), networkText, chalk.dim('timeout='), timeoutText);
+  logger.log("debug", `   ` + chalk.dim('redraw=') + redrawText + chalk.dim(' network=') + networkText + chalk.dim(' timeout=') + timeoutText);
 
   if ((screenHasRedrawn && networkSettled) || isTimeout) {
-    console.log('')
+    logger.log("debug", `   `);
     resolve("true");
   } else {
     checkCondition(resolve, startTime, timeoutMs);
@@ -136,7 +127,7 @@ async function checkCondition(resolve, startTime, timeoutMs) {
 }
 
 function wait(timeoutMs) {
-  console.log("")
+  logger.log("debug", `   `);
   return new Promise((resolve) => {
     const startTime = Date.now();
     checkCondition(resolve, startTime, timeoutMs);

package/lib/sdk.js

@@ -1,10 +1,12 @@
 const config = require("./config");
 const chalk = require("chalk");
 const session = require("./session");
-const package = require("../package.json");
-const version = package.version;
+const version = 'v4.1.0';
 
 const root = config["TD_API_ROOT"];
+const axios = require('axios');
+
+const log = require('./logger');
 
 // let token = null;
 
@@ -26,7 +28,7 @@ const parseBody = async (response, body) => {
       if (!contentType.includes("json") && !contentType.includes("text")) {
         return await response.arrayBuffer();
       }
-      body = await response.text();
+      body = response.data;
     }
 
     if (typeof body === "string") {
@@ -83,7 +85,7 @@ let auth = async () => {
   };
 
   try {
-    await fetch(url, config);
+    await axios(url, config);
     // token = res.data.token;
   } catch (error) {
     await outputError(error);
@@ -101,83 +103,81 @@ const req = async (path, data, onChunk) => {
 
   const url = path.startsWith("/api")
     ? [root, path].join("")
-    : [root, "api", "v" + version, "testdriver", path].join("/");
+    : [root, "api", version, "testdriver", path].join("/");
+
+  log.log("debug", `making request to ${url}`);
 
   const config = {
     method: "post",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
+    headers: { "Content-Type": "application/json" }, 
+    responseType: typeof onChunk === "function" ? "stream" : "json",
+    data: {
       ...data,
       session: session.get()?.id,
       stream: typeof onChunk === "function",
-    }),
+    },
   };
 
   try {
-    const response = await fetch(url, config);
-    if (response.status === 301) {
-      const redirectUrl = await response.text();
-      return req(redirectUrl, data, onChunk);
-    }
-    if (response.status >= 300) {
-      throw response;
-    }
-    const contentType = response.headers.get("Content-Type")?.toLowerCase();
+    let response;
+
+    response = await axios(url, config);
+
+    const contentType = response.headers["content-type"]?.toLowerCase();
     const isJsonl = contentType === "application/jsonl";
     let result;
+
     if (onChunk) {
       result = "";
       let lastLineIndex = -1;
-      const reader = response.clone().body.getReader();
-      while (true) {
-        const { done, value } = await reader.read().catch((err) => {
-          console.error("Body read failed with error:", err);
-          return { done: true };
-        });
-        if (done) {
-          break;
-        }
 
-        let chunk = new TextDecoder().decode(value);
+      await new Promise((resolve, reject) => {
 
-        result += chunk;
-        let events = [chunk];
-        if (isJsonl) {
+        // theres some kind of race condition here that makes things resolve
+        // before the stream is done
+        
+        response.data.on('data', (chunk) => {
+      
+          result += chunk.toString();
           const lines = result.split("\n");
-          events = lines
+
+          const events = lines
             .slice(lastLineIndex + 1, lines.length - 1)
             .filter((line) => line.length)
-            .map((line) => JSON.parse(line));
+            .map((line) => JSON.parse(line));  
+
+          for (const event of events) {
+            onChunk(event);
+          }
 
           lastLineIndex = lines.length - 2;
-        }
-        for (const chunk of events) {
-          await onChunk(chunk);
-        }
-      }
+        });
 
-      if (isJsonl) {
-        const events = result
-          .split("\n")
-          .slice(lastLineIndex + 1)
-          .filter((line) => line.length)
-          .map((line) => JSON.parse(line));
-        for (const event of events) {
-          await onChunk(event);
-        }
-      }
+        response.data.on('end', () => {
+
+          if (isJsonl) {
+            const events = result
+              .split("\n")
+              .slice(lastLineIndex + 2)
+              .filter((line) => line.length)
+              .map((line) => JSON.parse(line));
+
+            for (const event of events) {
+              onChunk(event);
+            }
+          }
+          
+          resolve();
+        });
+
+        response.data.on('error', (error) => {
+          reject(error);
+        });
+      });
     }
 
     const value = await parseBody(response, result);
 
-    if (!path.includes("analytics")) {
-      if (!value) {
-        throw new Error("Unexpected empty response body");
-      }
-      if (!("data" in value)) {
-        throw new Error("Missing data property in response body");
-      }
-    }
     return value;
   } catch (error) {
     await outputError(error);

package/lib/system.js

@@ -46,10 +46,6 @@ const captureAndResize = async (scale = 1, silent = false) => {
     let step1 = tmpFilename();
     let step2 = tmpFilename();
 
-    if (process.env["DEV"]) {
-      console.log(step2);
-    }
-
     await screenshot({ filename: step1, format: "png" });
 
     // Fetch the mouse position

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.1.12",
+  "version": "4.1.14",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {
@@ -17,6 +17,7 @@
   "dependencies": {
     "@electerm/strip-ansi": "^1.0.0",
     "active-win": "^8.2.1",
+    "axios": "^1.7.7",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",
     "datadog-winston": "^1.6.0",