testdriverai 4.0.81 → 4.1.1

package/eslint.config.js

@@ -16,6 +16,6 @@ module.exports = [
   {
     // this needs to be it's own object for some reason
     // https://github.com/eslint/eslint/issues/17400
-    ignores: ["lib/subimage/opencv.js", "node_modules/**", ".git"],
+    ignores: ["lib/subimage/opencv.js", "node_modules/**", ".git", "electron/terminal/xterm*"],
   },
 ];

package/index.js

@@ -46,6 +46,7 @@ const { showTerminal, hideTerminal } = require("./lib/focus-application");
 const isValidVersion = require("./lib/valid-version");
 const session = require("./lib/session");
 const notify = require("./lib/notify");
+const { emitter, events } = require("./lib/events.js");
 
 let lastPrompt = "";
 let terminalApp = "";
@@ -54,7 +55,7 @@ let executionHistory = [];
 let errorCounts = {};
 let errorLimit = 3;
 let checkCount = 0;
-let checkLimit = 3;
+let checkLimit = 7;
 let lastScreenshot = null;
 let rl;
 
@@ -160,8 +161,9 @@ function fileCompleter(line) {
 }
 
 function completer(line) {
-  let completions =
-    "/summarize /save /run /quit /assert /undo /manual".split(" ");
+  let completions = "/summarize /save /run /quit /assert /undo /manual".split(
+    " ",
+  );
   if (line.startsWith("/run ")) {
     return fileCompleter(line);
   } else {
@@ -256,8 +258,8 @@ const haveAIResolveError = async (error, markdown, depth = 0, undo = false) => {
     image,
   });
 
-  if (response) {
-    return await actOnMarkdown(response, depth, true);
+  if (response?.data) {
+    return await actOnMarkdown(response.data, depth, true);
   }
 };
 
@@ -275,19 +277,33 @@ const check = async () => {
 
   await delay(3000);
 
-  console.log("");
   log.log("info", chalk.dim("checking..."), "testdriver");
 
-  let thisScreenshot =await system.captureScreenBase64();
+  let thisScreenshot = await system.captureScreenBase64();
   let images = [lastScreenshot, thisScreenshot];
   let mousePosition = await system.getMousePosition();
   let activeWindow = await system.activeWin();
 
-  let response = await sdk.req("check", { tasks, images, mousePosition, activeWindow });
+  const mdStream = log.createMarkdownStreamLogger();
+  let response = await sdk.req(
+    "check",
+    {
+      tasks,
+      images,
+      mousePosition,
+      activeWindow,
+    },
+    (chunk) => {
+      if (chunk.type === "data") {
+        mdStream.log(chunk.data);
+      }
+    },
+  );
+  mdStream.end();
 
   lastScreenshot = thisScreenshot;
 
-  return response;
+  return response.data;
 };
 
 // command is transformed from a single yml entry generated by the AI into a JSON object
@@ -395,9 +411,6 @@ const aiExecute = async (message, validateAndLoop = false) => {
 
     let response = await check();
 
-    console.log("");
-    log.prettyMarkdown(response);
-
     let checkCodeblocks = [];
     try {
       checkCodeblocks = await parser.findCodeBlocks(response);
@@ -473,11 +486,15 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
       activeWindow: await system.activeWin(),
       image: lastScreenshot,
     },
-    (chunk) => mdStream.log(chunk),
+    (chunk) => {
+      if (chunk.type === "data") {
+        mdStream.log(chunk.data);
+      }
+    },
   );
   mdStream.end();
 
-  await aiExecute(message, validateAndLoop);
+  await aiExecute(message.data, validateAndLoop);
 
   log.log("debug", "showing prompt from humanInput response check");
 
@@ -505,11 +522,15 @@ const generate = async (type, count) => {
       activeWindow: await system.activeWin(),
       count,
     },
-    (chunk) => mdStream.log(chunk),
+    (chunk) => {
+      if (chunk.type === "data") {
+        mdStream.log(chunk.data);
+      }
+    },
   );
   mdStream.end();
 
-  let testPrompts = await parser.findGenerativePrompts(message);
+  let testPrompts = await parser.findGenerativePrompts(message.data);
 
   // for each testPrompt
   for (const testPrompt of testPrompts) {
@@ -619,6 +640,7 @@ const firstPrompt = async () => {
   // this is how we parse user input
   // notice that the AI is only called if the input is not a command
   rl.on("line", async (input) => {
+    emitter.emit(events.interactive, false);
     await setTerminalApp();
 
     setTerminalWindowTransparency(true);
@@ -762,7 +784,11 @@ let summarize = async (error = null) => {
       error: error?.toString(),
       tasks,
     },
-    (chunk) => mdStream.log(chunk),
+    (chunk) => {
+      if (chunk.type === "data") {
+        mdStream.log(chunk.data);
+      }
+    },
   );
   mdStream.end();
 
@@ -771,7 +797,7 @@ let summarize = async (error = null) => {
     resultFile = "/Windows/Temp/oiResult.log.log";
   }
   // write reply to /tmp/oiResult.log.log
-  fs.writeFileSync(resultFile, reply);
+  fs.writeFileSync(resultFile, reply.data);
 };
 
 // this function is responsible for saving the regression test script to a file
@@ -900,6 +926,7 @@ ${yaml.dump(step)}
 };
 
 const promptUser = () => {
+  emitter.emit(events.interactive, true);
   rl.prompt(true);
 };
 
@@ -964,7 +991,18 @@ const embed = async (file, depth) => {
   log.log("info", `${file} (end)`);
 };
 
+emitter.on(events.interactive, (data) => {
+  if (!terminalApp) return;
+  if (data) {
+    showTerminal(terminalApp);
+  } else {
+    hideTerminal(terminalApp);
+  }
+});
+
 (async () => {
+  // console.log(await  system.getPrimaryDisplay());
+
   // @todo add-auth
   // if (!process.env.DASHCAM_API_KEY) {
   //   log.log('info', chalk.red(`You must supply an API key`), 'system')
@@ -1009,13 +1047,13 @@ const embed = async (file, depth) => {
   await setTerminalApp();
 
   // should be start of new session
-  let sessionRes = await sdk.req("session/start", {
+  const sessionRes = await sdk.req("session/start", {
     systemInformationOsInfo: await system.getSystemInformationOsInfo(),
     mousePosition: await system.getMousePosition(),
     activeWindow: await system.activeWin(),
   });
 
-  session.set(sessionRes);
+  session.set(sessionRes.data);
 
   analytics.track("command", { command: thisCommand, file: thisFile });
 

package/lib/commander.js

@@ -11,6 +11,9 @@ const marky = require("marky");
 // object is a json representation of the individual yml command
 // the process turns markdown -> yml -> json -> js function execution
 const run = async (object, depth) => {
+
+  log("debug", { object, depth });
+
   // success returns null
   // if this is set, it means that we need to take more action and a new thread is spawned
   let response = null;

package/lib/commands.js

@@ -20,6 +20,7 @@ const os = require("os");
 const cliProgress = require("cli-progress");
 const redraw = require("./redraw");
 const logger = require("./logger");
+const { emitter, events } = require("./events.js");
 
 const niceSeconds = (ms) => {
   return Math.round(ms / 1000);
@@ -138,7 +139,6 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
   // take a screenshot
   log("info", "");
   log("info", chalk.dim("thinking..."), true);
-  log("info", "");
 
   if (async) {
     await sdk
@@ -147,7 +147,7 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
         image: await captureScreenBase64(),
       })
       .then((response) => {
-        return handleAssertResponse(response);
+        return handleAssertResponse(response.data);
       });
 
     return true;
@@ -156,7 +156,7 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
       expect: assertion,
       image: await captureScreenBase64(),
     });
-    return handleAssertResponse(response);
+    return handleAssertResponse(response.data);
   }
 };
 const scroll = async (direction = "down", amount = 300) => {
@@ -167,19 +167,19 @@ const scroll = async (direction = "down", amount = 300) => {
   switch (direction) {
     case "up":
       await robot.keyTap("pageup");
-      await redraw.wait(10000);
+      await redraw.wait(2500);
       break;
     case "down":
       await robot.keyTap("pagedown");
-      await redraw.wait(10000);
+      await redraw.wait(2500);
       break;
     case "left":
       await robot.scrollMouse(amount * -1, 0);
-      await redraw.wait(10000);
+      await redraw.wait(2500);
       break;
     case "right":
       await robot.scrollMouse(amount, 0);
-      await redraw.wait(10000);
+      await redraw.wait(2500);
       break;
     default:
       throw new AiError("Direction not found");
@@ -199,7 +199,8 @@ const click = async (x, y, button = "left", click = "single") => {
   robot.moveMouseSmooth(x, y, 0.1);
   await delay(1000); // wait for the mouse to move
   robot.mouseClick(button, double);
-  await redraw.wait(10000);
+  emitter.emit(events.mouseClick, { x, y, button, click });
+  await redraw.wait(5000);
   return;
 };
 
@@ -210,7 +211,7 @@ const hover = async (x, y) => {
   y = parseInt(y);
 
   await robot.moveMouseSmooth(x, y, 0.1);
-  await redraw.wait(10000);
+  await redraw.wait(2500);
 
   return;
 };
@@ -229,14 +230,13 @@ let commands = {
     action = "click",
     button = "left",
     clickType = "single",
-    method = "ai",
+    method = "turbo",
   ) => {
-    text = text.toString();
+    text = text ? text.toString() : null;
     description = description ? description.toString() : null;
 
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
-    log("info", "");
 
     const mdStream = logger.createMarkdownStreamLogger();
     let response = await sdk.req(
@@ -251,14 +251,20 @@ let commands = {
         description,
         displayMultiple: 1,
       },
-      (chunk) => mdStream.log(chunk),
+      (chunk) => {
+        if (chunk.type === "data") {
+          mdStream.log(chunk.data);
+        } else if (chunk.type === "closeMatches") {
+          emitter.emit(events.matches.show, chunk.data);
+        }
+      },
     );
     mdStream.end();
 
-    if (!response) {
+    if (!response.data) {
       throw new AiError("No text on screen matches description", true);
     } else {
-      return response;
+      return response.data;
     }
   },
   // uses our api to find all images on screen
@@ -271,22 +277,32 @@ let commands = {
     // take a screenshot
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
-    log("info", "");
 
-    let response = await sdk.req("hover/image", {
-      needle: description,
-      image: await captureScreenBase64(),
-      intent: action,
-      button,
-      clickType,
-      displayMultiple: 1,
-    });
+    const mdStream = logger.createMarkdownStreamLogger();
+    let response = await sdk.req(
+      "hover/image",
+      {
+        needle: description,
+        image: await captureScreenBase64(),
+        intent: action,
+        button,
+        clickType,
+        displayMultiple: 1,
+      },
+      (chunk) => {
+
+        if (chunk.type === "data") {
+          mdStream.log(chunk.data);
+        } else if (chunk.type === "closeMatches") {
+          emitter.emit(events.matches.show, chunk.data);
+        }
+      },
+    );
+    mdStream.end();
 
     if (!response?.data) {
       throw new AiError("No text on screen matches description", true);
     } else {
-      log("info", "");
-      logger.prettyMarkdown(response.data);
       return response.data;
     }
   },
@@ -315,7 +331,7 @@ let commands = {
     await redraw.start();
     string = string.toString();
     await robot.typeString(string);
-    await redraw.wait(10000);
+    await redraw.wait(5000);
     return;
   },
   // press keys
@@ -373,7 +389,7 @@ let commands = {
     // finally, press the keys
     robot.keyTap(keysPressed[0], modsToPress);
 
-    await redraw.wait(10000);
+    await redraw.wait(5000);
 
     // keyTap will release the normal keys, but will not release modifier keys
     // so we need to release the modifier keys manually
@@ -402,10 +418,11 @@ let commands = {
     let passed = false;
 
     while (durationPassed < timeout && !passed) {
-      passed = await sdk.req("assert", {
+      const response = await sdk.req("assert", {
         needle: description,
         image: await captureScreenBase64(),
       });
+      passed = response.data;
 
       durationPassed = new Date().getTime() - startTime;
       if (!passed) {
@@ -433,7 +450,7 @@ let commands = {
       );
     }
   },
-  "wait-for-text": async (text, timeout = 5000, method = "ai") => {
+  "wait-for-text": async (text, timeout = 5000, method = "turbo") => {
     await redraw.start();
 
     log("info", chalk.dim(`waiting for text: "${text}"...`), true);
@@ -444,12 +461,21 @@ let commands = {
     let passed = false;
 
     while (durationPassed < timeout && !passed) {
-      passed = await sdk.req("assert/text", {
-        needle: text,
-        method: method,
-        image: await captureScreenBase64(),
-      });
+      const response = await sdk.req(
+        "assert/text",
+        {
+          needle: text,
+          method: method,
+          image: await captureScreenBase64(),
+        },
+        (chunk) => {
+          if (chunk.type === "closeMatches") {
+            emitter.emit(events.matches.show, chunk.data);
+          }
+        },
+      );
 
+      passed = response.data;
       durationPassed = new Date().getTime() - startTime;
       if (!passed && durationPassed > timeout) {
         log(
@@ -459,7 +485,7 @@ let commands = {
           ),
           true,
         );
-        await redraw.wait(10000);
+        await redraw.wait(5000);
       }
     }
 
@@ -477,7 +503,7 @@ let commands = {
     text,
     direction = "down",
     maxDistance = 1200,
-    method = "ai",
+    method = "turbo",
   ) => {
     await redraw.start();
 
@@ -488,7 +514,7 @@ let commands = {
       await robot.keyTap("f", commandOrControl);
       // type the text
       await robot.typeString(text);
-      await redraw.wait(10000);
+      await redraw.wait(5000);
       await robot.keyTap("escape");
     } catch (e) {
       console.log(e);
@@ -503,12 +529,21 @@ let commands = {
     let passed = false;
 
     while (scrollDistance <= maxDistance && !passed) {
-      passed = await sdk.req("assert/text", {
-        needle: text,
-        method: method,
-        image: await captureScreenBase64(),
-      });
+      const response = await sdk.req(
+        "assert/text",
+        {
+          needle: text,
+          method: method,
+          image: await captureScreenBase64(),
+        },
+        (chunk) => {
+          if (chunk.type === "closeMatches") {
+            emitter.emit(events.matches.show, chunk.data);
+          }
+        },
+      );
 
+      passed = response.data;
       if (!passed) {
         log(
           "info",
@@ -548,11 +583,12 @@ let commands = {
     let passed = false;
 
     while (scrollDistance < maxDistance && !passed) {
-      passed = await sdk.req("assert", {
+      const response = await sdk.req("assert", {
         needle: `An image matching the description "${description}" appears on screen.`,
         image: await captureScreenBase64(),
       });
 
+      passed = response.data;
       if (!passed) {
         log(
           "info",

package/lib/config.js

@@ -9,12 +9,11 @@ require("dotenv").config();
 
 // Parse out true and false string values
 function parseValue(value) {
-  if (value === "false") {
-    return false;
-  }
-
-  if (value === "true") {
-    return true;
+  if (typeof value === "string") {
+    const normalizedValue = value.toLowerCase().trim();
+    if (["true", "false"].includes(normalizedValue)) {
+      return JSON.parse(normalizedValue);
+    }
   }
 
   return value;
@@ -29,6 +28,7 @@ const config = {
   TD_API_ROOT: "https://api.testdriver.ai",
   TD_DEV: parseValue(process.env["DEV"]),
   TD_PROFILE: false,
+  TD_OVERLAY: true,
 };
 
 // Find all env vars starting with TD_

package/lib/events.js

@@ -0,0 +1,38 @@
+const { EventEmitter } = require("events");
+
+const emitter = new EventEmitter();
+
+const events = {
+  mouseClick: "mouse-click",
+  screenCapture: {
+    start: "screen-capture:start",
+    end: "screen-capture:end",
+    error: "screen-capture:error",
+  },
+  interactive: "interactive",
+  terminal: {
+    stdout: "terminal:stdout",
+    stderr: "terminal:stderr",
+  },
+  matches: {
+    show: "matches:show",
+  },
+};
+
+const getValues = (obj) => {
+  if (["string", "number"].includes(typeof obj)) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(getValues);
+  }
+  if ([undefined, null].includes(obj)) {
+    return [];
+  }
+
+  return Object.values(obj).map(getValues).flat();
+};
+
+const eventsArray = getValues(events);
+
+module.exports = { events, emitter, eventsArray };

package/lib/overlay.js

@@ -0,0 +1,36 @@
+const path = require("path");
+const ipc = require("node-ipc").default;
+const { fork } = require("child_process");
+
+const { emitter, eventsArray } = require("./events.js");
+
+ipc.config.id = "testdriverai";
+ipc.config.retry = 50;
+ipc.config.silent = true;
+
+const electronProcess = fork(
+  path.join(__dirname, "../node_modules/electron/cli.js"),
+  [path.join(__dirname, "../electron/overlay.js")],
+  { stdio: "ignore" },
+);
+
+electronProcess.on("exit", (code) => {
+  process.exit(code);
+});
+
+process.on("exit", () => electronProcess.kill("SIGTERM"));
+process.on("SIGINT", () => electronProcess.kill("SIGTERM")); // Ctrl+C
+process.on("SIGTERM", () => electronProcess.kill("SIGTERM")); // Termination signal
+process.on("uncaughtException", () => electronProcess.kill("SIGTERM"));
+
+module.exports.electronProcessPromise = new Promise((resolve) => {
+  ipc.connectTo("testdriverai_overlay");
+  ipc.of.testdriverai_overlay.on("connect", () => {
+    eventsArray.forEach((event) =>
+      emitter.on(event, (data) =>
+        ipc.of.testdriverai_overlay.emit(event, data),
+      ),
+    );
+    resolve(electronProcess);
+  });
+});