testdriverai 4.0.80 → 4.1.1

package/lib/redraw.js

@@ -6,6 +6,10 @@ const { compare } = require("odiff-bin");
 // network
 const si = require('systeminformation');
 const chalk = require('chalk');
+
+const networkCooldownMs = 2000;
+const redrawThresholdPercent = 3;
+
 let lastTxBytes = null;
 let lastRxBytes = null;
 let measurements = [];
@@ -20,6 +24,7 @@ async function resetState() {
   measurements = [];
   networkSettled = true;
   lastUnsettled = null;
+  screenHasRedrawn = false;
 }
 
 async function updateNetwork() {
@@ -50,7 +55,7 @@ async function updateNetwork() {
   
       // log time since unsettlement
   
-      if ((new Date().getTime() - lastUnsettled) < 2000) {
+      if ((new Date().getTime() - lastUnsettled) < networkCooldownMs) {
         networkSettled = false;
       } else {
         
@@ -77,7 +82,7 @@ async function updateNetwork() {
     });
 }
 
-async function imageIsDifferent(image1Url, image2Url) {
+async function imageDiffPercent(image1Url, image2Url) {
 
   // generate a temporary file path
   const tmpImage = path.join(os.tmpdir(), `tmp-${Date.now()}.png`);
@@ -96,11 +101,7 @@ async function imageIsDifferent(image1Url, image2Url) {
     return false;
   } else {
     if (reason === "pixel-diff") {
-      if (diffPercentage > 10) {
-        return true;
-      } else {
-        return false;
-      }
+      return diffPercentage.toFixed(1);
     } else {
       return false;
     }
@@ -113,37 +114,33 @@ async function start() {
   resetState();
   watchNetwork = setInterval(updateNetwork, 500);
   startImage = await captureScreenPNG();
+  startImage = await captureScreenPNG(1, true);
   return startImage;
 }
 
 async function checkCondition(resolve, startTime, timeoutMs) {
-
-  let nowImage = await captureScreenPNG();
+  let nowImage = await captureScreenPNG(1, true);
   let timeElapsed = Date.now() - startTime;
+  let diffPercent = 0;
+  let isTimeout = timeElapsed > timeoutMs;
 
   if (!screenHasRedrawn) {
-    screenHasRedrawn = await imageIsDifferent(startImage, nowImage);
+    diffPercent = await imageDiffPercent(startImage, nowImage);
+    screenHasRedrawn = diffPercent > redrawThresholdPercent;
   }
 
-  if (screenHasRedrawn && networkSettled) {
-    clearInterval(watchNetwork);
-    resolve("Condition met");
-  } else if (Date.now() - timeElapsed > timeoutMs) {
-    clearInterval(watchNetwork);
-    resolve("Timeout reached");
-  } else {
+  // // log redraw as output
+  let redrawText = screenHasRedrawn ? chalk.green(`✓`) : chalk.dim(`${diffPercent}/${redrawThresholdPercent}%`);
+  let networkText = networkSettled ? chalk.green(`✓`) : chalk.dim(`${Math.floor((Date.now() - lastUnsettled) / 1000)}/${Math.floor(networkCooldownMs/1000)}s`);
+  let timeoutText = isTimeout ?  chalk.green(`✓`) : chalk.dim(`${Math.floor((timeElapsed)/1000)}/${(timeoutMs / 1000)}s`);
 
-    if (timeElapsed > 3000) {
-
-      if (!screenHasRedrawn) {
-        console.log(chalk.dim(`    waiting for screen redraw...`));
-      }
-      if (!networkSettled) {
-        console.log(chalk.dim(`    waiting for network to settle...`));
-      }
-
-    }
+  console.log(`   `, chalk.dim('redraw='), redrawText, chalk.dim('network='), networkText, chalk.dim('timeout='), timeoutText);
 
+  if ((screenHasRedrawn && networkSettled) || isTimeout) {
+    clearInterval(watchNetwork);
+    console.log('')
+    resolve("true");
+  } else {
     setTimeout(() => {
       checkCondition(resolve, startTime, timeoutMs);
     }, 1000);
@@ -151,6 +148,7 @@ async function checkCondition(resolve, startTime, timeoutMs) {
 }
 
 function wait(timeoutMs) {
+  console.log("")
   return new Promise((resolve) => {
     const startTime = Date.now();
     checkCondition(resolve, startTime, timeoutMs);

package/lib/sdk.js

@@ -5,7 +5,6 @@ const package = require("../package.json");
 const version = package.version;
 
 const root = config["TD_API_ROOT"];
-const shouldStream = config["TD_STREAM_RESPONSES"];
 
 // let token = null;
 
@@ -13,7 +12,7 @@ const outputError = async (error) => {
   if (error instanceof Response) {
     console.log(chalk.red(error.status), chalk.red(error.statusText));
     await parseBody(error)
-      .then((body) => console.log(chalk.red(body)))
+      .then((body) => console.log(chalk.red(JSON.stringify(body, null, 2))))
       .catch(() => {});
   } else {
     console.error("Error:", error);
@@ -27,13 +26,37 @@ const parseBody = async (response, body) => {
       if (!contentType.includes("json") && !contentType.includes("text")) {
         return await response.arrayBuffer();
       }
+      body = await response.text();
+    }
+
+    if (typeof body === "string") {
+      if (contentType.includes("jsonl")) {
+        const result = body
+          .split("\n")
+          .filter((line) => line.trim().length)
+          .map((line) => JSON.parse(line))
+          .reduce((result, { type, data }) => {
+            if (result[type]) {
+              if (typeof result[type] === "string") {
+                result[type] += data;
+              } else {
+                result[type].push(data);
+              }
+            } else {
+              result[type] = typeof data === "string" ? data : [data];
+            }
+            return result;
+          }, {});
+        for (const key of Object.keys(result)) {
+          if (Array.isArray(result[key]) && result[key].length === 1) {
+            result[key] = result[key][0];
+          }
+        }
+        return result;
+      }
       if (contentType.includes("json")) {
-        return await response.json();
+        return JSON.parse(body);
       }
-      return await response.text();
-    }
-    if (typeof body === "string" && contentType.includes("json")) {
-      return JSON.parse(body);
     }
     return body;
   } catch (err) {
@@ -99,10 +122,12 @@ const req = async (path, data, onChunk) => {
     if (response.status >= 300) {
       throw response;
     }
-
+    const contentType = response.headers.get("Content-Type")?.toLowerCase();
+    const isJsonl = contentType === "application/jsonl";
     let result;
     if (onChunk) {
       result = "";
+      let lastLineIndex = -1;
       const reader = response.body.getReader();
       while (true) {
         const { done, value } = await reader.read();
@@ -110,14 +135,33 @@ const req = async (path, data, onChunk) => {
           break;
         }
 
-        const chunk = new TextDecoder().decode(value);
+        let chunk = new TextDecoder().decode(value);
+
         result += chunk;
-        if (shouldStream) {
+        let events = [chunk];
+        if (isJsonl) {
+          const lines = result.split("\n");
+          events = lines
+            .slice(lastLineIndex + 1, lines.length - 1)
+            .filter((line) => line.length)
+            .map((line) => JSON.parse(line));
+
+          lastLineIndex = lines.length - 2;
+        }
+        for (const chunk of events) {
           await onChunk(chunk);
         }
       }
-      if (!shouldStream) {
-        await onChunk(result);
+
+      if (isJsonl) {
+        const events = result
+          .split("\n")
+          .slice(lastLineIndex + 1)
+          .filter((line) => line.length)
+          .map((line) => JSON.parse(line));
+        for (const event of events) {
+          await onChunk(event);
+        }
       }
     }
 

package/lib/system.js

@@ -7,6 +7,7 @@ const si = require("systeminformation");
 const activeWindow = require("active-win");
 const robot = require("robotjs");
 const sharp = require("sharp");
+const { emitter, events } = require("./events.js");
 
 let primaryDisplay = null;
 
@@ -31,45 +32,67 @@ const tmpFilename = () => {
   return path.join(os.tmpdir(), `${new Date().getTime() + Math.random()}.png`);
 };
 
-const captureAndResize = async (scale = 1) => {
-  let primaryDisplay = await getPrimaryDisplay();
-
-  let step1 = tmpFilename();
-  let step2 = tmpFilename();
-
-  if (process.env["DEV"]) {
-    console.log(step2)
+const captureAndResize = async (scale = 1, silent = false) => {
+  try {
+    const primaryDisplay = await getPrimaryDisplay();
+    if (!silent) {
+      emitter.emit(events.screenCapture.start, {
+        scale,
+        display: primaryDisplay,
+      });
+    }
+
+    let step1 = tmpFilename();
+    let step2 = tmpFilename();
+
+    if (process.env["DEV"]) {
+      console.log(step2);
+    }
+
+    await screenshot({ filename: step1, format: "png" });
+
+    // Fetch the mouse position
+    const mousePos = robot.getMousePos();
+
+    // Location of cursor image
+    const cursorPath = path.join(__dirname, "resources", "cursor.png");
+
+    // resize to 1:1 px ratio
+    await sharp(step1)
+      .resize(
+        Math.floor(primaryDisplay.currentResX * scale),
+        Math.floor(primaryDisplay.currentResY * scale),
+      )
+      // composite the mouse image ontop
+      .composite([{ input: cursorPath, left: mousePos.x, top: mousePos.y }])
+      .toFile(step2);
+    if (!silent) {
+      emitter.emit(events.screenCapture.end, {
+        scale,
+        display: primaryDisplay,
+      });
+    }
+    return step2;
+  } catch (error) {
+    if (!silent) {
+      emitter.emit(events.screenCapture.error, {
+        error,
+        scale,
+        display: primaryDisplay,
+      });
+    }
+    throw error;
   }
-
-  await screenshot({ filename: step1, format: "png" });
-
-  // Fetch the mouse position
-  const mousePos = robot.getMousePos();
-
-  // Location of cursor image
-  const cursorPath = path.join(__dirname, "resources", "cursor.png");
-
-  // resize to 1:1 px ratio
-  await sharp(step1)
-    .resize(
-      Math.floor(primaryDisplay.currentResX * scale),
-      Math.floor(primaryDisplay.currentResY * scale),
-    )
-    // composite the mouse image ontop
-    .composite([{ input: cursorPath, left: mousePos.x, top: mousePos.y}])
-    .toFile(step2);
-
-  return step2;
 };
 
 // our handy screenshot function
-const captureScreenBase64 = async (scale = 1) => {
-  let step2 = await captureAndResize(scale);
+const captureScreenBase64 = async (scale = 1, silent = false) => {
+  let step2 = await captureAndResize(scale, silent);
   return fs.readFileSync(step2, "base64");
 };
 
-const captureScreenPNG = async (scale = 1) => {
-  return await captureAndResize(scale);
+const captureScreenPNG = async (scale = 1, silent = false) => {
+  return await captureAndResize(scale, silent);
 };
 
 const platform = () => {
@@ -96,6 +119,7 @@ const getMousePosition = async () => {
 };
 
 module.exports = {
+  getPrimaryDisplay,
   captureScreenBase64,
   captureScreenPNG,
   getMousePosition,

package/main.js

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+const config = require("./lib/config");
+const { emitter, events } = require("./lib/events.js");
+
+if (!config.TD_OVERLAY) {
+  require("./index.js");
+} else {
+  // Intercept all stdout and stderr calls (works with console as well)
+  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  process.stdout.write = (...args) => {
+    const [data, encoding] = args;
+    emitter.emit(
+      events.terminal.stdout,
+      data.toString(typeof encoding === "string" ? encoding : undefined),
+    );
+    originalStdoutWrite(...args);
+  };
+
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  process.stderr.write = (...args) => {
+    const [data, encoding] = args;
+    emitter.emit(
+      events.terminal.stderr,
+      data.toString(typeof encoding === "string" ? encoding : undefined),
+    );
+    originalStderrWrite(...args);
+  };
+
+  require("./lib/overlay.js")
+    .electronProcessPromise.then(() => {
+      require("./index.js");
+    })
+    .catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+}

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "testdriverai",
-  "version": "4.0.80",
+  "version": "4.1.1",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
-  "main": "index.js",
+  "main": "main.js",
   "bin": {
-    "testdriverai": "./index.js"
+    "testdriverai": "./main.js"
   },
   "scripts": {
-    "start": "node index.js",
-    "dev": "DEV=true node index.js",
-    "debug": "DEV=true VERBOSE=true node index.js",
+    "start": "node main.js",
+    "dev": "DEV=true node main.js",
+    "debug": "DEV=true VERBOSE=true node main.js",
     "bundle": "node build.mjs"
   },
   "author": "",
@@ -22,6 +22,7 @@
     "datadog-winston": "^1.6.0",
     "decompress": "^4.2.1",
     "dotenv": "^16.4.5",
+    "electron": "^33.0.2",
     "jimp": "^0.22.12",
     "js-yaml": "^4.1.0",
     "mac-screen-capture-permissions": "^2.1.0",
@@ -29,6 +30,7 @@
     "marked": "^12.0.1",
     "marked-terminal": "^7.0.0",
     "marky": "^1.2.5",
+    "node-ipc": "^12.0.0",
     "node-notifier": "^10.0.1",
     "odiff-bin": "^3.1.2",
     "prompts": "^2.4.2",