testdriverai 4.0.73 → 4.0.75

package/index.js

@@ -57,7 +57,8 @@ let executionHistory = [];
 let errorCounts = {};
 let errorLimit = 3;
 let checkCount = 0;
-let checkLimit = 10;
+let checkLimit = 3;
+let lastScreenshot = null;
 let rl;
 
 // list of prompts that the user has given us
@@ -279,11 +280,17 @@ const check = async () => {
 
   console.log("");
   log.log("info", chalk.dim("checking..."), "testdriver");
-  let image = await system.captureScreenBase64();
+
+  let thisScreenshot =await system.captureScreenBase64();
+  let images = [lastScreenshot, thisScreenshot];
   let mousePosition = await system.getMousePosition();
   let activeWindow = await system.activeWin();
 
-  return await sdk.req("check", { tasks, image, mousePosition, activeWindow });
+  let response = await sdk.req("check", { tasks, images, mousePosition, activeWindow });
+
+  lastScreenshot = thisScreenshot;
+
+  return response;
 };
 
 // command is transformed from a single yml entry generated by the AI into a JSON object
@@ -458,7 +465,8 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
 
   log.log("info", "");
 
-  let image = await system.captureScreenBase64();
+  lastScreenshot = await system.captureScreenBase64();
+
   const mdStream = log.createMarkdownStreamLogger();
   let message = await sdk.req(
     "input",
@@ -466,7 +474,7 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
       input: currentTask,
       mousePosition: await system.getMousePosition(),
       activeWindow: await system.activeWin(),
-      image,
+      image: lastScreenshot,
     },
     (chunk) => mdStream.log(chunk),
   );

package/lib/commands.js

@@ -167,19 +167,19 @@ const scroll = async (direction = "down", amount = 300) => {
   switch (direction) {
     case "up":
       await robot.keyTap("pageup");
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "down":
       await robot.keyTap("pagedown");
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "left":
       await robot.scrollMouse(amount * -1, 0);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "right":
       await robot.scrollMouse(amount, 0);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     default:
       throw new AiError("Direction not found");
@@ -199,7 +199,7 @@ const click = async (x, y, button = "left", click = "single") => {
   robot.moveMouseSmooth(x, y, 0.1);
   await delay(1000); // wait for the mouse to move
   robot.mouseClick(button, double);
-  await redraw.wait(2000);
+  await redraw.wait(10000);
   return;
 };
 
@@ -210,7 +210,7 @@ const hover = async (x, y) => {
   y = parseInt(y);
 
   await robot.moveMouseSmooth(x, y, 0.1);
-  await redraw.wait(2000);
+  await redraw.wait(10000);
 
   return;
 };
@@ -236,6 +236,7 @@ let commands = {
 
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     const mdStream = logger.createMarkdownStreamLogger();
     let response = await sdk.req(
@@ -270,6 +271,7 @@ let commands = {
     // take a screenshot
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     let response = await sdk.req("hover/image", {
       needle: description,
@@ -313,7 +315,7 @@ let commands = {
     await redraw.start();
     string = string.toString();
     await robot.typeString(string);
-    await redraw.wait(1000);
+    await redraw.wait(10000);
     return;
   },
   // press keys
@@ -371,7 +373,7 @@ let commands = {
     // finally, press the keys
     robot.keyTap(keysPressed[0], modsToPress);
 
-    await redraw.wait(2000);
+    await redraw.wait(10000);
 
     // keyTap will release the normal keys, but will not release modifier keys
     // so we need to release the modifier keys manually
@@ -457,7 +459,7 @@ let commands = {
           ),
           true,
         );
-        await redraw.wait(2000);
+        await redraw.wait(10000);
       }
     }
 
@@ -486,7 +488,7 @@ let commands = {
       await robot.keyTap("f", commandOrControl);
       // type the text
       await robot.typeString(text);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       await robot.keyTap("escape");
     } catch (e) {
       console.log(e);

package/lib/network.js

@@ -0,0 +1,3 @@
+
+
+

package/lib/redraw.js

@@ -3,6 +3,80 @@ const os = require("os");
 const path = require("path");
 const { compare } = require("odiff-bin");
 
+// network
+const si = require('systeminformation');
+const chalk = require('chalk');
+let lastTxBytes = null;
+let lastRxBytes = null;
+let measurements = [];
+let networkSettled = true;
+let lastUnsettled = null;
+let watchNetwork = null;
+let screenHasRedrawn = null;
+
+async function resetState() {
+  lastTxBytes = null;
+  lastRxBytes = null;
+  measurements = [];
+  networkSettled = true;
+  lastUnsettled = null;
+}
+
+async function updateNetwork() {
+    si.networkStats().then(data => {
+      let thisRxBytes = data[0].rx_bytes;
+      let thisTxBytes = data[0].tx_bytes;
+  
+      let diffRxBytes = lastRxBytes !== null ? thisRxBytes - lastRxBytes : 0;
+      let diffTxBytes = lastTxBytes !== null ? thisTxBytes - lastTxBytes : 0;
+  
+      lastRxBytes = thisRxBytes;
+      lastTxBytes = thisTxBytes;
+  
+      measurements.push({ rx: diffRxBytes, tx: diffTxBytes });
+  
+      if (measurements.length > 60) {
+        measurements.shift();
+      }
+  
+      let avgRx = measurements.reduce((acc, m) => acc + m.rx, 0) / measurements.length;
+      let avgTx = measurements.reduce((acc, m) => acc + m.tx, 0) / measurements.length;
+  
+      let stdDevRx = Math.sqrt(measurements.reduce((acc, m) => acc + Math.pow(m.rx - avgRx, 2), 0) / measurements.length);
+      let stdDevTx = Math.sqrt(measurements.reduce((acc, m) => acc + Math.pow(m.tx - avgTx, 2), 0) / measurements.length);
+  
+      let zIndexRx = stdDevRx !== 0 ? (diffRxBytes - avgRx) / stdDevRx : 0;
+      let zIndexTx = stdDevTx !== 0 ? (diffTxBytes - avgTx) / stdDevTx : 0;
+  
+      // log time since unsettlement
+  
+      if ((new Date().getTime() - lastUnsettled) < 2000) {
+        networkSettled = false;
+      } else {
+        
+        if ((zIndexRx < 0 && zIndexTx < 0) ) {
+          lastUnsettled = null;
+          networkSettled = true;
+        } else {
+          lastUnsettled = new Date().getTime();
+          networkSettled = false;
+        }
+  
+      }
+    
+      if (process.env["DEV"]) {
+
+        if (!networkSettled) {
+          console.log(chalk.red(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`));
+        } else {
+          console.log(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`);
+        }
+        
+      }
+  
+    });
+}
+
 async function imageIsDifferent(image1Url, image2Url) {
 
   // generate a temporary file path
@@ -14,7 +88,7 @@ async function imageIsDifferent(image1Url, image2Url) {
     tmpImage,
     {
       failOnLayoutDiff: false,
-      outputDiffMask: false,
+      outputDiffMask: false
     },
   );
 
@@ -22,7 +96,7 @@ async function imageIsDifferent(image1Url, image2Url) {
     return false;
   } else {
     if (reason === "pixel-diff") {
-      if (diffPercentage > 15) {
+      if (diffPercentage > 5) {
         return true;
       } else {
         return false;
@@ -36,22 +110,43 @@ async function imageIsDifferent(image1Url, image2Url) {
 let startImage = null;
 
 async function start() {
+  resetState();
+  watchNetwork = setInterval(updateNetwork, 500);
   startImage = await captureScreenPNG();
   return startImage;
 }
 
 async function checkCondition(resolve, startTime, timeoutMs) {
+
   let nowImage = await captureScreenPNG();
-  let result = await imageIsDifferent(startImage, nowImage);
+  let timeElapsed = Date.now() - startTime;
+
+  if (!screenHasRedrawn) {
+    screenHasRedrawn = await imageIsDifferent(startImage, nowImage);
+  }
 
-  if (result) {
+  if (screenHasRedrawn && networkSettled) {
+    clearInterval(watchNetwork);
     resolve("Condition met");
-  } else if (Date.now() - startTime >= timeoutMs) {
+  } else if (Date.now() - timeElapsed > timeoutMs) {
+    clearInterval(watchNetwork);
     resolve("Timeout reached");
   } else {
+
+    if (timeElapsed > 3000) {
+
+      if (!screenHasRedrawn) {
+        console.log(chalk.dim(`    waiting for screen redraw...`));
+      }
+      if (!networkSettled) {
+        console.log(chalk.dim(`    waiting for network to settle...`));
+      }
+
+    }
+
     setTimeout(() => {
       checkCondition(resolve, startTime, timeoutMs);
-    }, 250);
+    }, 1000);
   }
 }
 

package/lib/system.js

@@ -37,6 +37,10 @@ const captureAndResize = async (scale = 1) => {
   let step1 = tmpFilename();
   let step2 = tmpFilename();
 
+  if (process.env["DEV"]) {
+    console.log(step2)
+  }
+
   await screenshot({ filename: step1, format: "png" });
 
   // resize to 1:1 px ratio

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.0.73",
+  "version": "4.0.75",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {