testdriverai 4.0.74 → 4.0.75

package/index.js

@@ -57,7 +57,8 @@ let executionHistory = [];
 let errorCounts = {};
 let errorLimit = 3;
 let checkCount = 0;
-let checkLimit = 10;
+let checkLimit = 3;
+let lastScreenshot = null;
 let rl;
 
 // list of prompts that the user has given us
@@ -279,11 +280,17 @@ const check = async () => {
 
   console.log("");
   log.log("info", chalk.dim("checking..."), "testdriver");
-  let image = await system.captureScreenBase64();
+
+  let thisScreenshot =await system.captureScreenBase64();
+  let images = [lastScreenshot, thisScreenshot];
   let mousePosition = await system.getMousePosition();
   let activeWindow = await system.activeWin();
 
-  return await sdk.req("check", { tasks, image, mousePosition, activeWindow });
+  let response = await sdk.req("check", { tasks, images, mousePosition, activeWindow });
+
+  lastScreenshot = thisScreenshot;
+
+  return response;
 };
 
 // command is transformed from a single yml entry generated by the AI into a JSON object
@@ -458,7 +465,8 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
 
   log.log("info", "");
 
-  let image = await system.captureScreenBase64();
+  lastScreenshot = await system.captureScreenBase64();
+
   const mdStream = log.createMarkdownStreamLogger();
   let message = await sdk.req(
     "input",
@@ -466,7 +474,7 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
       input: currentTask,
       mousePosition: await system.getMousePosition(),
       activeWindow: await system.activeWin(),
-      image,
+      image: lastScreenshot,
     },
     (chunk) => mdStream.log(chunk),
   );

package/lib/redraw.js

@@ -12,8 +12,9 @@ let measurements = [];
 let networkSettled = true;
 let lastUnsettled = null;
 let watchNetwork = null;
+let screenHasRedrawn = null;
 
-async function resetNetwork() {
+async function resetState() {
   lastTxBytes = null;
   lastRxBytes = null;
   measurements = [];
@@ -87,7 +88,7 @@ async function imageIsDifferent(image1Url, image2Url) {
     tmpImage,
     {
       failOnLayoutDiff: false,
-      outputDiffMask: false,
+      outputDiffMask: false
     },
   );
 
@@ -95,7 +96,7 @@ async function imageIsDifferent(image1Url, image2Url) {
     return false;
   } else {
     if (reason === "pixel-diff") {
-      if (diffPercentage > 15) {
+      if (diffPercentage > 5) {
         return true;
       } else {
         return false;
@@ -109,26 +110,43 @@ async function imageIsDifferent(image1Url, image2Url) {
 let startImage = null;
 
 async function start() {
-  resetNetwork();
+  resetState();
   watchNetwork = setInterval(updateNetwork, 500);
   startImage = await captureScreenPNG();
   return startImage;
 }
 
 async function checkCondition(resolve, startTime, timeoutMs) {
+
   let nowImage = await captureScreenPNG();
-  let screenDiff = await imageIsDifferent(startImage, nowImage);
+  let timeElapsed = Date.now() - startTime;
+
+  if (!screenHasRedrawn) {
+    screenHasRedrawn = await imageIsDifferent(startImage, nowImage);
+  }
 
-  if (screenDiff && networkSettled) {
+  if (screenHasRedrawn && networkSettled) {
     clearInterval(watchNetwork);
     resolve("Condition met");
-  } else if (Date.now() - startTime >= timeoutMs) {
+  } else if (Date.now() - timeElapsed > timeoutMs) {
     clearInterval(watchNetwork);
     resolve("Timeout reached");
   } else {
+
+    if (timeElapsed > 3000) {
+
+      if (!screenHasRedrawn) {
+        console.log(chalk.dim(`    waiting for screen redraw...`));
+      }
+      if (!networkSettled) {
+        console.log(chalk.dim(`    waiting for network to settle...`));
+      }
+
+    }
+
     setTimeout(() => {
       checkCondition(resolve, startTime, timeoutMs);
-    }, 250);
+    }, 1000);
   }
 }
 

package/lib/system.js

@@ -37,6 +37,10 @@ const captureAndResize = async (scale = 1) => {
   let step1 = tmpFilename();
   let step2 = tmpFilename();
 
+  if (process.env["DEV"]) {
+    console.log(step2)
+  }
+
   await screenshot({ filename: step1, format: "png" });
 
   // resize to 1:1 px ratio

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.0.74",
+  "version": "4.0.75",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {