testdriverai 4.0.72 → 4.0.74

package/README.md

@@ -81,7 +81,7 @@ If you have multiple monitors, make sure you do this on your primary display.
 
 Now, just tell TestDriver what you want it to do. For now, stick with single commands like "click sign up" and "scroll down."
 
-Later, try `/explore` to perform higher level objectives like "complete the onboarding."
+Later, try to perform higher level objectives like "complete the onboarding."
 
 ```yaml
 > Click on sign up

package/index.js

@@ -163,7 +163,7 @@ function fileCompleter(line) {
 
 function completer(line) {
   let completions =
-    "/summarize /save /run /quit /explore /assert /undo /manual".split(" ");
+    "/summarize /save /run /quit /assert /undo /manual".split(" ");
   if (line.startsWith("/run ")) {
     return fileCompleter(line);
   } else {
@@ -524,7 +524,7 @@ const generate = async (type, count) => {
     let list = testPrompt.listsOrdered[0];
 
     let contents = list
-      .map((item, index) => `${index + 1}. /explore ${item}`)
+      .map((item, index) => `${index + 1}. ${item}`)
       .join("\n");
     fs.writeFileSync(path1, contents);
   }
@@ -635,8 +635,6 @@ const firstPrompt = async () => {
       await exit();
     } else if (input.indexOf("/save") == 0) {
       await save({ filepath: commands[1] });
-    } else if (input.indexOf("/explore") == 0) {
-      await humanInput(commands.slice(1).join(" "), true);
     } else if (input.indexOf("/undo") == 0) {
       await undo();
     } else if (input.indexOf("/assert") == 0) {
@@ -648,7 +646,7 @@ const firstPrompt = async () => {
     } else if (input.indexOf("/generate") == 0) {
       await generate(commands[1], commands[2]);
     } else {
-      await humanInput(input, false);
+      await humanInput(input, true);
     }
 
     setTerminalWindowTransparency(false);

package/lib/commands.js

@@ -167,19 +167,19 @@ const scroll = async (direction = "down", amount = 300) => {
   switch (direction) {
     case "up":
       await robot.keyTap("pageup");
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "down":
       await robot.keyTap("pagedown");
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "left":
       await robot.scrollMouse(amount * -1, 0);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     case "right":
       await robot.scrollMouse(amount, 0);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       break;
     default:
       throw new AiError("Direction not found");
@@ -199,7 +199,7 @@ const click = async (x, y, button = "left", click = "single") => {
   robot.moveMouseSmooth(x, y, 0.1);
   await delay(1000); // wait for the mouse to move
   robot.mouseClick(button, double);
-  await redraw.wait(2000);
+  await redraw.wait(10000);
   return;
 };
 
@@ -210,7 +210,7 @@ const hover = async (x, y) => {
   y = parseInt(y);
 
   await robot.moveMouseSmooth(x, y, 0.1);
-  await redraw.wait(2000);
+  await redraw.wait(10000);
 
   return;
 };
@@ -236,6 +236,7 @@ let commands = {
 
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     const mdStream = logger.createMarkdownStreamLogger();
     let response = await sdk.req(
@@ -270,6 +271,7 @@ let commands = {
     // take a screenshot
     log("info", "");
     log("info", chalk.dim("thinking..."), true);
+    log("info", "");
 
     let response = await sdk.req("hover/image", {
       needle: description,
@@ -313,7 +315,7 @@ let commands = {
     await redraw.start();
     string = string.toString();
     await robot.typeString(string);
-    await redraw.wait(1000);
+    await redraw.wait(10000);
     return;
   },
   // press keys
@@ -371,7 +373,7 @@ let commands = {
     // finally, press the keys
     robot.keyTap(keysPressed[0], modsToPress);
 
-    await redraw.wait(2000);
+    await redraw.wait(10000);
 
     // keyTap will release the normal keys, but will not release modifier keys
     // so we need to release the modifier keys manually
@@ -457,7 +459,7 @@ let commands = {
           ),
           true,
         );
-        await redraw.wait(2000);
+        await redraw.wait(10000);
       }
     }
 
@@ -486,7 +488,7 @@ let commands = {
       await robot.keyTap("f", commandOrControl);
       // type the text
       await robot.typeString(text);
-      await redraw.wait(2000);
+      await redraw.wait(10000);
       await robot.keyTap("escape");
     } catch (e) {
       console.log(e);

package/lib/network.js

@@ -0,0 +1,3 @@
+
+
+

package/lib/redraw.js

@@ -3,6 +3,79 @@ const os = require("os");
 const path = require("path");
 const { compare } = require("odiff-bin");
 
+// network
+const si = require('systeminformation');
+const chalk = require('chalk');
+let lastTxBytes = null;
+let lastRxBytes = null;
+let measurements = [];
+let networkSettled = true;
+let lastUnsettled = null;
+let watchNetwork = null;
+
+async function resetNetwork() {
+  lastTxBytes = null;
+  lastRxBytes = null;
+  measurements = [];
+  networkSettled = true;
+  lastUnsettled = null;
+}
+
+async function updateNetwork() {
+    si.networkStats().then(data => {
+      let thisRxBytes = data[0].rx_bytes;
+      let thisTxBytes = data[0].tx_bytes;
+  
+      let diffRxBytes = lastRxBytes !== null ? thisRxBytes - lastRxBytes : 0;
+      let diffTxBytes = lastTxBytes !== null ? thisTxBytes - lastTxBytes : 0;
+  
+      lastRxBytes = thisRxBytes;
+      lastTxBytes = thisTxBytes;
+  
+      measurements.push({ rx: diffRxBytes, tx: diffTxBytes });
+  
+      if (measurements.length > 60) {
+        measurements.shift();
+      }
+  
+      let avgRx = measurements.reduce((acc, m) => acc + m.rx, 0) / measurements.length;
+      let avgTx = measurements.reduce((acc, m) => acc + m.tx, 0) / measurements.length;
+  
+      let stdDevRx = Math.sqrt(measurements.reduce((acc, m) => acc + Math.pow(m.rx - avgRx, 2), 0) / measurements.length);
+      let stdDevTx = Math.sqrt(measurements.reduce((acc, m) => acc + Math.pow(m.tx - avgTx, 2), 0) / measurements.length);
+  
+      let zIndexRx = stdDevRx !== 0 ? (diffRxBytes - avgRx) / stdDevRx : 0;
+      let zIndexTx = stdDevTx !== 0 ? (diffTxBytes - avgTx) / stdDevTx : 0;
+  
+      // log time since unsettlement
+  
+      if ((new Date().getTime() - lastUnsettled) < 2000) {
+        networkSettled = false;
+      } else {
+        
+        if ((zIndexRx < 0 && zIndexTx < 0) ) {
+          lastUnsettled = null;
+          networkSettled = true;
+        } else {
+          lastUnsettled = new Date().getTime();
+          networkSettled = false;
+        }
+  
+      }
+    
+      if (process.env["DEV"]) {
+
+        if (!networkSettled) {
+          console.log(chalk.red(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`));
+        } else {
+          console.log(new Date().getTime(), `,${zIndexRx}`, `,${zIndexTx}`);
+        }
+        
+      }
+  
+    });
+}
+
 async function imageIsDifferent(image1Url, image2Url) {
 
   // generate a temporary file path
@@ -36,17 +109,21 @@ async function imageIsDifferent(image1Url, image2Url) {
 let startImage = null;
 
 async function start() {
+  resetNetwork();
+  watchNetwork = setInterval(updateNetwork, 500);
   startImage = await captureScreenPNG();
   return startImage;
 }
 
 async function checkCondition(resolve, startTime, timeoutMs) {
   let nowImage = await captureScreenPNG();
-  let result = await imageIsDifferent(startImage, nowImage);
+  let screenDiff = await imageIsDifferent(startImage, nowImage);
 
-  if (result) {
+  if (screenDiff && networkSettled) {
+    clearInterval(watchNetwork);
     resolve("Condition met");
   } else if (Date.now() - startTime >= timeoutMs) {
+    clearInterval(watchNetwork);
     resolve("Timeout reached");
   } else {
     setTimeout(() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.0.72",
+  "version": "4.0.74",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {