testdriverai 4.0.33 → 4.0.34

package/index.js

@@ -170,7 +170,7 @@ const haveAIResolveError = async (error, markdown, depth = 0, undo = false) => {
 
   let image;
   if (error.attachScreenshot) {
-    image = await system.captureScreenBase64();
+    image = await system.captureScreenBase64(.5);
   } else {
     image = null;
   }
@@ -196,7 +196,7 @@ const check = async () => {
 
   console.log('')
   log.log('info', chalk.dim('checking...'), 'testdriver')  
-  let image = await system.captureScreenBase64();
+  let image = await system.captureScreenBase64(.5);
   let mousePosition = await system.getMousePosition();
   let activeWindow = await system.activeWin();
   return await sdk.req('check', {tasks, image, mousePosition, activeWindow});
@@ -382,7 +382,7 @@ const humanInput = async (currentTask, validateAndLoop = false) => {
 
   log.log('info', '');
 
-  let image = await system.captureScreenBase64();
+  let image = await system.captureScreenBase64(.5);
   let message = await sdk.req('input', {
     input: currentTask, 
     mousePosition: await system.getMousePosition(),
@@ -409,7 +409,7 @@ const generate = async (type) => {
 
   log.log('info', '');
 
-  let image = await system.captureScreenBase64();
+  let image = await system.captureScreenBase64(.5);
   let message = await sdk.req('generate', {
     type,
     image});
@@ -420,9 +420,8 @@ const generate = async (type) => {
 
   // for each testPrompt
   for (const testPrompt of testPrompts) {
-    // write a file called testprompt.headings[0].replace(' ', '-').toLowerCase().md
     // with the contents of the testPrompt
-    let fileName = testPrompt.headings[0].trim().replace(/ /g, '-').toLowerCase() + '.md';
+    let fileName = sanitizeFilename(testPrompt.headings[0]).trim().replace(/ /g, '-').toLowerCase() + '.md';
     let path1 = path.join(process.cwd(), 'testdriver', '.generate', fileName);
     let contents = testPrompt.listsOrdered[0].map((item, index) => `${index + 1}. /explore ${item}`).join('\n');
     fs.writeFileSync(path1, contents);
@@ -634,7 +633,7 @@ let summarize = async (error = null) => {
   log.log('info', chalk.dim('reviewing test...'), true);
 
   // let text = prompts.summarize(tasks, error);
-  let image = await system.captureScreenBase64();
+  let image = await system.captureScreenBase64(.5);
 
   log.log('info', chalk.dim('summarizing...'), true);
 

package/lib/commands.js

@@ -1,7 +1,7 @@
 // the actual commands to interact with the system
 const sdk = require('./sdk');
 const chalk = require('chalk');
-const {captureScreenBase64, captureScreenPNG, platform, getDisplayMultiple, activeWin} = require('./system');
+const {captureScreenBase64, captureScreenPNG, platform, activeWin} = require('./system');
 const {log} = require('./logger');
 const keymap = require('./keymap');
 const { focusApplication } = require('./focus-application');
@@ -31,7 +31,6 @@ class AiError extends Error {
 const commandOrControl = process.platform === 'darwin' ? 'command' : 'control';
 
 const findImageOnScreen = async (relativePath, haystack, restrictToWindow) => {
-  let displayMultiple = await getDisplayMultiple();
 
   // move the file from filePath to `testdriver/screenshots`
   let rootpath = path.join(cwd(), `testdriver`, `screenshots`, platform());
@@ -94,10 +93,10 @@ const findImageOnScreen = async (relativePath, haystack, restrictToWindow) => {
 
           results = results.filter((el) => {
 
-            return el.centerX * displayMultiple > activeWindow.bounds.x 
-              && el.centerX * displayMultiple < activeWindow.bounds.x + activeWindow.bounds.width 
-              && el.centerY * displayMultiple > activeWindow.bounds.y 
-              && el.centerY * displayMultiple < activeWindow.bounds.y + activeWindow.bounds.height;
+            return el.centerX > activeWindow.bounds.x 
+              && el.centerX < activeWindow.bounds.x + activeWindow.bounds.width 
+              && el.centerY > activeWindow.bounds.y 
+              && el.centerY < activeWindow.bounds.y + activeWindow.bounds.height;
   
           });
         }
@@ -250,7 +249,7 @@ let commands = {
       button,
       clickType,
       description, 
-      displayMultiple: await getDisplayMultiple()
+      displayMultiple: 1
     });
 
     if (!response.data) {
@@ -275,7 +274,7 @@ let commands = {
       intent: action,     
       button,
       clickType,
-      displayMultiple: await getDisplayMultiple()
+      displayMultiple: 1
     });
 
     if (!response?.data) {
@@ -293,16 +292,14 @@ let commands = {
 
     let result = await findImageOnScreen(relativePath, image)
 
-    let displayMultiple = await getDisplayMultiple();
-
     if (!result) {
       throw new AiError(`Image not found: ${relativePath}`)
     } else {
 
       if (action === 'click') {
-        await click(result.centerX * displayMultiple, result.centerY * displayMultiple, button, clickType);
+        await click(result.centerX, result.centerY, button, clickType);
       } else if (action === 'hover') {
-        await hover(result.centerX * displayMultiple, result.centerY * displayMultiple);
+        await hover(result.centerX, result.centerY);
       }
     }
     

package/lib/sdk.js

@@ -1,7 +1,7 @@
 
 // custom "sdk" for calling our API
-// const root = "https://replayable-dev-ian-mac-m1-16.ngrok.io";
-const root = "https://replayable-api-production.herokuapp.com";
+const root = "https://replayable-dev-ian-mac-m1-16.ngrok.io";
+// const root = "https://replayable-api-production.herokuapp.com";
 const chalk = require('chalk');
 const axios = require('axios');
 const session = require('./session')

package/lib/system.js

@@ -8,7 +8,6 @@ const activeWindow = require('active-win');
 const robot = require('robotjs');
 const sharp = require('sharp')
 
-let displayMultiple = 0;
 let primaryDisplay = null;
 
 // get the primary display
@@ -27,54 +26,34 @@ const getSystemInformationOsInfo = async() => {
   return await si.osInfo();
 }
 
-// this hepls us understand how to scale things for retina screens
-const calculateDisplayMultiple = async () => {
-  let primaryDisplay = await getPrimaryDisplay();
-  displayMultiple = primaryDisplay.currentResX / primaryDisplay.resolutionX;
-};
-
-const getDisplayMultiple = async () => {
-
-  if (!displayMultiple) {
-    await calculateDisplayMultiple();
-  }
-  
-  return displayMultiple;
-};
-
 const tmpFilename = () => {
   return path.join(os.tmpdir(), `${new Date().getTime() + Math.random()}.png`); 
 }
 
-// our handy screenshot function
-const captureScreenBase64 = async () => {
-
+const captureAndResize = async(scale = 1) => {
   let primaryDisplay = await getPrimaryDisplay();
   
   let step1 = tmpFilename();
   let step2 = tmpFilename();
 
-  await screenshot({ filename: step1, format: 'png' });
+  let step1Screenshot = await screenshot({ filename: step1, format: 'png' });
 
-  // // resize to 1:1 px ratio
+// resize to 1:1 px ratio
   await sharp(step1)
-    .resize(primaryDisplay.currentResX, primaryDisplay.currentResY)
+    .resize(Math.floor(primaryDisplay.currentResX * scale), Math.floor(primaryDisplay.currentResY * scale))
     .toFile(step2);
 
-  // let image = fs.readFileSync(step1, "base64");
-  let image = fs.readFileSync(step2, "base64");
-
-  return image;
+  return step2;
+}
 
+// our handy screenshot function
+const captureScreenBase64 = async (scale = 1) => {
+  let step2 = await captureAndResize(scale);
+  return fs.readFileSync(step2, "base64");
 };
 
-const captureScreenPNG = async () => {
-      
-    let step1 = tmpFilename();
-    await screenshot({ filename: step1, format: 'png' });
-    
-    return step1;
-  
+const captureScreenPNG = async (scale = 1) => {
+  return await captureAndResize(scale);
 }
 
 const platform = () => {
@@ -103,8 +82,6 @@ const getMousePosition = async () => {
 module.exports = {
   captureScreenBase64,
   captureScreenPNG,
-  getDisplayMultiple,
-  calculateDisplayMultiple,
   getMousePosition,
   primaryDisplay,
   activeWin,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.0.33",
+  "version": "4.0.34",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {