testdriverai 4.1.57 → 4.2.0-test.0

package/.github/workflows/test_interp.yml

@@ -0,0 +1,27 @@
+
+name: Example Todo App
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: testdriverai/action@alexrowe/yml-interpolation-test
+        with:
+          os: windows
+          branch: alexrowe/yml-interpolation
+          key: ${{secrets.TESTDRIVER_API_KEY}}
+          prompt: |
+            1. /run test.yml
+          prerun: |
+            Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "${{ env.WEBSITE_URL }}"
+
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WEBSITE_URL: "https://kzmgtp9k2gtuuw2iquk2.lite.vusercontent.net/"
+          TD_PASSWORD: "passW0rd!"

package/.github/workflows/trigger_testdriver_dev.yml

@@ -1,8 +1,8 @@
 name: Trigger Testdriver Dev Workflow
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
+#
+# on:
+#   pull_request:
+#     types: [opened, synchronize, reopened]
 
 env:
   API_REF: main

package/agent.js

@@ -202,11 +202,8 @@ if (!commandHistory.length) {
   ];
 }
 
-const exit = async (failed = true, shouldSave = false) => {
-  
-  if (shouldSave) {
-    await save();
-  }
+const exit = async (failed = true) => {
+  await save();
 
   analytics.track("exit", { failed });
 
@@ -644,16 +641,6 @@ const actOnMarkdown = async (content, depth, pushToHistory = false) => {
 // simple function to backfill the chat history with a prompt and
 // then call `promptUser()` to get the user input
 const firstPrompt = async () => {
-
-  // should be start of new session
-  const sessionRes = await sdk.req("session/start", {
-    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
-    mousePosition: await system.getMousePosition(),
-    activeWindow: await system.activeWin(),
-  });
-
-  session.set(sessionRes.data.id);
-
   // readline is what allows us to get user input
   rl = readline.createInterface({
     terminal: true,
@@ -694,7 +681,7 @@ const firstPrompt = async () => {
     if (input.indexOf("/summarize") == 0) {
       await summarize();
     } else if (input.indexOf("/quit") == 0) {
-      await exit(false, true);
+      await exit();
     } else if (input.indexOf("/save") == 0) {
       await save({ filepath: commands[1] });
     } else if (input.indexOf("/undo") == 0) {
@@ -704,7 +691,7 @@ const firstPrompt = async () => {
     } else if (input.indexOf("/manual") == 0) {
       await manualInput(commands.slice(1).join(" "));
     } else if (input.indexOf("/run") == 0) {
-      await run(commands[1], commands[2] == "true", commands[3] == "true");
+      await run(commands[1], commands[2], commands[3]);
     } else if (input.indexOf("/generate") == 0) {
       await generate(commands[1], commands[2]);
     } else {
@@ -718,7 +705,7 @@ const firstPrompt = async () => {
   // if file exists, load it
   if (fs.existsSync(thisFile)) {
     analytics.track("load");
-    let object = await generator.hydrateFromYML(
+    let object = await generator.ymlToHistory(
       fs.readFileSync(thisFile, "utf-8"),
     );
 
@@ -848,7 +835,7 @@ let save = async ({ filepath = thisFile, silent = false } = {}) => {
   }
 
   // write reply to /tmp/oiResult.log.log
-  let regression = await generator.dumpToYML(executionHistory);
+  let regression = await generator.historyToYml(executionHistory);
   try {
     fs.writeFileSync(filepath, regression);
   } catch (e) {
@@ -875,10 +862,9 @@ ${regression}
 // this will load a regression test from a file location
 // it parses the markdown file and executes the codeblocks exactly as if they were
 // generated by the AI in a single prompt
-let run = async (file, shouldSave = false, shouldExit = true) => {
+let run = async (file, overwrite = false, shouldExit = true) => {
 
   setTerminalWindowTransparency(true);
-  emitter.emit(events.interactive, false);
 
   log.log("info", chalk.cyan(`running ${file}...`));
 
@@ -898,9 +884,12 @@ let run = async (file, shouldSave = false, shouldExit = true) => {
     await exit(true);
   }
 
+  console.log(yml)
+  console.log(process.env)
   // Inject environment variables into any ${VAR} strings
   yml = parser.interpolate(yml, process.env);
 
+  console.log(yml)
   let ymlObj = null;
   try {
     ymlObj = await yaml.load(yml);
@@ -948,14 +937,13 @@ ${yaml.dump(step)}
     await actOnMarkdown(markdown, 0, true);
   }
 
-  if (shouldSave) {
+  if (overwrite || overwrite == "true") {
     await save({ filepath: file });
   }
 
   setTerminalWindowTransparency(false);
-  emitter.emit(events.interactive, true);
 
-  if (shouldExit) {
+  if (shouldExit || shouldExit == "true") {
     await summarize();
     await exit(false);
   }
@@ -1071,6 +1059,15 @@ const start = async () => {
     console.log("");
   }
 
+  // should be start of new session
+  const sessionRes = await sdk.req("session/start", {
+    systemInformationOsInfo: await system.getSystemInformationOsInfo(),
+    mousePosition: await system.getMousePosition(),
+    activeWindow: await system.activeWin(),
+  });
+
+  session.set(sessionRes.data);
+
   analytics.track("command", { command: thisCommand, file: thisFile });
 
   if (thisCommand == "edit") {

package/chrome.ps1

@@ -0,0 +1,33 @@
+<#
+.SYNOPSIS
+Launches Google Chrome and navigates to a specified URL.
+
+.DESCRIPTION
+This PowerShell script will check for the existence of the Google Chrome
+browser at a predefined location, and if found, will open it and navigate to
+the URL provided as an argument to the script.
+
+.PARAMETER url
+The URL that Google Chrome will navigate to upon launch.
+
+.EXAMPLE
+.\chrome.ps1 -url "http://www.example.com"
+This example will open Google Chrome and navigate to http://www.example.com.
+
+.NOTES
+Google Chrome must be installed in the default location for this script
+to function correctly.
+
+#>
+
+param(
+    [string]$url
+)
+
+$chromePath = "C:\Program Files\Google\Chrome\Application\chrome.exe"
+
+if (Test-Path $chromePath) {
+    Start-Process $chromePath $url
+} else {
+    Write-Error "Chrome is not installed at the expected location: $chromePath"
+}

package/lib/cli.js

@@ -0,0 +1,22 @@
+const { program } = require("commander");
+
+program.description("");
+program
+  .argument("[command]", "Command to run")
+  .option(
+    "-c, --cwd <string>",
+    "working directory to run in"
+  )
+  .option(
+    "-i, --id <string>",
+    "ID of server to connect to"
+  )
+  .option(
+    "-o, --output-file <string>",
+    "Output file to write the output of the command"
+  )
+  .option(
+    "-e, --env <string>",
+    "Extra environment variables"
+  )
+  .parse();

package/lib/commands.js

@@ -160,12 +160,15 @@ const assert = async (assertion, shouldThrow = false, async = false) => {
     return handleAssertResponse(response.data);
   }
 };
-const scroll = async (direction = "down", amount = 300, method = "mouse") => {
+const scroll = async (direction = "down", amount = 300, method = "keyboard") => {
   await redraw.start();
 
   amount = parseInt(amount);
 
   if (method === "mouse") {
+    // after experimenting, 200 is a good default for mouse, mostly as mouse will be called only when the user asks for it
+    // and that happens when keyboard scrolling cannot do things when there's a pop up that needs to be scrolled over and 
+    // pop ups are usually smaller and needs a smaller amount of scrolling
     amount = 200;
   }
 

package/lib/generator.js

@@ -2,7 +2,6 @@
 const yaml = require("js-yaml");
 const chalk = require("chalk");
 const package = require("../package.json");
-const session = require("./session");
 // do the actual parsing
 // this library is very strict
 // note that errors are sent to the AI will it may self-heal
@@ -53,30 +52,25 @@ const jsonToManual = function (json, colors = true) {
   return params;
 };
 
-const dumpToYML = async function (inputArray) {
-
+const historyToYml = async function (inputArray) {
   // use yml dump to convert json to yml
   let yml = await yaml.dump({
     version: package.version,
-    session: session.get(),
     steps: inputArray,
   });
 
   return yml;
 };
 
-const hydrateFromYML = async function (yml) {
+const ymlToHistory = async function (yml) {
   // use yml load to convert yml to json
   let json = await yaml.load(yml);
-
-  session.set(json.session);
-
   return json;
 };
 
 module.exports = {
   manualToYml,
-  dumpToYML,
-  hydrateFromYML,
+  historyToYml,
+  ymlToHistory,
   jsonToManual,
 };

package/lib/sdk.js

@@ -106,7 +106,7 @@ const req = async (path, data, onChunk) => {
     responseType: typeof onChunk === "function" ? "stream" : "json",
     data: {
       ...data,
-      session: session.get(),
+      session: session.get()?.id,
       stream: typeof onChunk === "function",
     },
   };

package/lib/session.js

@@ -5,8 +5,6 @@ module.exports = {
     return session;
   },
   set: (s) => {
-    if (s) {
-      session = s;
-    }
+    session = s;
   },
 };

package/lib/system.js

@@ -2,11 +2,12 @@
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
+const screenshot = require("screenshot-desktop");
 const si = require("systeminformation");
+const activeWindow = require("active-win");
 const robot = require("robotjs");
 const sharp = require("sharp");
 const { emitter, events } = require("./events.js");
-const { Monitor } = require("node-screenshots");
 
 let primaryDisplay = null;
 
@@ -47,11 +48,7 @@ const captureAndResize = async (scale = 1, silent = false, mouse = false) => {
     let step1 = tmpFilename();
     let step2 = tmpFilename();
 
-    const monitors = Monitor.all();
-    const primaryMonitor = monitors.find(monitor => monitor.isPrimary);
-    const image = await primaryMonitor.captureImage(); // Capture the image asynchronously
-    const buffer = await image.toPng(); // Convert the image to PNG format
-    fs.writeFileSync(step1, buffer); // Save the image to a file
+    await screenshot({ filename: step1, format: "png" });
 
     // Fetch the mouse position
     const mousePos = robot.getMousePos();
@@ -114,25 +111,9 @@ const platform = () => {
   return platform;
 };
 
-// Import get-windows using dynamic import for ES module compatibility
-let activeWindowFn = null;
-const initializeActiveWindow = async () => {
-  if (!activeWindowFn) {
-    const { activeWindow } = await import('get-windows');
-    activeWindowFn = activeWindow;
-  }
-  return activeWindowFn;
-};
-
 // this is the focused window
 const activeWin = async () => {
-  try {
-    const activeWindow = await initializeActiveWindow();
-    return await activeWindow();
-  } catch (error) {
-    console.error('Error getting active window:', error);
-    return null;
-  }
+  return await activeWindow();
 };
 
 const getMousePosition = async () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "4.1.57",
+  "version": "4.2.0-test.0",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {
@@ -17,6 +17,7 @@
   "license": "ISC",
   "dependencies": {
     "@electerm/strip-ansi": "^1.0.0",
+    "active-win": "^8.2.1",
     "axios": "^1.7.7",
     "chalk": "^4.1.2",
     "cli-progress": "^3.12.0",
@@ -24,7 +25,6 @@
     "decompress": "^4.2.1",
     "dotenv": "^16.4.5",
     "electron": "^33.0.2",
-    "get-windows": "^9.2.0",
     "jimp": "^0.22.12",
     "js-yaml": "^4.1.0",
     "mac-screen-capture-permissions": "^2.1.0",
@@ -34,13 +34,14 @@
     "marky": "^1.2.5",
     "node-ipc": "^12.0.0",
     "node-notifier": "^10.0.1",
-    "node-screenshots": "0.2.1",
     "odiff-bin": "^3.1.2",
     "prompts": "^2.4.2",
     "remark-parse": "^11.0.0",
+    "rimraf": "^5.0.5",
     "robotjs": "^0.6.0",
     "sanitize-filename": "^1.6.3",
     "say": "^0.16.0",
+    "screenshot-desktop": "^1.15.0",
     "semver": "^7.6.2",
     "sharp": "^0.33.5",
     "systeminformation": "^5.23.5",
@@ -48,19 +49,16 @@
     "uuid": "^10.0.0",
     "winston": "^3.13.0"
   },
-  "overrides": {
-    "glob": "^11.0.1",
-    "rimraf": "^5.0.10"
-  },
   "devDependencies": {
     "@eslint/js": "^9.10.0",
-    "chai": "^5.1.2",
     "esbuild": "0.20.2",
     "esbuild-plugin-fileloc": "^0.0.6",
     "eslint": "^9.10.0",
     "globals": "^15.9.0",
-    "mocha": "^10.8.2",
     "node-addon-api": "^8.0.0",
+    "node-gyp": "^10.1.0",
+    "mocha": "^10.8.2",
+    "chai": "^5.1.2",
     "prettier": "3.3.3"
   },
   "optionalDependencies": {

package/test.js

@@ -0,0 +1,31 @@
+const screenshot = require("screenshot-desktop");
+const robot = require("robotjs");
+const Jimp = require('jimp');
+const fs = require("fs");
+
+async function go () {
+    await screenshot({ filename: "shot.png", format: "png" });
+    let pos = robot.getMousePos()
+    console.log(pos);
+    let im = await Jimp.read("shot.png");
+
+    // write im as a base64 string js file that can be imported in js 
+    const base64 = await im.getBase64Async(Jimp.MIME_PNG);
+    fs.writeFileSync("pointer_image.js", `module.exports = \n"${base64}";\n`);
+
+    let pointer = await Jimp.read("./cursor.png");
+    console.log(im.getWidth(), im.getHeight());
+    im.composite(pointer, pos.x * 2 , pos.y * 2);
+
+    im.write("shot.png");
+};
+
+go();
+
+const screenshot = require("screenshot-desktop");
+
+async function go () {
+    await screenshot({ filename: "shot.png", format: "png" });
+};
+
+go();

package/test.yml

@@ -0,0 +1,18 @@
+version: 4.1.40
+steps:
+  - prompt: Enter Password n stuff
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Username 
+        description: username field
+        action: click
+      - command: type
+        text: ${TD_PASSWORD}
+      - command: hover-text
+        text: Password
+        description: Password field
+        action: click
+      - command: type
+        text: ${TD_PASSWORD}

package/.github/dependabot.yml

@@ -1,11 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "npm" # See documentation for possible values
-    directory: "/" # Location of package manifests
-    schedule:
-      interval: "weekly"