testdriverai 5.7.7 → 5.7.9

package/.github/workflows/testdriver.yml

@@ -1,4 +1,10 @@
-name: TestDriver.ai / Run / Regressions
+name: TestDriver.ai
+
+permissions:
+  actions: read
+  contents: read
+  statuses: write
+  pull-requests: write
 
 on:
   push:
@@ -6,17 +12,11 @@ on:
   pull_request:
   workflow_dispatch:
   schedule:
-    - cron: '0 13 * * *'  # Every day at 1 PM UTC (adjust if needed for timezone)
-
-permissions:
-  actions: read
-  contents: read
-  statuses: write
-  pull-requests: write
+    - cron: "0 0 * * *" 
 
 jobs:
   gather-test-files:
-    name: Setup Test Matrix (./testdriver/*.yml)
+    name: Gather Test Files
     runs-on: ubuntu-latest
     outputs:
       test_files: ${{ steps.test_list.outputs.files }}
@@ -25,44 +25,38 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: ${{ github.event.ref }}
+
       - name: Find all test files and extract filenames
         id: test_list
         run: |
-          FILES=$(ls ./testdriver/*.yml)
+          FILES=$(ls ./testdriver/*.yaml)
           FILENAMES=$(basename -a $FILES)
           FILES_JSON=$(echo "$FILENAMES" | jq -R -s -c 'split("\n")[:-1]')
           echo "::set-output name=files::$FILES_JSON"
-
   test:
     needs: gather-test-files
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       matrix:
+        os: [linux, windows]
         test: ${{ fromJson(needs.gather-test-files.outputs.test_files) }}
-        os: ['linux', 'windows']
       fail-fast: false
-    name: ${{ matrix.test }} (${{ matrix.os }})
+    name: ${{ matrix.os }} - ${{ matrix.test }}
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.ref }}
-
-      - name: Display filename being tested
-        run: echo "Running job for file: ${{ matrix.test }} on ${{ matrix.os }}"
 
       - uses: testdriverai/action@main
         with:
-          key: ${{ secrets.TESTDRIVER_API_KEY }}
-          os: ${{ matrix.os }}
-          version: "5.2.3"
-          prompt: 1. /run testdriver/${{ matrix.test }}
+          os: ${{ contains(matrix.os, 'windows') && 'windows' || 'linux' }}
+          version: "alpha"
           prerun: |
-            cd $env:TEMP
-            npm init -y
-            npm install dashcam-chrome
-            Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "${{ env.TD_WEBSITE }}"
             exit
+          key: ${{ secrets.TESTDRIVER_API_KEY }}
+          prompt: | 
+            1. /run testdriver/${{ matrix.test }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           FORCE_COLOR: "3"
+          TD_WEBSITE: "https://testdriver-sandbox.vercel.app"
+          WEBSITE_URL: "example.com"

package/agent.js

@@ -621,7 +621,7 @@ const generate = async (type, count, baseYaml, skipYaml = false) => {
   logger.info("");
 
   if (baseYaml && !skipYaml) {
-    await run(baseYaml, false, false);
+    await run(baseYaml, false, false, false);
   }
 
   let image = await system.captureScreenBase64();
@@ -1068,7 +1068,7 @@ let runRawYML = async (yml) => {
 // this will load a regression test from a file location
 // it parses the markdown file and executes the codeblocks exactly as if they were
 // generated by the AI in a single prompt
-let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
+let run = async (file = thisFile, shouldSave = false, shouldExit = true, pushToHistory = true) => {
   setTerminalWindowTransparency(true);
   emitter.emit(events.interactive, false);
 
@@ -1092,10 +1092,12 @@ let run = async (file = thisFile, shouldSave = false, shouldExit = true) => {
     logger.info(``, null);
     logger.info(chalk.yellow(`> ${step.prompt || "no prompt"}`), null);
 
-    executionHistory.push({
-      prompt: step.prompt,
-      commands: [], // run will overwrite the commands
-    });
+    if (pushToHistory) {
+      executionHistory.push({
+        prompt: step.prompt,
+        commands: [], // run will overwrite the commands
+      });
+    }
 
     if (!step.commands && !step.prompt) {
       logger.info(chalk.red("No commands or prompt found"));
@@ -1167,7 +1169,7 @@ const iffy = async (condition, then, otherwise, depth) => {
 const embed = async (file, depth) => {
   analytics.track("embed", { file });
 
-  logger.info(generator.jsonToManual({ command: "embed", file }));
+  logger.info(generator.jsonToManual({ command: "run", file }));
 
   depth = depth + 1;
 
@@ -1280,7 +1282,7 @@ const start = async () => {
   } else if (thisCommand == "run") {
     await buildEnv();
     errorLimit = 100;
-    run(thisFile);
+    run(thisFile, false, true, true);
   } else if (thisCommand == "init") {
     await init();
     process.exit(0);
@@ -1352,7 +1354,7 @@ const runPrerun = async () => {
     "prerun.yaml",
   );
   if (fs.existsSync(prerunFile)) {
-    await run(prerunFile, false, false);
+    await run(prerunFile, false, false, false);
   }
 };
 

package/lib/commands.js

@@ -53,7 +53,7 @@ class AiError extends Error {
   }
 }
 
-const commandOrControl = platform() === "darwin" ? "command" : "control";
+const commandOrControl = platform() === "mac" ? "command" : "control";
 
 const findImageOnScreen = async (relativePath, haystack, restrictToWindow) => {
   // move the file from filePath to `testdriver/screenshots`
@@ -488,11 +488,19 @@ let commands = {
         }
       });
 
+      logger.info(
+        chalk.dim(
+          `pressing ${keysPressed[0]} with modifiers: ${modsToPress.join(", ")}`,
+        ),
+        true,
+      );
+
       modsToPress.forEach((key) => {
         robot.keyToggle(key, "down");
-      });
+      })
 
       robot.keyTap(keysPressed[0], modsToPress);
+
     } else {
       await sandbox.send({ type: "press", keys: keysPressed });
     }
@@ -795,7 +803,7 @@ let commands = {
           chalk.dim(`running value of \`${plat}\` on this machine...`),
           true,
         );
-        result = await exec(windows_code, { cwd: cwd() });
+        result = await exec(windows_code, { cwd: cwd(), shell: "powershell.exe" });
       } else if (plat == "mac") {
         logger.info(
           chalk.dim(`running value of \`${plat}\` on this machine...`),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "5.7.7",
+  "version": "5.7.9",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "index.js",
   "bin": {

package/testdriver/assert.yaml

@@ -0,0 +1,7 @@
+version: 5.7.7
+session: 682f5d92811bd5a322c0e636
+steps:
+  - prompt: assert the testdriver login page shows
+    commands:
+      - command: assert
+        expect: the TestDriver.ai Sandbox login page is displayed

package/testdriver/exec-js.yaml

@@ -0,0 +1,40 @@
+version: 5.7.7
+session: 682f5fc22856e2eb40664269
+steps:
+  - prompt: fetch user data from API
+    commands:
+      - command: exec
+        output: user
+        lang: js
+        linux: >
+          const response = await
+          fetch('https://jsonplaceholder.typicode.com/users');
+
+          const user = await response.json();
+
+          console.log('user', user[0]);
+
+          result = user[0].email;
+        mac: >
+          const response = await
+          fetch('https://jsonplaceholder.typicode.com/users');
+
+          const user = await response.json();
+
+          console.log('user', user[0]);
+
+          result = user[0].email;
+      - command: hover-text
+        text: Username
+        description: input field for username
+        action: click
+      - command: type
+        text: ${OUTPUT.user}
+  - prompt: assert that the username field shows a valid email address
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: assert
+        expect: >-
+          the username field contains "Sincere@april.biz" which is a valid email
+          address

package/testdriver/exec-shell.yaml

@@ -0,0 +1,21 @@
+version: 5.1.1
+session: 67f00511acbd9ccac373edf7
+steps:
+  - prompt: launch chrome
+    commands:
+      - command: exec
+        lang: shell
+        linux: |
+          jumpapp google-chrome --disable-fre --no-default-browser-check --no-first-run "${TD_WEBSITE}" &
+          exit
+        mac: |
+          open -na "Google Chrome" --args --disable-fre --no-default-browser-check --no-first-run --disable-features=PasswordManagerEnabled "${TD_WEBSITE}" &
+          exit
+        windows:
+          Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "${TD_WEBSITE}"
+          exit
+      - command: wait-for-text
+        text: ${TD_WEBSITE}
+        timeout: 30000
+      - command: assert
+        expect: Google Chrome is running

package/testdriver/hover-image.yaml

@@ -0,0 +1,17 @@
+version: 5.7.7
+session: 682f60bf2856e2eb4066427f
+steps:
+  - commands:
+    - command: run
+      file: testdriver/snippets/login.yaml
+  - prompt: click on the image of a shopping cart
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-image
+        description: shopping cart icon next to the Cart text in the top right corner
+        action: click
+  - prompt: assert that you see an empty shopping cart
+    commands:
+      - command: assert
+        expect: Your cart is empty

package/testdriver/hover-text-with-description.yaml

@@ -0,0 +1,28 @@
+version: 5.7.7
+session: 682f62f6811bd5a322c0e726
+steps:
+  - commands:
+    - command: run
+      file: testdriver/snippets/login.yaml
+  - prompt: click on add to cart under the testdriver hat
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Add to Cart
+        description: add to cart button under TestDriver Hat
+        action: click
+  - prompt: click on the cart
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Cart
+        description: cart button in the top right corner
+        action: click
+  - prompt: assert the testdriver hat is in the cart
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: assert
+        expect: TestDriver Hat is in the cart

package/testdriver/hover-text.yaml

@@ -0,0 +1,13 @@
+version: 5.7.7
+session: 682f5aab811bd5a322c0e5a1
+steps:
+  - prompt: click on sign in
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Sign In
+        description: black button below the password field
+        action: click
+      - command: assert
+        expect: an error shows that fields are required

package/testdriver/lifecycle/prerun.yaml

@@ -0,0 +1,19 @@
+version: 5.1.1
+session: 67f00511acbd9ccac373edf7
+steps:
+  - prompt: launch chrome
+    commands:
+      - command: exec
+        lang: shell
+        linux: |
+          jumpapp google-chrome --disable-fre --no-default-browser-check --no-first-run "${TD_WEBSITE}" &
+          exit
+        mac: |
+          open -na "Google Chrome" --args --disable-fre --no-default-browser-check --no-first-run --disable-features=PasswordManagerEnabled "${TD_WEBSITE}" &
+          exit
+        windows:
+          Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "${TD_WEBSITE}"
+          exit
+      - command: wait-for-text
+        text: ${TD_WEBSITE}
+        timeout: 30000

package/testdriver/match-image.yaml

@@ -0,0 +1,15 @@
+version: 5.7.7
+session: 682f5d92811bd5a322c0e636
+steps:
+  - commands:
+    - command: run
+      file: testdriver/snippets/login.yaml
+  - prompt: assert the testdriver login page shows
+    commands:
+      - command: match-image
+        path: cart.png
+        action: click
+  - prompt: assert that you see an empty shopping cart
+    commands:
+      - command: assert
+        expect: Your cart is empty

package/testdriver/press-keys.yaml

@@ -0,0 +1,13 @@
+version: 5.7.7
+session: 682f645a2856e2eb406642f8
+steps:
+  - prompt: press command d
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: press-keys
+        keys:
+          - command
+          - d
+      - command: assert
+        expect: the bookmark dialog is open

package/testdriver/remember.yaml

@@ -1,16 +1,7 @@
-version: 5.6.2
-session: 681cdec532fef6c6f161c4a4
+version: 5.7.7
+session: 682f66241f5b5f003ee139cb
 steps:
-  - prompt: search for the current track in spotify
+  - prompt: remember the username of the user who cannot login
     commands:
       - command: focus-application
-        name: spotify
-      - command: remember
-        output: track
-        description: the track now playing in spotify
-      - command: hover-text
-        text: What do you want to play?
-        description: the spotify search bar
-        action: click
-      - command: type
-        text: ${OUTPUT.track}
+        name: Google Chrome

package/testdriver/scroll-until-text.yaml

@@ -0,0 +1,20 @@
+version: 5.7.7
+session: 682f62a1811bd5a322c0e70b
+steps:
+  - commands:
+    - command: run
+      file: testdriver/snippets/login.yaml
+  - prompt: scroll until text testdriver socks
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: scroll-until-text
+        text: testdriver socks
+        direction: down
+        method: keyboard
+  - prompt: assert testdriver socks appears on screen
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: assert
+        expect: TestDriver Socks appears on screen

package/testdriver/snippets/login.yaml

@@ -0,0 +1,23 @@
+version: 5.7.7
+session: 682f6071811bd5a322c0e6dd
+steps:
+  - prompt: log into the app
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Username
+        description: username input field
+        action: click
+      - command: type
+        text: standard_user
+      - command: hover-text
+        text: Password
+        description: password input field
+        action: click
+      - command: type
+        text: secret_password
+      - command: hover-text
+        text: Sign in
+        description: sign in button
+        action: click

package/testdriver/type.yaml

@@ -0,0 +1,29 @@
+version: 5.7.7
+session: 682f5c4c811bd5a322c0e5c1
+steps:
+  - prompt: enter standard_user within the username field
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: hover-text
+        text: Username
+        description: input field for username
+        action: click
+      - command: type
+        text: standard_user
+  - prompt: assert that standard_user shows in the username field
+    commands:
+      - command: assert
+        expect: the username field contains "standard_user"
+  - prompt: click on sign in
+    commands:
+      - command: hover-text
+        text: Sign in
+        description: black button below the password field
+        action: click
+  - prompt: assert that "please fill out this field" shows in the password field
+    commands:
+      - command: focus-application
+        name: Google Chrome
+      - command: assert
+        expect: Please fill out this field is visible near the password field

package/testdriver/focus.yaml

@@ -1,15 +0,0 @@
-version: 5.6.5
-session: 682a09d01e7bdbd8dadde346
-steps:
-  - prompt: focus chrome
-    commands:
-      - command: focus-application
-        name: Google Chrome
-  - prompt: focus chrome
-    commands:
-      - command: focus-application
-        name: Google Chrome
-  - prompt: focus chrome
-    commands:
-      - command: focus-application
-        name: Google Chrome

package/testdriver/overwrite.yaml

@@ -1,25 +0,0 @@
-version: 5.7.0
-session: 682a077a1e7bdbd8dadde32f
-steps:
-  - prompt: open chrome
-    commands:
-      - command: press-keys
-        keys:
-          - command
-          - space
-      - command: type
-        text: Google Chrome
-      - command: press-keys
-        keys:
-          - enter
-  - prompt: search for youtube
-    commands:
-      - command: press-keys
-        keys:
-          - command
-          - space
-      - command: type
-        text: Google Chrome
-      - command: press-keys
-        keys:
-          - enter

package/testdriver/spotify.yaml

@@ -1,5 +0,0 @@
-version: 5.6.2
-session: 681cdec532fef6c6f161c4a4
-steps:
-  - prompt: focus spotify
-  - prompt: focus chrome

package/testdriver/testdriver_2025-05-14T18-59-04-439Z.yaml

@@ -1,7 +0,0 @@
-version: 5.6.6
-session: 6824e7ff58a65c2836a75af3
-steps:
-  - prompt: log in
-    commands:
-      - command: focus-application
-        name: Google Chrome

package/testdriver/testdriver_2025-05-14T18-59-35-628Z.yaml

@@ -1,7 +0,0 @@
-version: 5.6.6
-session: 6824e81ca8c2f5886f8c8eb8
-steps:
-  - prompt: log in
-    commands:
-  - command: focus-application
-    name: Google Chrome