testdriverai 7.2.12 → 7.2.14

package/.github/workflows/acceptance-linux-scheduled.yaml

@@ -0,0 +1,70 @@
+name: Scheduled Acceptance Tests (Linux)
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    # Every 6 hours
+    - cron: '0 */6 * * *'
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  test-linux:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+    
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Install Sentry CLI
+      run: npm install -g @sentry/cli
+
+    - name: Sentry Cron Check-in (Start)
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        CHECK_IN_ID=$(sentry-cli monitors checkin testdriver-linux-acceptance --status in_progress 2>&1 | grep -oE '[a-f0-9-]{36}' | head -1)
+        echo "CHECK_IN_ID=$CHECK_IN_ID" >> $GITHUB_ENV
+
+    - name: Run Linux tests
+      id: tests
+      run: npx vitest run test/testdriver/*.test.mjs
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+        TD_OS: linux
+
+    - name: Sentry Cron Check-in (Success)
+      if: success()
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        sentry-cli monitors checkin testdriver-linux-acceptance --check-in-id ${{ env.CHECK_IN_ID }} --status ok
+
+    - name: Sentry Cron Check-in (Failure)
+      if: failure()
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        sentry-cli monitors checkin testdriver-linux-acceptance --check-in-id ${{ env.CHECK_IN_ID }} --status error
+
+    - name: Upload test results to Sentry Prevent
+      if: ${{ !cancelled() }}
+      uses: getsentry/prevent-action@v0
+      with:
+        token: ${{ secrets.SENTRY_PREVENT_TOKEN }}

package/.github/workflows/acceptance-windows-scheduled.yaml

@@ -0,0 +1,70 @@
+name: Scheduled Acceptance Tests (Windows)
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    # Every day at midnight UTC
+    - cron: '0 0 * * *'
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  test-windows:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+        cache: 'npm'
+    
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Install Sentry CLI
+      run: npm install -g @sentry/cli
+
+    - name: Sentry Cron Check-in (Start)
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        CHECK_IN_ID=$(sentry-cli monitors checkin testdriver-windows-acceptance --status in_progress 2>&1 | grep -oE '[a-f0-9-]{36}' | head -1)
+        echo "CHECK_IN_ID=$CHECK_IN_ID" >> $GITHUB_ENV
+
+    - name: Run Windows tests
+      id: tests
+      run: npx vitest run test/testdriver/*.test.mjs
+      env:
+        TD_API_KEY: ${{ secrets.TD_API_KEY }}
+        TD_OS: windows
+
+    - name: Sentry Cron Check-in (Success)
+      if: success()
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        sentry-cli monitors checkin testdriver-windows-acceptance --check-in-id ${{ env.CHECK_IN_ID }} --status ok
+
+    - name: Sentry Cron Check-in (Failure)
+      if: failure()
+      env:
+        SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+        SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      run: |
+        sentry-cli monitors checkin testdriver-windows-acceptance --check-in-id ${{ env.CHECK_IN_ID }} --status error
+
+    - name: Upload test results to Sentry Prevent
+      if: ${{ !cancelled() }}
+      uses: getsentry/prevent-action@v0
+      with:
+        token: ${{ secrets.SENTRY_PREVENT_TOKEN }}

package/.github/workflows/acceptance.yaml

@@ -37,6 +37,8 @@ jobs:
     - name: Upload test results to Sentry Prevent
       if: ${{ !cancelled() }}
       uses: getsentry/prevent-action@v0
+      with:
+        token: ${{ secrets.SENTRY_PREVENT_TOKEN }}
 
     - name: Publish Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2

package/.testdriver/last-sandbox

@@ -1,7 +1,7 @@
 {
-  "sandboxId": "i-034317335b2eb9678",
-  "os": "windows",
+  "sandboxId": "i4kn8zrl6y0483ooe10pl",
+  "os": "linux",
   "ami": null,
   "instanceType": null,
-  "timestamp": "2025-12-22T23:09:20.319Z"
+  "timestamp": "2026-01-06T19:39:48.956Z"
 }

package/agent/lib/commander.js

@@ -110,10 +110,8 @@ commands:
         case "hover-text":
           emitter.emit(events.log.log, generator.jsonToManual(object));
           response = await commands["hover-text"](
-            object.text,
             object.description,
             object.action,
-            object.method,
             object.timeout,
           );
           break;

package/agent/lib/commands.js

@@ -719,47 +719,40 @@ const createCommands = (
     hover: hover,
     /**
      * Hover over text on screen
-     * @param {Object|string} options - Options object or text (for backward compatibility)
-     * @param {string} options.text - Text to find and hover over
-     * @param {string|null} [options.description] - Optional description of the element
+     * @param {Object|string} options - Options object or description (for backward compatibility)
+     * @param {string} options.description - Description of the element to find
      * @param {string} [options.action='click'] - Action to perform
      * @param {number} [options.timeout=5000] - Timeout in milliseconds
      */
     "hover-text": async (...args) => {
-      let text, description, action, timeout;
+      let description, action, timeout;
       
       // Handle both object and positional argument styles
-      if (isObjectArgs(args, ['text', 'description', 'action', 'timeout'])) {
-        ({ text, description = null, action = 'click', timeout = 5000 } = args[0]);
+      if (isObjectArgs(args, ['description', 'action', 'timeout'])) {
+        ({ description, action = 'click', timeout = 5000 } = args[0]);
       } else {
-        // Legacy positional: hoverText(text, description, action, timeout)
-        [text, description = null, action = 'click', timeout = 5000] = args;
+        // Legacy positional: hoverText(description, action, timeout)
+        [description, action = 'click', timeout = 5000] = args;
+      }
+      
+      if (!description) {
+        throw new CommandError("hover-text requires a description parameter");
       }
       
+      description = description.toString();
+      
       emitter.emit(
         events.log.narration,
-        theme.dim(
-          `searching for "${text}"${description ? ` (${description})` : ""}...`,
-        ),
+        theme.dim(`searching for "${description}"...`),
       );
 
-      text = text ? text.toString() : null;
-
       // wait for the text to appear on screen
-      await commands["wait-for-text"]({ text, timeout });
-
-      description = description ? description.toString() : null;
+      await commands["wait-for-text"]({ text: description, timeout });
 
       emitter.emit(events.log.narration, theme.dim("thinking..."), true);
 
-      // Combine text and description into element parameter
-      let element = text;
-      if (description) {
-        element = `"${text}" with description ${description}`;
-      }
-
       let response = await sdk.req("find", {
-        element,
+        element: description,
         image: await system.captureScreenBase64(),
       });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.2.12",
+  "version": "7.2.14",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "exports": {

package/test/testdriver/chrome-extension.test.mjs

@@ -35,7 +35,7 @@ describe("Chrome Extension Test", () => {
     // When clicked, it shows a popup with "Hello Extensions"
     
     // First, let's verify Chrome loaded and we can see the page
-    const pageResult = await testdriver.assert("the testdriver.ai website is visible");
+    const pageResult = await testdriver.assert("the google.com website is visible");
     expect(pageResult).toBeTruthy();
 
     // Click on the extensions button (puzzle piece icon) in Chrome toolbar
@@ -57,8 +57,7 @@ describe("Chrome Extension Test", () => {
     // Launch Chrome with Loom loaded by its Chrome Web Store ID
     // Loom ID: liecbddmkiiihnedobmlmillhodjkdmb
     await testdriver.provision.chromeExtension({
-      extensionId: 'liecbddmkiiihnedobmlmillhodjkdmb',
-      url: 'https://testdriver.ai'
+      extensionId: 'liecbddmkiiihnedobmlmillhodjkdmb'
     });
 
     // Click on the extensions button (puzzle piece icon) in Chrome toolbar

package/test/testdriver/act.test.mjs

@@ -1,30 +0,0 @@
-/**
- * TestDriver SDK - Act Test (Vitest)
- * Tests the AI exploratory loop (act) functionality
- */
-
-import { describe, expect, it } from "vitest";
-import { TestDriver } from "../../lib/vitest/hooks.mjs";
-
-describe("Act Test", () => {
-  it("should use act to search for testdriver on Google", async (context) => {
-    const testdriver = TestDriver(context, { newSandbox: true });
-    
-    // provision.chrome() automatically calls ready() and starts dashcam
-    await testdriver.provision.chrome({
-      url: 'https://www.google.com',
-    });
-
-    // Use act to search for testdriver
-    let actRes = await testdriver.act("click on the empty search box, type 'testdriver', and hit enter. do not click the plus button in the search bar");
-
-    console.log("Act response:", actRes);
-
-    // Assert the search results are displayed
-    const result = await testdriver.assert(
-      "search results for testdriver are visible",
-    );
-
-    expect(result).toBeTruthy();
-  });
-});

package/tests/example.test.js

@@ -1,33 +0,0 @@
-import { test, expect } from 'vitest';
-import { TestDriver } from 'testdriverai/vitest/hooks';
-import { login } from './login.js';
-
-test('should login and add item to cart', async (context) => {
-
-  // Create TestDriver instance - automatically connects to sandbox
-  const testdriver = TestDriver(context);
-
-  // Launch chrome and navigate to demo app
-  await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
-
-  // Use the login snippet to handle authentication
-  // This demonstrates how to reuse test logic across multiple tests
-  await login(testdriver);
-
-  // Add item to cart
-  const addToCartButton = await testdriver.find(
-    'add to cart button under TestDriver Hat'
-  );
-  await addToCartButton.click();
-
-  // Open cart
-  const cartButton = await testdriver.find(
-    'cart button in the top right corner'
-  );
-  await cartButton.click();
-
-  // Verify item in cart
-  const result = await testdriver.assert('TestDriver Hat is in the cart');
-  expect(result).toBeTruthy();
-  
-});

package/tests/login.js

@@ -1,28 +0,0 @@
-/**
- * Login snippet - reusable login function
- * 
- * This demonstrates how to create reusable test snippets that can be
- * imported and used across multiple test files.
- */
-export async function login(testdriver) {
-
-  // The password is displayed on screen, have TestDriver extract it
-  const password = await testdriver.extract('the password');
-
-  // Find the username field
-  const usernameField = await testdriver.find(
-    'Username, label above the username input field on the login form'
-  );
-  await usernameField.click();
-
-  // Type username
-  await testdriver.type('standard_user');
-
-  // Enter password form earlier 
-  // Marked as secret so it's not logged or stored
-  await testdriver.pressKeys(['tab']);
-  await testdriver.type(password, { secret: true });
-
-  // Submit the form
-  await testdriver.find('submit button on the login form').click();
-}

package/tests/table-sort-enrollments.test.mjs

@@ -1,72 +0,0 @@
-/**
- * Test case 7: Sort by Enrollments (ascending, numeric)
- * 
- * 1. Open page: https://practicetestautomation.com/practice-test-table/
- * 2. Set Sort by = Enrollments
- * 3. Verify visible rows are ordered from smallest to largest enrollment
- * 4. Verify numbers with commas sort correctly
- * 5. Fails if the sort is lexicographic (string-based) instead of numeric
- * 
- * EXPECTED: This test should PASS if enrollments are sorted ascending numerically
- *           This test should FAIL if sort is lexicographic or descending
- */
-import { describe, expect, it } from "vitest";
-import { TestDriver } from "../lib/vitest/hooks.mjs";
-
-describe("Table Sort - Enrollments", () => {
-
-  it("should sort enrollments numerically ascending", async (context) => {
-    const testdriver = TestDriver(context, { 
-      newSandbox: true, 
-      headless: false 
-    });
-    
-    // Step 1: Open the page
-    await testdriver.provision.chrome({
-      url: 'https://practicetestautomation.com/practice-test-table/',
-    });
-
-    // Scroll down to see the Sort by dropdown
-    await testdriver.scroll("down");
-    await testdriver.scroll("down");
-    
-    // Step 2: Find and click the Sort by dropdown
-    const sortDropdown = await testdriver.find("Sort by dropdown", { timeout: 15000 });
-    console.log("Sort by dropdown found:", sortDropdown.found());
-    expect(sortDropdown.found()).toBeTruthy();
-    await sortDropdown.click();
-    
-    // Select "Enrollments" from the dropdown options
-    const enrollmentsOption = await testdriver.find("Enrollments option", { timeout: 10000 });
-    console.log("Enrollments option found:", enrollmentsOption.found());
-    expect(enrollmentsOption.found()).toBeTruthy();
-    await enrollmentsOption.click();
-    
-    // Scroll to the TOP of the page to see the first rows of the sorted table
-    await testdriver.scroll("up");
-    await testdriver.scroll("up");
-    await testdriver.scroll("up");
-    
-    // Small wait for sort to complete
-    await testdriver.pressKeys([""]); // No-op to add small delay
-    
-    // Scroll down just enough to see the table header and first data rows
-    await testdriver.scroll("down");
-    
-    // Step 3: Verify the sort is NUMERIC ASCENDING (smallest to largest)
-    // For numeric ascending: 10 < 11 < 50 < 1000 < 1200 < 1365
-    // If lexicographic: "1,000" < "10" < "11" < "1,200" < "1365" < "50" (wrong - strings compare char by char)
-    // 
-    // The TEST PASSES if first rows show small numbers like 10, 11, 50
-    // The TEST FAILS if first rows show large numbers like 1000, 1200, 1365
-    const sortResult = await testdriver.assert(
-      "The table is sorted by Enrollments in ASCENDING numeric order. " +
-      "The FIRST visible enrollment numbers in the table should be the SMALLEST values. " +
-      "For example, if the data contains values like 10, 11, 50, 1000, and 1365, " +
-      "then 10 or 11 should appear at the TOP of the sorted table. " +
-      "If a large number like 1365 appears first, the sort is WRONG (descending or lexicographic)."
-    );
-    console.log("Numeric ascending sort assertion result:", sortResult);
-    expect(sortResult).toBeTruthy();
-  });
-});

package/tests/table-sort-experiment.test.mjs

@@ -1,42 +0,0 @@
-/**
- * Experiment file - reconnects to existing sandbox
- * Test case 7: Sort by Enrollments (ascending, numeric)
- * Run AFTER table-sort-setup.test.mjs passes (within 2 minutes)
- */
-import { describe, expect, it } from "vitest";
-import { TestDriver } from "../lib/vitest/hooks.mjs";
-
-describe("Table Sort Experiment", () => {
-
-  it("should sort by Enrollments and verify numeric ordering", async (context) => {
-    const testdriver = TestDriver(context, { 
-      newSandbox: true, 
-      headless: false,
-      reconnect: true  // Reconnects to last sandbox
-    });
-    
-    // NO provision here! The sandbox is already running from setup.test.mjs
-    
-    // Click on the Sort By dropdown and select Enrollments
-    const sortDropdown = await testdriver.find("Sort By dropdown");
-    console.log("Sort dropdown found:", sortDropdown.found());
-    await sortDropdown.click();
-    
-    // Select Enrollments from the dropdown
-    const enrollmentsOption = await testdriver.find("Enrollments option in dropdown");
-    console.log("Enrollments option found:", enrollmentsOption.found());
-    await enrollmentsOption.click();
-    
-    // Verify the table is now sorted by enrollments (ascending, numeric)
-    // The numbers should be ordered from smallest to largest: 10, 11, 50, 1000, 1,200
-    // If lexicographic: 1,000, 1,200, 10, 11, 50 (wrong - strings sort by first character)
-    const result = await testdriver.assert("The table shows enrollment numbers in ascending numeric order where smaller numbers like 10 or 11 appear before larger numbers like 1000 or 1200");
-    console.log("Sort assertion result:", result);
-    expect(result).toBeTruthy();
-    
-    // Also verify numbers with commas sort correctly (1,200 should be after 1,000)
-    const commaResult = await testdriver.assert("Numbers with commas like 1,000 and 1,200 are sorted correctly as numbers, not as text");
-    console.log("Comma sort assertion:", commaResult);
-    expect(commaResult).toBeTruthy();
-  });
-});

package/tests/table-sort-setup.test.mjs

@@ -1,59 +0,0 @@
-/**
- * Test case 7: Sort by Enrollments (ascending, numeric)
- * 
- * 1. Open page: https://practicetestautomation.com/practice-test-table/
- * 2. Set Sort by = Enrollments
- * 3. Verify visible rows are ordered from smallest to largest enrollment
- * 4. Verify numbers with commas sort correctly
- * 5. Fails if the sort is lexicographic
- */
-import { describe, expect, it } from "vitest";
-import { TestDriver } from "../lib/vitest/hooks.mjs";
-
-describe("Table Sort - Enrollments", () => {
-
-  it("should sort enrollments numerically ascending", async (context) => {
-    const testdriver = TestDriver(context, { 
-      newSandbox: true, 
-      headless: false 
-    });
-    
-    // Step 1: Open the page
-    await testdriver.provision.chrome({
-      url: 'https://practicetestautomation.com/practice-test-table/',
-    });
-
-    // Wait for page to load - find the Sort By dropdown
-    const sortDropdown = await testdriver.find("Sort By dropdown or select box", { timeout: 30000 });
-    console.log("Sort dropdown found:", sortDropdown.found());
-    expect(sortDropdown.found()).toBeTruthy();
-    
-    // Step 2: Click the Sort By dropdown
-    await sortDropdown.click();
-    
-    // Select "Enrollments" from the dropdown options
-    const enrollmentsOption = await testdriver.find("Enrollments option in the dropdown list", { timeout: 10000 });
-    console.log("Enrollments option found:", enrollmentsOption.found());
-    await enrollmentsOption.click();
-    
-    // Step 3 & 4: Verify the sort is numeric ascending (not lexicographic)
-    // If numeric: smaller numbers (10, 11, 50) come before larger numbers (1000, 1200)
-    // If lexicographic: would sort as strings (1,000 before 10 because '1' < '1')
-    const sortResult = await testdriver.assert(
-      "The table enrollment column is sorted in ascending NUMERIC order: " +
-      "smaller enrollment numbers like 10, 11, 50 appear BEFORE larger numbers like 1000 or 1200. " +
-      "This is correct numeric sorting, not alphabetical/lexicographic sorting."
-    );
-    console.log("Numeric sort assertion result:", sortResult);
-    expect(sortResult).toBeTruthy();
-    
-    // Step 5: Verify numbers with commas are handled correctly
-    // 1,200 should be treated as 1200 (greater than 1000), not as a string
-    const commaResult = await testdriver.assert(
-      "Any enrollment numbers with commas (like 1,200 or 1,000) are sorted correctly as numbers, " +
-      "with 1,000 appearing before 1,200 in ascending order"
-    );
-    console.log("Comma number sorting result:", commaResult);
-    expect(commaResult).toBeTruthy();
-  });
-});