testdriverai 7.2.3 → 7.2.9

package/sdk.js

@@ -263,6 +263,48 @@ class ElementNotFoundError extends Error {
   }
 }
 
+/**
+ * Custom error class for act() failures
+ * Includes task execution details and retry information
+ */
+class ActError extends Error {
+  /**
+   * @param {string} message - Error message
+   * @param {Object} details - Additional details about the failure
+   * @param {string} details.task - The task that was attempted
+   * @param {number} details.tries - Number of check attempts made
+   * @param {number} details.maxTries - Maximum tries that were allowed
+   * @param {number} details.duration - Total execution time in milliseconds
+   * @param {Error} [details.cause] - The underlying error that caused the failure
+   */
+  constructor(message, details = {}) {
+    super(message);
+    this.name = "ActError";
+    this.task = details.task;
+    this.tries = details.tries;
+    this.maxTries = details.maxTries;
+    this.duration = details.duration;
+    this.cause = details.cause;
+    this.timestamp = new Date().toISOString();
+
+    // Capture stack trace
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, ActError);
+    }
+
+    // Enhance error message with execution details
+    this.message += `\n\n=== Act Execution Details ===`;
+    this.message += `\nTask: "${this.task}"`;
+    this.message += `\nTries: ${this.tries}/${this.maxTries}`;
+    this.message += `\nDuration: ${this.duration}ms`;
+    this.message += `\nTimestamp: ${this.timestamp}`;
+    
+    if (this.cause) {
+      this.message += `\nUnderlying error: ${this.cause.message}`;
+    }
+  }
+}
+
 /**
  * Element class representing a located or to-be-located element
  */
@@ -2188,10 +2230,7 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     const absoluteTimestamp = Date.now();
     const startTime = absoluteTimestamp;
 
-    // Log finding all action
     const { events } = require("./agent/events.js");
-    const findingMessage = formatter.formatElementsFinding(description);
-    this.emitter.emit(events.log.log, findingMessage);
 
     try {
       const screenshot = await this.system.captureScreenBase64();
@@ -2257,16 +2296,16 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
       const duration = Date.now() - startTime;
 
       if (response && response.elements && response.elements.length > 0) {
-        // Log found elements
-        const foundMessage = formatter.formatElementsFound(
+        // Single log at the end - found elements
+        const formattedMessage = formatter.formatFindAllSingleLine(
           description,
           response.elements.length,
           {
-            duration: `${duration}ms`,
+            duration: duration,
             cacheHit: response.cached || false,
           },
         );
-        this.emitter.emit(events.log.log, foundMessage);
+        this.emitter.emit(events.log.narration, formattedMessage, true);
 
         // Create Element instances for each found element
         const elements = response.elements.map((elementData) => {
@@ -2316,7 +2355,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
 
         // Log debug information when elements are found
         if (process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG) {
-          const { events } = require("./agent/events.js");
           this.emitter.emit(
             events.log.debug,
             `✓ Found ${elements.length} element(s): "${description}"`,
@@ -2330,6 +2368,19 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
 
         return elements;
       } else {
+        const duration = Date.now() - startTime;
+        
+        // Single log at the end - no elements found
+        const formattedMessage = formatter.formatFindAllSingleLine(
+          description,
+          0,
+          {
+            duration: duration,
+            cacheHit: response?.cached || false,
+          },
+        );
+        this.emitter.emit(events.log.narration, formattedMessage, true);
+
         // No elements found - track interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
         if (sessionId && this.sandbox?.send) {
@@ -2354,6 +2405,18 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
         return [];
       }
     } catch (error) {
+      const duration = Date.now() - startTime;
+      
+      // Single log at the end - error
+      const formattedMessage = formatter.formatFindAllSingleLine(
+        description,
+        0,
+        {
+          duration: duration,
+        },
+      );
+      this.emitter.emit(events.log.narration, formattedMessage, true);
+
       // Track findAll error interaction (fire-and-forget, don't block)
       const sessionId = this.getSessionId();
       if (sessionId && this.sandbox?.send) {
@@ -2371,8 +2434,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
         });
       }
 
-      const { events } = require("./agent/events.js");
-      this.emitter.emit(events.log.log, `Error in findAll: ${error.message}`);
       return [];
     }
   }
@@ -2771,6 +2832,11 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     // handles forwarding to sandbox. This prevents duplicate output to server.
     this.emitter.on("log:**", (message) => {
       const event = this.emitter.event;
+
+      if (event.includes("markdown")) {
+        return;
+      }
+
       if (event === events.log.debug && !debugMode) return;
       if (this.loggingEnabled && message) {
         const prefixedMessage = this.testContext
@@ -2920,39 +2986,11 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     const platform = options.platform || this.config.TD_PLATFORM || "linux";
 
     // Auto-detect sandbox ID from the active sandbox if not provided
-    const sandboxId = options.sandboxId || this.agent?.sandbox?.id || null;
+    // For E2B (Linux), the instance has sandboxId; for AWS (Windows), it has instanceId
+    const sandboxId = options.sandboxId || this.instance?.sandboxId || this.instance?.instanceId || this.agent?.sandboxId || null;
 
     // Get or create session ID using the agent's newSession method
     let sessionId = this.agent?.sessionInstance?.get() || null;
-    
-    // If no session exists, create one using the agent's method
-    if (!sessionId && this.agent?.newSession) {
-      try {
-        await this.agent.newSession();
-        sessionId = this.agent.sessionInstance.get();
-        
-        // Save session ID to file for reuse across test runs
-        if (sessionId) {
-          const sessionFile = path.join(os.homedir(), '.testdriverai-session');
-          fs.writeFileSync(sessionFile, sessionId, { encoding: 'utf-8' });
-        }
-      } catch (error) {
-        // Log but don't fail - tests can run without a session
-        console.warn('Failed to create session:', error.message);
-      }
-    }
-    
-    // If still no session, try reading from file (for reporter/separate processes)
-    if (!sessionId) {
-      try {
-        const sessionFile = path.join(os.homedir(), '.testdriverai-session');
-        if (fs.existsSync(sessionFile)) {
-          sessionId = fs.readFileSync(sessionFile, 'utf-8').trim();
-        }
-      } catch (error) {
-        // Ignore file read errors
-      }
-    }
 
     const data = {
       runId: options.runId,
@@ -3076,26 +3114,98 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
    * This is the SDK equivalent of the CLI's exploratory loop
    *
    * @param {string} task - Natural language description of what to do
-   * @param {Object} options - Execution options
-   * @param {boolean} [options.validateAndLoop=false] - Whether to validate completion and retry if incomplete
-   * @returns {Promise<string|void>} Final AI response if validateAndLoop is true
+   * @param {Object} [options] - Execution options
+   * @param {number} [options.tries=7] - Maximum number of check/retry attempts before giving up
+   * @returns {Promise<ActResult>} Result object with success status and details
+   * @throws {ActError} When the task fails after all tries are exhausted
+   *
+   * @typedef {Object} ActResult
+   * @property {boolean} success - Whether the task completed successfully
+   * @property {string} task - The original task that was executed
+   * @property {number} tries - Number of check attempts made
+   * @property {number} maxTries - Maximum tries that were allowed
+   * @property {number} duration - Total execution time in milliseconds
+   * @property {string} [response] - AI's final response if available
    *
    * @example
    * // Simple execution
-   * await client.act('Click the submit button');
+   * const result = await client.act('Click the submit button');
+   * console.log(result.success); // true
+   *
+   * @example
+   * // With custom retry limit
+   * const result = await client.act('Fill out the contact form', { tries: 10 });
+   * console.log(`Completed in ${result.tries} tries`);
    *
    * @example
-   * // With validation loop
-   * const result = await client.act('Fill out the contact form', { validateAndLoop: true });
-   * console.log(result); // AI's final assessment
+   * // Handle failures
+   * try {
+   *   await client.act('Complete the checkout process', { tries: 3 });
+   * } catch (error) {
+   *   console.log(`Failed after ${error.tries} tries: ${error.message}`);
+   * }
    */
-  async act(task) {
+  async act(task, options = {}) {
     this._ensureConnected();
 
-    this.analytics.track("sdk.act", { task });
+    const { tries = 7 } = options;
+
+    this.analytics.track("sdk.act", { task, tries });
+
+    const { events } = require("./agent/events.js");
+    const startTime = Date.now();
 
-    // Use the agent's exploratoryLoop method directly
-    return await this.agent.exploratoryLoop(task, false, true, false);
+    // Store original checkLimit and set custom one if provided
+    const originalCheckLimit = this.agent.checkLimit;
+    this.agent.checkLimit = tries;
+    
+    // Reset check count for this act() call
+    const originalCheckCount = this.agent.checkCount;
+    this.agent.checkCount = 0;
+
+    // Emit scoped start marker for act()
+    this.emitter.emit(events.log.log, formatter.formatActStart(task));
+
+    try {
+      // Use the agent's exploratoryLoop method directly
+      const response = await this.agent.exploratoryLoop(task, false, true, false);
+      
+      const duration = Date.now() - startTime;
+      const triesUsed = this.agent.checkCount;
+      
+      this.emitter.emit(events.log.log, formatter.formatActComplete(duration, true));
+      
+      // Restore original checkLimit
+      this.agent.checkLimit = originalCheckLimit;
+      this.agent.checkCount = originalCheckCount;
+      
+      return {
+        success: true,
+        task,
+        tries: triesUsed,
+        maxTries: tries,
+        duration,
+        response: response || undefined,
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      const triesUsed = this.agent.checkCount;
+      
+      this.emitter.emit(events.log.log, formatter.formatActComplete(duration, false, error.message));
+      
+      // Restore original checkLimit
+      this.agent.checkLimit = originalCheckLimit;
+      this.agent.checkCount = originalCheckCount;
+      
+      // Create an enhanced error with additional context using ActError class
+      throw new ActError(`Act failed: ${error.message}`, {
+        task,
+        tries: triesUsed,
+        maxTries: tries,
+        duration,
+        cause: error,
+      });
+    }
   }
 
   /**
@@ -3103,15 +3213,16 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
    * Execute a natural language task using AI
    *
    * @param {string} task - Natural language description of what to do
-   * @param {Object} options - Execution options
-   * @param {boolean} [options.validateAndLoop=false] - Whether to validate completion and retry if incomplete
-   * @returns {Promise<string|void>} Final AI response if validateAndLoop is true
+   * @param {Object} [options] - Execution options
+   * @param {number} [options.tries=7] - Maximum number of check/retry attempts
+   * @returns {Promise<ActResult>} Result object with success status and details
    */
-  async ai(task) {
-    return await this.act(task);
+  async ai(task, options) {
+    return await this.act(task, options);
   }
 }
 
 module.exports = TestDriverSDK;
 module.exports.Element = Element;
 module.exports.ElementNotFoundError = ElementNotFoundError;
+module.exports.ActError = ActError;

package/test/testdriver/act.test.mjs

@@ -0,0 +1,30 @@
+/**
+ * TestDriver SDK - Act Test (Vitest)
+ * Tests the AI exploratory loop (act) functionality
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Act Test", () => {
+  it("should use act to search for testdriver on Google", async (context) => {
+    const testdriver = TestDriver(context, { newSandbox: true });
+    
+    // provision.chrome() automatically calls ready() and starts dashcam
+    await testdriver.provision.chrome({
+      url: 'https://www.google.com',
+    });
+
+    // Use act to search for testdriver
+    let actRes = await testdriver.act("click on the empty search box, type 'testdriver', and hit enter. do not click the plus button in the search bar");
+
+    console.log("Act response:", actRes);
+
+    // Assert the search results are displayed
+    const result = await testdriver.assert(
+      "search results for testdriver are visible",
+    );
+
+    expect(result).toBeTruthy();
+  });
+});

package/test/testdriver/assert.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Assert Test", () => {
   it("should assert the testdriver login page shows", async (context) => {
-    const testdriver = TestDriver(context, { newSandbox: true });
+    const testdriver = TestDriver(context, { newSandbox: true, headless: false });
     
     // provision.chrome() automatically calls ready() and starts dashcam
     await testdriver.provision.chrome({

package/test/testdriver/hover-text.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Hover Text Test", () => {
   it("should click Sign In and verify error message", async (context) => {
-    const testdriver = TestDriver(context, { headless: true, newSandbox: true, cacheKey: 'hover-text-test' });
+    const testdriver = TestDriver(context, { headless: false, newSandbox: true, cacheKey: 'hover-text-test' });
     await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
 
     // Click on Sign In button using new find() API

package/test/testdriver/setup/testHelpers.mjs

@@ -6,7 +6,6 @@
 import crypto from "crypto";
 import { config } from "dotenv";
 import fs from "fs";
-import os from "os";
 import path, { dirname } from "path";
 import { fileURLToPath } from "url";
 import TestDriver from "../../../sdk.js";
@@ -51,85 +50,6 @@ console.log(
   process.env.TD_OS || "Not set (will default to linux)",
 );
 
-// Global test results storage
-const testResults = {
-  tests: [],
-  startTime: Date.now(),
-};
-
-/**
- * Store test result with dashcam URL
- * @param {string} testName - Name of the test
- * @param {string} testFile - Test file path
- * @param {string|null} dashcamUrl - Dashcam URL if available
- * @param {Object} sessionInfo - Session information
- */
-export function storeTestResult(
-  testName,
-  testFile,
-  dashcamUrl,
-  sessionInfo = {},
-) {
-
-  // Extract replay object ID from dashcam URL
-  let replayObjectId = null;
-  if (dashcamUrl) {
-    const replayIdMatch = dashcamUrl.match(/\/replay\/([^?]+)/);
-    replayObjectId = replayIdMatch ? replayIdMatch[1] : null;
-    if (replayObjectId) {
-      console.log(`   Replay Object ID: ${replayObjectId}`);
-    }
-  }
-
-  testResults.tests.push({
-    name: testName,
-    file: testFile,
-    dashcamUrl,
-    replayObjectId,
-    sessionId: sessionInfo.sessionId,
-    timestamp: new Date().toISOString(),
-  });
-}
-
-/**
- * Get all test results
- * @returns {Object} All collected test results
- */
-export function getTestResults() {
-  return {
-    ...testResults,
-    endTime: Date.now(),
-    duration: Date.now() - testResults.startTime,
-  };
-}
-
-/**
- * Save test results to a JSON file
- * @param {string} outputPath - Path to save the results
- */
-export function saveTestResults(outputPath = "test-results/sdk-summary.json") {
-  const results = getTestResults();
-  const dir = path.dirname(outputPath);
-
-  // Create directory if it doesn't exist
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
-
-  fs.writeFileSync(outputPath, JSON.stringify(results, null, 2));
-  console.log(`\n📊 Test results saved to: ${outputPath}`);
-
-  // Also print dashcam URLs to console
-  console.log("\n🎥 Dashcam URLs:");
-  results.tests.forEach((test) => {
-    if (test.dashcamUrl) {
-      console.log(`  ${test.name}: ${test.dashcamUrl}`);
-    }
-  });
-
-  return results;
-}
-
 /**
  * Intercept console logs and forward to TestDriver sandbox
  * @param {TestDriver} client - TestDriver client instance
@@ -494,21 +414,9 @@ export async function teardownTest(client, options = {}) {
       console.log("⏭️  Postrun skipped (disabled in options)");
     }
 
-    // Write test result to a file for the reporter to pick up (cross-process communication)
+    // Use Vitest's task.meta for cross-process communication with the reporter
     if (options.task) {
-      const testResultFile = path.join(
-        os.tmpdir(),
-        "testdriver-results",
-        `${options.task.id}.json`,
-      );
-
       try {
-        // Ensure directory exists
-        const dir = path.dirname(testResultFile);
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir, { recursive: true });
-        }
-
         // Get test file path - make it relative to project root
         const absolutePath =
           options.task.file?.filepath || options.task.file?.name || "unknown";
@@ -523,33 +431,15 @@ export async function teardownTest(client, options = {}) {
           testOrder = options.task.suite.tasks.indexOf(options.task);
         }
 
-        // Note: Duration is calculated by Vitest and passed via result.duration
-        // We include it in the test result file so the reporter can use it
-
-        // Get duration from Vitest result
-        const result = options.task.result?.();
-        const duration = result?.duration || 0;
-
-        // Write test result with dashcam URL, platform, and metadata
-        const testResult = {
-          testId: options.task.id,
-          testName: options.task.name,
-          testFile: testFile,
-          testOrder: testOrder,
-          dashcamUrl: dashcamUrl,
-          replayObjectId: dashcamUrl
-            ? dashcamUrl.match(/\/replay\/([^?]+)/)?.[1]
-            : null,
-          platform: client.os, // Include platform from SDK client (source of truth)
-          timestamp: Date.now(),
-          duration: duration, // Include duration from Vitest
-        };
-
-        fs.writeFileSync(testResultFile, JSON.stringify(testResult, null, 2));
-   
+        // Set metadata on task for the reporter to pick up
+        options.task.meta.dashcamUrl = dashcamUrl;
+        options.task.meta.platform = client.os; // Include platform from SDK client (source of truth)
+        options.task.meta.testFile = testFile;
+        options.task.meta.testOrder = testOrder;
+        options.task.meta.sessionId = client.getSessionId?.() || null;
       } catch (error) {
         console.error(
-          `[TestHelpers] ❌ Failed to write test result file:`,
+          `[TestHelpers] ❌ Failed to set test metadata:`,
           error.message,
         );
       }

package/tests/example.test.js

@@ -0,0 +1,33 @@
+import { test, expect } from 'vitest';
+import { TestDriver } from 'testdriverai/vitest/hooks';
+import { login } from './login.js';
+
+test('should login and add item to cart', async (context) => {
+
+  // Create TestDriver instance - automatically connects to sandbox
+  const testdriver = TestDriver(context);
+
+  // Launch chrome and navigate to demo app
+  await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
+
+  // Use the login snippet to handle authentication
+  // This demonstrates how to reuse test logic across multiple tests
+  await login(testdriver);
+
+  // Add item to cart
+  const addToCartButton = await testdriver.find(
+    'add to cart button under TestDriver Hat'
+  );
+  await addToCartButton.click();
+
+  // Open cart
+  const cartButton = await testdriver.find(
+    'cart button in the top right corner'
+  );
+  await cartButton.click();
+
+  // Verify item in cart
+  const result = await testdriver.assert('TestDriver Hat is in the cart');
+  expect(result).toBeTruthy();
+  
+});

package/tests/login.js

@@ -0,0 +1,28 @@
+/**
+ * Login snippet - reusable login function
+ * 
+ * This demonstrates how to create reusable test snippets that can be
+ * imported and used across multiple test files.
+ */
+export async function login(testdriver) {
+
+  // The password is displayed on screen, have TestDriver extract it
+  const password = await testdriver.extract('the password');
+
+  // Find the username field
+  const usernameField = await testdriver.find(
+    'Username, label above the username input field on the login form'
+  );
+  await usernameField.click();
+
+  // Type username
+  await testdriver.type('standard_user');
+
+  // Enter password form earlier 
+  // Marked as secret so it's not logged or stored
+  await testdriver.pressKeys(['tab']);
+  await testdriver.type(password, { secret: true });
+
+  // Submit the form
+  await testdriver.find('submit button on the login form').click();
+}

package/vitest.config.js

@@ -0,0 +1,18 @@
+import { defineConfig } from 'vitest/config';
+import TestDriver from 'testdriverai/vitest';
+import { config } from 'dotenv';
+
+// Load environment variables from .env file
+config();
+
+export default defineConfig({
+  test: {
+    testTimeout: 300000,
+    hookTimeout: 300000,
+    reporters: [
+      'default',
+      TestDriver(),
+    ],
+    setupFiles: ['testdriverai/vitest/setup'],
+  },
+});

package/vitest.config.mjs

@@ -10,8 +10,9 @@ export default defineConfig({
   test: {
     testTimeout: 900000,
     hookTimeout: 900000,
+    disableConsoleIntercept: true,
     reporters: [
-      'verbose',
+      'default',
       TestDriver(),
       ['junit', { outputFile: 'test-report.junit.xml' }]
     ],