testdriverai 7.2.2 → 7.2.9

package/sdk.js

@@ -263,6 +263,48 @@ class ElementNotFoundError extends Error {
   }
 }
 
+/**
+ * Custom error class for act() failures
+ * Includes task execution details and retry information
+ */
+class ActError extends Error {
+  /**
+   * @param {string} message - Error message
+   * @param {Object} details - Additional details about the failure
+   * @param {string} details.task - The task that was attempted
+   * @param {number} details.tries - Number of check attempts made
+   * @param {number} details.maxTries - Maximum tries that were allowed
+   * @param {number} details.duration - Total execution time in milliseconds
+   * @param {Error} [details.cause] - The underlying error that caused the failure
+   */
+  constructor(message, details = {}) {
+    super(message);
+    this.name = "ActError";
+    this.task = details.task;
+    this.tries = details.tries;
+    this.maxTries = details.maxTries;
+    this.duration = details.duration;
+    this.cause = details.cause;
+    this.timestamp = new Date().toISOString();
+
+    // Capture stack trace
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, ActError);
+    }
+
+    // Enhance error message with execution details
+    this.message += `\n\n=== Act Execution Details ===`;
+    this.message += `\nTask: "${this.task}"`;
+    this.message += `\nTries: ${this.tries}/${this.maxTries}`;
+    this.message += `\nDuration: ${this.duration}ms`;
+    this.message += `\nTimestamp: ${this.timestamp}`;
+    
+    if (this.cause) {
+      this.message += `\nUnderlying error: ${this.cause.message}`;
+    }
+  }
+}
+
 /**
  * Element class representing a located or to-be-located element
  */
@@ -1702,26 +1744,49 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
        * @returns {Promise<void>}
        */
       vscode: async (options = {}) => {
-        this._ensureConnected();
+        // Automatically wait for connection to be ready
+        await this.ready();
         
         const {
           workspace = null,
           extensions = [],
         } = options;
 
+        const shell = this.os === 'windows' ? 'pwsh' : 'sh';
+
+        // If dashcam is available, set up file logging
+        if (this._dashcam) {
+          // Create the log file on the remote machine
+          const logPath = this.os === "windows" 
+            ? "C:\\Users\\testdriver\\Documents\\testdriver.log"
+            : "/tmp/testdriver.log";
+          
+          const createLogCmd = this.os === "windows"
+            ? `New-Item -ItemType File -Path "${logPath}" -Force | Out-Null`
+            : `touch ${logPath}`;
+          
+          await this.exec(shell, createLogCmd, 10000, true);
+          await this._dashcam.addFileLog(logPath, "TestDriver Log");
+        }
+        
+        // Automatically start dashcam if not already recording
+        if (!this._dashcam || !this._dashcam.recording) {
+          await this.dashcam.start();
+        }
+
         // Install extensions if provided
         for (const extension of extensions) {
-          const shell = this.os === 'windows' ? 'pwsh' : 'sh';
+          console.log(`[provision.vscode] Installing extension: ${extension}`);
           await this.exec(
             shell,
-            `code --install-extension ${extension}`,
-            60000,
+            `code --install-extension ${extension} --force`,
+            120000,
             true
           );
+          console.log(`[provision.vscode] ✅ Extension installed: ${extension}`);
         }
 
         // Launch VS Code
-        const shell = this.os === 'windows' ? 'pwsh' : 'sh';
         const workspaceArg = workspace ? `"${workspace}"` : '';
         
         if (this.os === 'windows') {
@@ -1738,10 +1803,148 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
           );
         }
 
+        // Wait for VS Code to start up
+        await new Promise(resolve => setTimeout(resolve, 3000));
+
         // Wait for VS Code to be ready
         await this.focusApplication('Visual Studio Code');
       },
 
+      /**
+       * Download and install an application
+       * @param {Object} options - Installer options
+       * @param {string} options.url - URL to download the installer from
+       * @param {string} [options.filename] - Filename to save as (auto-detected from URL if not provided)
+       * @param {string} [options.appName] - Application name to focus after install
+       * @param {boolean} [options.launch=true] - Whether to launch the app after installation
+       * @returns {Promise<string>} Path to the downloaded file
+       * @example
+       * // Install a .deb package on Linux (auto-detected)
+       * await testdriver.provision.installer({
+       *   url: 'https://example.com/app.deb',
+       *   appName: 'MyApp'
+       * });
+       * 
+       * @example
+       * // Download and run custom commands
+       * const filePath = await testdriver.provision.installer({
+       *   url: 'https://example.com/app.AppImage',
+       *   launch: false
+       * });
+       * await testdriver.exec('sh', `chmod +x "${filePath}" && "${filePath}" &`, 10000);
+       */
+      installer: async (options = {}) => {
+        // Automatically wait for connection to be ready
+        await this.ready();
+        
+        const {
+          url,
+          filename,
+          appName,
+          launch = true,
+        } = options;
+
+        if (!url) {
+          throw new Error('[provision.installer] url is required');
+        }
+
+        const shell = this.os === 'windows' ? 'pwsh' : 'sh';
+
+        // If dashcam is available, set up file logging
+        if (this._dashcam) {
+          const logPath = this.os === "windows" 
+            ? "C:\\Users\\testdriver\\Documents\\testdriver.log"
+            : "/tmp/testdriver.log";
+          
+          const createLogCmd = this.os === "windows"
+            ? `New-Item -ItemType File -Path "${logPath}" -Force | Out-Null`
+            : `touch ${logPath}`;
+          
+          await this.exec(shell, createLogCmd, 10000, true);
+          await this._dashcam.addFileLog(logPath, "TestDriver Log");
+        }
+        
+        // Automatically start dashcam if not already recording
+        if (!this._dashcam || !this._dashcam.recording) {
+          await this.dashcam.start();
+        }
+
+        // Determine filename from URL if not provided
+        const urlObj = new URL(url);
+        const detectedFilename = filename || urlObj.pathname.split('/').pop() || 'installer';
+        
+        // Determine download directory and full path
+        const downloadDir = this.os === 'windows' 
+          ? 'C:\\Users\\testdriver\\Downloads'
+          : '/tmp';
+        const filePath = this.os === 'windows'
+          ? `${downloadDir}\\${detectedFilename}`
+          : `${downloadDir}/${detectedFilename}`;
+
+        console.log(`[provision.installer] Downloading ${url}...`);
+
+        // Download the file
+        if (this.os === 'windows') {
+          await this.exec(
+            shell,
+            `Invoke-WebRequest -Uri "${url}" -OutFile "${filePath}"`,
+            300000, // 5 min timeout for download
+            true
+          );
+        } else {
+          await this.exec(
+            shell,
+            `curl -L -o "${filePath}" "${url}"`,
+            300000,
+            true
+          );
+        }
+
+        console.log(`[provision.installer] ✅ Downloaded to ${filePath}`);
+
+        // Auto-detect install command based on file extension
+        const ext = detectedFilename.split('.').pop()?.toLowerCase();
+        let installCommand = null;
+        
+        if (this.os === 'windows') {
+          if (ext === 'msi') {
+            installCommand = `Start-Process msiexec -ArgumentList '/i', '"${filePath}"', '/quiet', '/norestart' -Wait`;
+          } else if (ext === 'exe') {
+            installCommand = `Start-Process "${filePath}" -ArgumentList '/S' -Wait`;
+          }
+        } else if (this.os === 'linux') {
+          if (ext === 'deb') {
+            installCommand = `sudo dpkg -i "${filePath}" && sudo apt-get install -f -y`;
+          } else if (ext === 'rpm') {
+            installCommand = `sudo rpm -i "${filePath}"`;
+          } else if (ext === 'appimage') {
+            installCommand = `chmod +x "${filePath}"`;
+          } else if (ext === 'sh') {
+            installCommand = `chmod +x "${filePath}" && "${filePath}"`;
+          }
+        } else if (this.os === 'darwin') {
+          if (ext === 'dmg') {
+            installCommand = `hdiutil attach "${filePath}" -mountpoint /Volumes/installer && cp -R /Volumes/installer/*.app /Applications/ && hdiutil detach /Volumes/installer`;
+          } else if (ext === 'pkg') {
+            installCommand = `sudo installer -pkg "${filePath}" -target /`;
+          }
+        }
+
+        if (installCommand) {
+          console.log(`[provision.installer] Installing...`);
+          await this.exec(shell, installCommand, 300000, true);
+          console.log(`[provision.installer] ✅ Installation complete`);
+        }
+
+        // Launch and focus the app if appName is provided and launch is true
+        if (appName && launch) {
+          await new Promise(resolve => setTimeout(resolve, 2000));
+          await this.focusApplication(appName);
+        }
+
+        return filePath;
+      },
+
       /**
        * Launch Electron app
        * @param {Object} options - Electron launch options
@@ -2027,10 +2230,7 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     const absoluteTimestamp = Date.now();
     const startTime = absoluteTimestamp;
 
-    // Log finding all action
     const { events } = require("./agent/events.js");
-    const findingMessage = formatter.formatElementsFinding(description);
-    this.emitter.emit(events.log.log, findingMessage);
 
     try {
       const screenshot = await this.system.captureScreenBase64();
@@ -2096,16 +2296,16 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
       const duration = Date.now() - startTime;
 
       if (response && response.elements && response.elements.length > 0) {
-        // Log found elements
-        const foundMessage = formatter.formatElementsFound(
+        // Single log at the end - found elements
+        const formattedMessage = formatter.formatFindAllSingleLine(
           description,
           response.elements.length,
           {
-            duration: `${duration}ms`,
+            duration: duration,
             cacheHit: response.cached || false,
           },
         );
-        this.emitter.emit(events.log.log, foundMessage);
+        this.emitter.emit(events.log.narration, formattedMessage, true);
 
         // Create Element instances for each found element
         const elements = response.elements.map((elementData) => {
@@ -2155,7 +2355,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
 
         // Log debug information when elements are found
         if (process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG) {
-          const { events } = require("./agent/events.js");
           this.emitter.emit(
             events.log.debug,
             `✓ Found ${elements.length} element(s): "${description}"`,
@@ -2169,6 +2368,19 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
 
         return elements;
       } else {
+        const duration = Date.now() - startTime;
+        
+        // Single log at the end - no elements found
+        const formattedMessage = formatter.formatFindAllSingleLine(
+          description,
+          0,
+          {
+            duration: duration,
+            cacheHit: response?.cached || false,
+          },
+        );
+        this.emitter.emit(events.log.narration, formattedMessage, true);
+
         // No elements found - track interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
         if (sessionId && this.sandbox?.send) {
@@ -2193,6 +2405,18 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
         return [];
       }
     } catch (error) {
+      const duration = Date.now() - startTime;
+      
+      // Single log at the end - error
+      const formattedMessage = formatter.formatFindAllSingleLine(
+        description,
+        0,
+        {
+          duration: duration,
+        },
+      );
+      this.emitter.emit(events.log.narration, formattedMessage, true);
+
       // Track findAll error interaction (fire-and-forget, don't block)
       const sessionId = this.getSessionId();
       if (sessionId && this.sandbox?.send) {
@@ -2210,8 +2434,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
         });
       }
 
-      const { events } = require("./agent/events.js");
-      this.emitter.emit(events.log.log, `Error in findAll: ${error.message}`);
       return [];
     }
   }
@@ -2606,17 +2828,21 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     createMarkdownLogger(this.emitter);
 
     // Set up basic event logging
+    // Note: We only console.log here - the console spy in vitest/hooks.mjs 
+    // handles forwarding to sandbox. This prevents duplicate output to server.
     this.emitter.on("log:**", (message) => {
       const event = this.emitter.event;
+
+      if (event.includes("markdown")) {
+        return;
+      }
+
       if (event === events.log.debug && !debugMode) return;
       if (this.loggingEnabled && message) {
         const prefixedMessage = this.testContext
           ? `[${this.testContext}] ${message}`
           : message;
         console.log(prefixedMessage);
-        
-        // Also forward to sandbox for dashcam
-        this._forwardLogToSandbox(prefixedMessage);
       }
     });
 
@@ -2675,36 +2901,6 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     });
   }
 
-  /**
-   * Forward log message to sandbox for debugger display
-   * @private
-   * @param {string} message - Log message to forward
-   */
-  _forwardLogToSandbox(message) {
-    try {
-      // Only forward if sandbox is connected
-      if (this.sandbox && this.sandbox.instanceSocketConnected) {
-        // Don't send objects as they cause base64 encoding errors
-        if (typeof message === "object") {
-          return;
-        }
-
-        // Add test context prefix if available
-        const prefixedMessage = this.testContext
-          ? `[${this.testContext}] ${message}`
-          : message;
-
-        this.sandbox.send({
-          type: "output",
-          output: Buffer.from(prefixedMessage).toString("base64"),
-        });
-      }
-    } catch {
-      // Silently fail to avoid breaking the log flow
-      // console.error("Error forwarding log to sandbox:", error);
-    }
-  }
-
   /**
    * Open URL in default browser
    * @private
@@ -2790,39 +2986,11 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
     const platform = options.platform || this.config.TD_PLATFORM || "linux";
 
     // Auto-detect sandbox ID from the active sandbox if not provided
-    const sandboxId = options.sandboxId || this.agent?.sandbox?.id || null;
+    // For E2B (Linux), the instance has sandboxId; for AWS (Windows), it has instanceId
+    const sandboxId = options.sandboxId || this.instance?.sandboxId || this.instance?.instanceId || this.agent?.sandboxId || null;
 
     // Get or create session ID using the agent's newSession method
     let sessionId = this.agent?.sessionInstance?.get() || null;
-    
-    // If no session exists, create one using the agent's method
-    if (!sessionId && this.agent?.newSession) {
-      try {
-        await this.agent.newSession();
-        sessionId = this.agent.sessionInstance.get();
-        
-        // Save session ID to file for reuse across test runs
-        if (sessionId) {
-          const sessionFile = path.join(os.homedir(), '.testdriverai-session');
-          fs.writeFileSync(sessionFile, sessionId, { encoding: 'utf-8' });
-        }
-      } catch (error) {
-        // Log but don't fail - tests can run without a session
-        console.warn('Failed to create session:', error.message);
-      }
-    }
-    
-    // If still no session, try reading from file (for reporter/separate processes)
-    if (!sessionId) {
-      try {
-        const sessionFile = path.join(os.homedir(), '.testdriverai-session');
-        if (fs.existsSync(sessionFile)) {
-          sessionId = fs.readFileSync(sessionFile, 'utf-8').trim();
-        }
-      } catch (error) {
-        // Ignore file read errors
-      }
-    }
 
     const data = {
       runId: options.runId,
@@ -2946,26 +3114,98 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
    * This is the SDK equivalent of the CLI's exploratory loop
    *
    * @param {string} task - Natural language description of what to do
-   * @param {Object} options - Execution options
-   * @param {boolean} [options.validateAndLoop=false] - Whether to validate completion and retry if incomplete
-   * @returns {Promise<string|void>} Final AI response if validateAndLoop is true
+   * @param {Object} [options] - Execution options
+   * @param {number} [options.tries=7] - Maximum number of check/retry attempts before giving up
+   * @returns {Promise<ActResult>} Result object with success status and details
+   * @throws {ActError} When the task fails after all tries are exhausted
+   *
+   * @typedef {Object} ActResult
+   * @property {boolean} success - Whether the task completed successfully
+   * @property {string} task - The original task that was executed
+   * @property {number} tries - Number of check attempts made
+   * @property {number} maxTries - Maximum tries that were allowed
+   * @property {number} duration - Total execution time in milliseconds
+   * @property {string} [response] - AI's final response if available
    *
    * @example
    * // Simple execution
-   * await client.act('Click the submit button');
+   * const result = await client.act('Click the submit button');
+   * console.log(result.success); // true
+   *
+   * @example
+   * // With custom retry limit
+   * const result = await client.act('Fill out the contact form', { tries: 10 });
+   * console.log(`Completed in ${result.tries} tries`);
    *
    * @example
-   * // With validation loop
-   * const result = await client.act('Fill out the contact form', { validateAndLoop: true });
-   * console.log(result); // AI's final assessment
+   * // Handle failures
+   * try {
+   *   await client.act('Complete the checkout process', { tries: 3 });
+   * } catch (error) {
+   *   console.log(`Failed after ${error.tries} tries: ${error.message}`);
+   * }
    */
-  async act(task) {
+  async act(task, options = {}) {
     this._ensureConnected();
 
-    this.analytics.track("sdk.act", { task });
+    const { tries = 7 } = options;
+
+    this.analytics.track("sdk.act", { task, tries });
 
-    // Use the agent's exploratoryLoop method directly
-    return await this.agent.exploratoryLoop(task, false, true, false);
+    const { events } = require("./agent/events.js");
+    const startTime = Date.now();
+
+    // Store original checkLimit and set custom one if provided
+    const originalCheckLimit = this.agent.checkLimit;
+    this.agent.checkLimit = tries;
+    
+    // Reset check count for this act() call
+    const originalCheckCount = this.agent.checkCount;
+    this.agent.checkCount = 0;
+
+    // Emit scoped start marker for act()
+    this.emitter.emit(events.log.log, formatter.formatActStart(task));
+
+    try {
+      // Use the agent's exploratoryLoop method directly
+      const response = await this.agent.exploratoryLoop(task, false, true, false);
+      
+      const duration = Date.now() - startTime;
+      const triesUsed = this.agent.checkCount;
+      
+      this.emitter.emit(events.log.log, formatter.formatActComplete(duration, true));
+      
+      // Restore original checkLimit
+      this.agent.checkLimit = originalCheckLimit;
+      this.agent.checkCount = originalCheckCount;
+      
+      return {
+        success: true,
+        task,
+        tries: triesUsed,
+        maxTries: tries,
+        duration,
+        response: response || undefined,
+      };
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      const triesUsed = this.agent.checkCount;
+      
+      this.emitter.emit(events.log.log, formatter.formatActComplete(duration, false, error.message));
+      
+      // Restore original checkLimit
+      this.agent.checkLimit = originalCheckLimit;
+      this.agent.checkCount = originalCheckCount;
+      
+      // Create an enhanced error with additional context using ActError class
+      throw new ActError(`Act failed: ${error.message}`, {
+        task,
+        tries: triesUsed,
+        maxTries: tries,
+        duration,
+        cause: error,
+      });
+    }
   }
 
   /**
@@ -2973,15 +3213,16 @@ with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
    * Execute a natural language task using AI
    *
    * @param {string} task - Natural language description of what to do
-   * @param {Object} options - Execution options
-   * @param {boolean} [options.validateAndLoop=false] - Whether to validate completion and retry if incomplete
-   * @returns {Promise<string|void>} Final AI response if validateAndLoop is true
+   * @param {Object} [options] - Execution options
+   * @param {number} [options.tries=7] - Maximum number of check/retry attempts
+   * @returns {Promise<ActResult>} Result object with success status and details
    */
-  async ai(task) {
-    return await this.act(task);
+  async ai(task, options) {
+    return await this.act(task, options);
   }
 }
 
 module.exports = TestDriverSDK;
 module.exports.Element = Element;
 module.exports.ElementNotFoundError = ElementNotFoundError;
+module.exports.ActError = ActError;

package/test/testdriver/act.test.mjs

@@ -0,0 +1,30 @@
+/**
+ * TestDriver SDK - Act Test (Vitest)
+ * Tests the AI exploratory loop (act) functionality
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Act Test", () => {
+  it("should use act to search for testdriver on Google", async (context) => {
+    const testdriver = TestDriver(context, { newSandbox: true });
+    
+    // provision.chrome() automatically calls ready() and starts dashcam
+    await testdriver.provision.chrome({
+      url: 'https://www.google.com',
+    });
+
+    // Use act to search for testdriver
+    let actRes = await testdriver.act("click on the empty search box, type 'testdriver', and hit enter. do not click the plus button in the search bar");
+
+    console.log("Act response:", actRes);
+
+    // Assert the search results are displayed
+    const result = await testdriver.assert(
+      "search results for testdriver are visible",
+    );
+
+    expect(result).toBeTruthy();
+  });
+});

package/test/testdriver/assert.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Assert Test", () => {
   it("should assert the testdriver login page shows", async (context) => {
-    const testdriver = TestDriver(context, { newSandbox: true });
+    const testdriver = TestDriver(context, { newSandbox: true, headless: false });
     
     // provision.chrome() automatically calls ready() and starts dashcam
     await testdriver.provision.chrome({

package/test/testdriver/hover-text.test.mjs

@@ -8,7 +8,7 @@ import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Hover Text Test", () => {
   it("should click Sign In and verify error message", async (context) => {
-    const testdriver = TestDriver(context, { headless: true, newSandbox: true, cacheKey: 'hover-text-test' });
+    const testdriver = TestDriver(context, { headless: false, newSandbox: true, cacheKey: 'hover-text-test' });
     await testdriver.provision.chrome({ url: 'http://testdriver-sandbox.vercel.app/login' });
 
     // Click on Sign In button using new find() API

package/test/testdriver/installer.test.mjs

@@ -0,0 +1,47 @@
+/**
+ * TestDriver SDK - Installer Test (Vitest)
+ * Tests the provision.installer() method for downloading and installing apps
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Provision Installer", () => {
+  it(
+    "should download and install a .deb package on Linux",
+    async (context) => {
+      const testdriver = TestDriver(context, { newSandbox: true });
+      
+      // Install bat (a cat clone with syntax highlighting) using provision.installer
+      const filePath = await testdriver.provision.installer({
+        url: 'https://github.com/sharkdp/bat/releases/download/v0.24.0/bat_0.24.0_amd64.deb',
+      });
+
+      // Verify the file was downloaded
+      expect(filePath).toContain('bat');
+
+      // Verify bat was installed by running it
+      await testdriver.exec('sh', 'bat --version', 10000);
+    },
+  );
+
+  it(
+    "should download a shell script and verify it exists",
+    async (context) => {
+      const testdriver = TestDriver(context, { newSandbox: true });
+      
+      // Download a shell script (nvm installer)
+      const filePath = await testdriver.provision.installer({
+        url: 'https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh',
+        launch: false, // Don't auto-run the script
+      });
+
+      // Verify the file was downloaded
+      expect(filePath).toContain('install.sh');
+
+      // Verify the file is executable
+      const result = await testdriver.exec('sh', `ls -la "${filePath}"`, 10000);
+      expect(result).toBeTruthy();
+    },
+  );
+});