testdriverai 7.1.0 → 7.1.2

package/sdk.js

@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+// Auto-load environment variables from .env file if it exists
+require("dotenv").config();
 
 const fs = require("fs");
 const path = require("path");
@@ -1123,8 +1124,8 @@ class TestDriverSDK {
         ? options.redrawThreshold
         : { diffThreshold: options.redrawThreshold };
     } else {
-      // Default: disabled
-      this.redrawOptions = { enabled: false };
+      // Default: enabled (as of v7.2)
+      this.redrawOptions = { enabled: true };
     }
     // Keep redrawThreshold for backwards compatibility in connect()
     this.redrawThreshold = this.redrawOptions;
@@ -1178,7 +1179,7 @@ class TestDriverSDK {
    */
   get dashcam() {
     if (!this._dashcam) {
-      const { Dashcam } = require("./src/core/index.js");
+      const { Dashcam } = require("./lib/core/index.js");
       // Don't pass apiKey - let Dashcam use its default key
       this._dashcam = new Dashcam(this);
     }
@@ -1222,6 +1223,19 @@ class TestDriverSDK {
 
         // If dashcam is available and recording, add web logs for this domain
         if (this._dashcam) {
+    
+            // Create the log file on the remote machine
+            const shell = this.os === "windows" ? "pwsh" : "sh";
+            const logPath = this.os === "windows" 
+            ? "C:\\Users\\testdriver\\Documents\\testdriver.log"
+            : "/tmp/testdriver.log";
+            
+            const createLogCmd = this.os === "windows"
+            ? `New-Item -ItemType File -Path "${logPath}" -Force | Out-Null`
+            : `touch ${logPath}`;
+            
+            await this.exec(shell, createLogCmd, 10000, true);
+          
           console.log('[provision.chrome] Adding web logs to dashcam...');
           try {
             const urlObj = new URL(url);
@@ -1229,6 +1243,9 @@ class TestDriverSDK {
             const pattern = `*${domain}*`;
             await this._dashcam.addWebLog(pattern, 'Web Logs');
             console.log(`[provision.chrome] ✅ Web logs added to dashcam (pattern: ${pattern})`);
+
+            await this._dashcam.addFileLog(logPath, "TestDriver Log");
+
           } catch (error) {
             console.warn('[provision.chrome] ⚠️  Failed to add web logs:', error.message);
           }
@@ -1472,6 +1489,9 @@ class TestDriverSDK {
       this.agent.sandboxOs = connectOptions.os;
     } else if (this.sandboxOs) {
       this.agent.sandboxOs = this.sandboxOs;
+    } else {
+      // Fall back to this.os (which defaults to "linux")
+      this.agent.sandboxOs = this.os;
     }
 
     // Set redrawThreshold on agent's cliArgs.options
@@ -1506,13 +1526,8 @@ class TestDriverSDK {
     this._setupCommandMethods();
 
     this.connected = true;
-    
-    // Expose whether we reconnected to an existing sandbox or created a new one
-    this.isReconnected = this.agent.isReconnected || false;
-    
     this.analytics.track("sdk.connect", {
       sandboxId: this.instance?.instanceId,
-      isReconnected: this.isReconnected,
     });
 
     return this.instance;
@@ -1525,13 +1540,19 @@ class TestDriverSDK {
    * @returns {Promise<void>}
    */
   async disconnect() {
+    // Track disconnect event if we were connected
     if (this.connected && this.instance) {
-      // Track disconnect event
       this.analytics.track("sdk.disconnect");
+    }
 
-      this.connected = false;
-      this.instance = null;
+    // Always close the sandbox WebSocket connection to clean up resources
+    // This ensures we don't leave orphaned connections even if connect() failed
+    if (this.sandbox && typeof this.sandbox.close === 'function') {
+      this.sandbox.close();
     }
+
+    this.connected = false;
+    this.instance = null;
   }
 
   /**
@@ -2197,6 +2218,9 @@ class TestDriverSDK {
    * @private
    */
   _setupLogging() {
+    // Track the last fatal error message to throw on exit
+    let lastFatalError = null;
+
     // Set up markdown logger
     createMarkdownLogger(this.emitter);
 
@@ -2219,6 +2243,11 @@ class TestDriverSDK {
       if (this.loggingEnabled) {
         const event = this.emitter.event;
         console.error(event, ":", data);
+        
+        // Capture fatal errors
+        if (event === events.error.fatal) {
+          lastFatalError = data;
+        }
       }
     });
 
@@ -2245,6 +2274,19 @@ class TestDriverSDK {
       }
     });
 
+    // Handle exit events - throw error with meaningful message instead of calling process.exit
+    // This allows test frameworks like Vitest to properly catch and display the error
+    this.emitter.on(events.exit, (exitCode) => {
+      if (exitCode !== 0) {
+        // Create an error with the fatal error message if available
+        const errorMessage = lastFatalError || 'TestDriver fatal error';
+        const error = new Error(errorMessage);
+        error.name = 'TestDriverFatalError';
+        error.exitCode = exitCode;
+        throw error;
+      }
+    });
+
     // Handle show window events for sandbox visualization
     this.emitter.on("show-window", async (url) => {
       if (this.loggingEnabled) {
@@ -2380,7 +2422,7 @@ class TestDriverSDK {
     );
     await sdk.auth();
 
-    const platform = options.platform || this.config.TD_PLATFORM || "windows";
+    const platform = options.platform || this.config.TD_PLATFORM || "linux";
 
     // Auto-detect sandbox ID from the active sandbox if not provided
     const sandboxId = options.sandboxId || this.agent?.sandbox?.id || null;
@@ -2545,21 +2587,34 @@ class TestDriverSDK {
    *
    * @example
    * // Simple execution
-   * await client.ai('Click the submit button');
+   * await client.act('Click the submit button');
    *
    * @example
    * // With validation loop
-   * const result = await client.ai('Fill out the contact form', { validateAndLoop: true });
+   * const result = await client.act('Fill out the contact form', { validateAndLoop: true });
    * console.log(result); // AI's final assessment
    */
-  async ai(task) {
+  async act(task) {
     this._ensureConnected();
 
-    this.analytics.track("sdk.ai", { task });
+    this.analytics.track("sdk.act", { task });
 
     // Use the agent's exploratoryLoop method directly
     return await this.agent.exploratoryLoop(task, false, true, false);
   }
+
+  /**
+   * @deprecated Use act() instead
+   * Execute a natural language task using AI
+   *
+   * @param {string} task - Natural language description of what to do
+   * @param {Object} options - Execution options
+   * @param {boolean} [options.validateAndLoop=false] - Whether to validate completion and retry if incomplete
+   * @returns {Promise<string|void>} Final AI response if validateAndLoop is true
+   */
+  async ai(task) {
+    return await this.act(task);
+  }
 }
 
 module.exports = TestDriverSDK;

package/{self-hosted.yml → setup/aws/self-hosted.yml}

@@ -78,7 +78,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: us-east-2
           AWS_LAUNCH_TEMPLATE_ID: lt-00d02f31cfc602f27
-          AMI_ID: ami-055cd47506a2f39bb
+          AMI_ID: ami-086b5b4b86d78987c
           RESOLUTION: 1920x1080
       - name: Run TestDriver
         run: node bin/testdriverai.js run testdriver/acceptance/${{ matrix.test }} --ip="${{ steps.aws-setup.outputs.public-ip }}" --junit=out.xml

package/{testdriver/acceptance-sdk → test/manual}/test-console-logs.test.mjs

@@ -4,7 +4,7 @@
  */
 
 import { describe, expect, it } from "vitest";
-import { TestDriver } from "../../src/vitest/hooks.mjs";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Console Log Test", () => {
   it("should capture console logs and send them to dashcam", async (context) => {

package/test/manual/test-find-api.js

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+
+/**
+ * Quick test of the new find() API
+ * This is a simple smoke test to verify the Element class works
+ */
+
+const TestDriver = require("../sdk");
+
+async function testFindAPI() {
+  console.log("Testing new find() API...\n");
+
+  const client = new TestDriver("test-key", {
+    logging: false,
+  });
+
+  // Test 1: Create element without connecting
+  console.log("✓ Test 1: Creating Element instance");
+  const element = client.find("test element");
+  console.log("  Element description:", element.description);
+  console.log("  Element found():", element.found());
+  console.log("  Element coordinates:", element.getCoordinates());
+
+  // Test 2: Verify element methods exist
+  console.log("\n✓ Test 2: Verifying Element methods");
+  console.log("  Has find():", typeof element.find === "function");
+  console.log("  Has click():", typeof element.click === "function");
+  console.log("  Has hover():", typeof element.hover === "function");
+  console.log(
+    "  Has doubleClick():",
+    typeof element.doubleClick === "function",
+  );
+  console.log("  Has rightClick():", typeof element.rightClick === "function");
+  console.log("  Has mouseDown():", typeof element.mouseDown === "function");
+  console.log("  Has mouseUp():", typeof element.mouseUp === "function");
+  console.log("  Has found():", typeof element.found === "function");
+  console.log(
+    "  Has getCoordinates():",
+    typeof element.getCoordinates === "function",
+  );
+
+  // Test 3: Verify error handling for clicking unfound element
+  console.log("\n✓ Test 3: Error handling for unfound element");
+  try {
+    await element.click();
+    console.log("  ❌ Should have thrown error");
+  } catch (error) {
+    console.log("  ✓ Correctly throws error:", error.message);
+  }
+
+  // Test 4: Verify TypeScript types exist (if running from TypeScript)
+  console.log("\n✓ Test 4: SDK methods");
+  console.log("  Has find():", typeof client.find === "function");
+  console.log(
+    "  Has deprecated hoverText():",
+    typeof client.hoverText === "undefined" ? "not yet connected" : "exists",
+  );
+  console.log(
+    "  Has deprecated waitForText():",
+    typeof client.waitForText === "undefined" ? "not yet connected" : "exists",
+  );
+
+  console.log("\n✅ All basic tests passed!");
+  console.log(
+    "\nNote: Full integration tests require connection to TestDriver sandbox.",
+  );
+  console.log("See examples/sdk-find-example.js for complete usage examples.");
+}
+
+testFindAPI().catch((error) => {
+  console.error("Test failed:", error);
+  process.exit(1);
+});

package/test/manual/test-init.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Test script for the init command
+# Usage: ./test-init.sh
+
+set -e
+
+echo "🧪 Testing testdriverai init command..."
+echo ""
+
+# Cleanup function
+cleanup() {
+  echo "🧹 Cleaning up test directories..."
+  rm -rf /tmp/testdriver-test-*
+}
+
+trap cleanup EXIT
+
+# Test 1: Full init with all features
+echo "1️⃣  Testing full initialization (package.json, tests, workflow, npm install)..."
+cd /tmp
+mkdir -p testdriver-test-init
+cd testdriver-test-init
+node /Users/ianjennings/Development/cli/bin/testdriverai.js init
+echo ""
+echo "✅ Files created:"
+find . -type f -not -path "./node_modules/*" | sort
+echo ""
+echo "📄 package.json:"
+cat package.json
+echo ""
+echo "📄 vitest.config.js:"
+cat vitest.config.js
+echo ""
+echo "📄 GitHub workflow:"
+cat .github/workflows/testdriver.yml
+echo ""
+echo "📄 First 15 lines of tests/example.test.js:"
+head -15 tests/example.test.js
+echo ""
+echo "📦 Installed dependencies:"
+npm list --depth=0
+echo ""
+echo "---"
+echo ""
+
+# Test 2: Help command
+echo "2️⃣  Testing help command..."
+node /Users/ianjennings/Development/cli/bin/testdriverai.js init --help
+echo ""
+echo "---"
+echo ""
+
+echo "✅ All tests passed!"

package/test/manual/test-prompt-cache.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+/**
+ * Simple test to verify prompt caching functionality
+ *
+ * This test demonstrates that:
+ * 1. First .prompt() call makes an API request and caches the YAML response
+ * 2. Second .prompt() call with the same prompt uses the cached YAML
+ * 3. Cache can be disabled with TD_NO_PROMPT_CACHE=true
+ */
+
+const TestDriver = require("./sdk.js");
+const promptCache = require("./agent/lib/cache.js");
+
+async function testPromptCache() {
+  console.log("Testing prompt caching functionality...\n");
+
+  const client = new TestDriver(process.env.TD_API_KEY, {
+    os: "linux",
+    logging: true,
+  });
+
+  try {
+    // Connect to sandbox
+    console.log("Connecting to sandbox...");
+    await client.connect();
+    console.log("Connected!\n");
+
+    const testPrompt = "click the search button";
+
+    // Clear cache for this prompt to start fresh
+    const cachePath = promptCache.getCachePath(testPrompt);
+    console.log(`Cache path for "${testPrompt}": ${cachePath}\n`);
+
+    // Test 1: First call (should make API request and cache)
+    console.log("Test 1: First .ai() call (should cache the response)");
+    const stats1 = promptCache.getCacheStats();
+    console.log(`Cache before: ${stats1.count} files`);
+
+    await client.ai(testPrompt);
+
+    const stats2 = promptCache.getCacheStats();
+    console.log(`Cache after: ${stats2.count} files`);
+    console.log(
+      `Cache hit: ${promptCache.hasCache(testPrompt) ? "YES" : "NO"}\n`,
+    );
+
+    // Test 2: Second call (should use cache)
+    console.log(
+      "Test 2: Second .ai() call with same prompt (should use cache)",
+    );
+    console.log('Look for "(using cached response)" message above...\n');
+    await client.ai(testPrompt);
+
+    // Test 3: Third call with cache disabled (should make API call)
+    console.log(
+      "\nTest 3: Third .ai() call with cache=false (should bypass cache)",
+    );
+    await client.ai(testPrompt, false);
+
+    // Test 4: Show cache contents
+    console.log("\nTest 4: Cache contents");
+    const cachedYaml = promptCache.readCache(testPrompt);
+    if (cachedYaml) {
+      console.log("Cached YAML preview (first 500 chars):");
+      console.log(cachedYaml.substring(0, 500));
+      console.log("...\n");
+    }
+
+    // Test 5: Cache statistics
+    console.log("Test 5: Cache statistics");
+    const finalStats = promptCache.getCacheStats();
+    console.log(`Total cached prompts: ${finalStats.count}`);
+    console.log(`Cache files:`, finalStats.files.slice(0, 5));
+
+    console.log("\n✅ Prompt caching test completed!");
+    console.log("\nTo disable caching, pass false: client.ai(prompt, false)");
+    console.log("To clear cache, delete .testdriver/.cache/*.yaml files");
+  } catch (error) {
+    console.error("❌ Test failed:", error.message);
+    throw error;
+  } finally {
+    // Disconnect
+    await client.disconnect();
+  }
+}
+
+// Run the test
+if (require.main === module) {
+  testPromptCache().catch((error) => {
+    console.error("Test error:", error);
+    process.exit(1);
+  });
+}
+
+module.exports = testPromptCache;

package/test/manual/test-provision-auth.mjs

@@ -0,0 +1,22 @@
+/**
+ * Quick test to verify provision() authentication works
+ */
+
+import { test } from 'vitest';
+import { provision } from '../../lib/presets/index.mjs';
+
+test('provision auth test', async (context) => {
+  console.log('Starting provision...');
+  
+  const { testdriver, dashcam } = await provision('chrome', {
+    url: 'http://testdriver-sandbox.vercel.app/',
+  }, context);
+  
+  console.log('✅ Provision complete!');
+  console.log('TestDriver:', testdriver);
+  console.log('Dashcam:', dashcam);
+  
+  // Try a simple find
+  const result = await testdriver.find('any element');
+  console.log('Find result:', result);
+});

package/test/manual/test-sandbox-render.js

@@ -0,0 +1,28 @@
+const TestDriver = require("./sdk.js");
+
+async function test() {
+  console.log("Testing sandbox rendering...");
+
+  const client = new TestDriver(process.env.TD_API_KEY, {
+    os: process.env.TEST_PLATFORM || "linux",
+    headless: false, // Should open browser
+    logging: true,
+  });
+
+  try {
+    console.log("Connecting to sandbox...");
+    const instance = await client.connect();
+    console.log("Connected to instance:", instance);
+
+    // Wait a bit to see if browser opens
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+
+    await client.disconnect();
+    console.log("Test completed successfully");
+  } catch (error) {
+    console.error("Test failed:", error);
+    process.exit(1);
+  }
+}
+
+test();

package/test/manual/test-sdk-methods.js

@@ -0,0 +1,15 @@
+const SDK = require("./sdk.js");
+
+const client = new SDK("test-key");
+
+// Get all public methods (non-private, non-constructor)
+const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(client))
+  .filter((m) => !m.startsWith("_") && m !== "constructor")
+  .sort();
+
+console.log("Public SDK Methods:");
+console.log(methods.join(", "));
+console.log("\nTotal:", methods.length, "methods");
+
+// Check if commands will be set up after connect
+console.log("\nCommands before connect:", client.commands);

package/test/manual/test-sdk-refactor.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+/**
+ * Quick test to verify SDK refactoring works correctly
+ */
+
+const TestDriver = require("./sdk.js");
+
+async function test() {
+  console.log("Testing SDK refactor...\n");
+
+  // Test 1: SDK construction
+  console.log("✓ Test 1: Creating SDK instance");
+  const client = new TestDriver(process.env.TD_API_KEY || "test-key", {
+    logging: false,
+  });
+
+  console.log("  - agent exists:", !!client.agent);
+  console.log("  - emitter exists:", !!client.emitter);
+  console.log("  - config exists:", !!client.config);
+  console.log("  - session exists:", !!client.session);
+  console.log("  - apiClient exists:", !!client.apiClient);
+  console.log("  - analytics exists:", !!client.analytics);
+  console.log("  - sandbox exists:", !!client.sandbox);
+  console.log("  - system exists:", !!client.system);
+
+  // Test 2: Check agent methods are accessible
+  console.log("\n✓ Test 2: Checking agent methods");
+  console.log(
+    "  - agent.exploratoryLoop exists:",
+    typeof client.agent.exploratoryLoop,
+  );
+  console.log("  - agent.buildEnv exists:", typeof client.agent.buildEnv);
+  console.log(
+    "  - agent.getRecentSandboxId exists:",
+    typeof client.agent.getRecentSandboxId,
+  );
+
+  // Test 3: Check SDK methods
+  console.log("\n✓ Test 3: Checking SDK methods");
+  console.log("  - ai() exists:", typeof client.ai);
+  console.log("  - auth() exists:", typeof client.auth);
+  console.log("  - connect() exists:", typeof client.connect);
+  console.log("  - disconnect() exists:", typeof client.disconnect);
+
+  console.log("\n✅ All basic tests passed!");
+}
+
+test().catch((error) => {
+  console.error("\n❌ Test failed:", error.message);
+  console.error(error.stack);
+  process.exit(1);
+});

package/test/manual/test-stack-trace.mjs

@@ -0,0 +1,57 @@
+/**
+ * Quick test to verify stack trace filtering works
+ */
+
+// Mock the MatchError similar to commands.js
+class MatchError extends Error {
+  constructor(message, fatal = false) {
+    super(message);
+    this.fatal = fatal;
+    this.attachScreenshot = true;
+  }
+}
+
+// Simulate SDK wrapper with stack filtering (improved version)
+class TestSDK {
+  constructor() {
+    const command = async (message) => {
+      // Simulate the command throwing an error
+      throw new MatchError(`AI Assertion failed: ${message}`, true);
+    };
+
+    // Wrap the method with proper stack trace handling
+    this.assert = async function (...args) {
+      // Capture the call site for better error reporting
+      const callSite = {};
+      Error.captureStackTrace(callSite, this.assert);
+
+      try {
+        return await command(...args);
+      } catch (error) {
+        // Replace the stack trace to point to the actual caller
+        if (Error.captureStackTrace && callSite.stack) {
+          const errorMessage = error.stack?.split("\n")[0];
+          const callerStack = callSite.stack?.split("\n").slice(1);
+          error.stack = errorMessage + "\n" + callerStack.join("\n");
+        }
+        throw error;
+      }
+    }.bind(this);
+  }
+}
+
+// Test it
+async function runTest() {
+  const client = new TestSDK();
+
+  try {
+    console.log("Testing stack trace...\n");
+    await client.assert("home page appears"); // Line 42 - this should show in stack
+  } catch (error) {
+    console.log("Error caught!");
+    console.log("Stack trace:");
+    console.log(error.stack);
+  }
+}
+
+runTest();

package/test/testdriver/assert.test.mjs

@@ -0,0 +1,41 @@
+/**
+ * TestDriver SDK - Assert Test (Vitest)
+ * Converted from: testdriver/acceptance/assert.yaml
+ */
+
+import { describe, expect, it } from "vitest";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
+
+describe("Assert Test", () => {
+  it("should assert the testdriver login page shows", async (context) => {
+    const testdriver = TestDriver(context, { newSandbox: true });
+    
+    // provision.chrome() automatically calls ready() and starts dashcam
+    await testdriver.provision.chrome({
+      url: 'http://testdriver-sandbox.vercel.app/login',
+    });
+
+    // Assert the TestDriver.ai Sandbox login page is displayed
+    const result = await testdriver.assert(
+      "the TestDriver.ai Sandbox login page is displayed",
+    );
+
+    expect(result).toBeTruthy();
+  });
+  // it("should assert the testdriver login page shows 2", async (context) => {
+  //   const testdriver = TestDriver(context, { newSandbox: true });
+    
+  //   // provision.chrome() automatically calls ready() and starts dashcam
+  //   await testdriver.provision.chrome({
+  //     url: 'http://testdriver-sandbox.vercel.app/login',
+  //   });
+
+  //   // Assert the TestDriver.ai Sandbox login page is displayed
+  //   const result = await testdriver.assert(
+  //     "the TestDriver.ai Sandbox login page is displayed",
+  //   );
+
+  //   expect(result).toBeTruthy();
+  // });
+});
+

package/{testdriver/acceptance-sdk → test/testdriver}/auto-cache-key-demo.test.mjs

@@ -7,7 +7,7 @@
  */
 
 import { describe, expect, it } from "vitest";
-import { TestDriver } from "../../src/vitest/hooks.mjs";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 describe("Auto Cache Key Demo", () => {
   it("should use auto-generated cache key based on file hash", async (context) => {

package/{testdriver/acceptance-sdk → test/testdriver}/drag-and-drop.test.mjs

@@ -4,7 +4,7 @@
  */
 
 import { describe, expect, it } from "vitest";
-import { TestDriver } from "../../src/vitest/hooks.mjs";
+import { TestDriver } from "../../lib/vitest/hooks.mjs";
 
 const isLinux = (process.env.TD_OS || "linux") === "linux";