testdriverai 6.2.1 → 7.0.0

package/{.github/workflows/self-hosted.yml → self-hosted.yml}

@@ -1,10 +1,19 @@
 name: AWS
-
 on:
   workflow_dispatch:
   push:
+    branches:
+      - main
     paths-ignore:
       - "docs/**"
+  pull_request:
+    branches:
+      - main
+    types:
+      - ready_for_review
+  pull_request_review:
+    types:
+      - submitted
 
 jobs:
   gather:
@@ -54,6 +63,7 @@ jobs:
       - name: Setup AWS Instance
         id: aws-setup
         run: |
+          chmod +x ./setup/aws/spawn-runner.sh
           OUTPUT=$(./setup/aws/spawn-runner.sh | tee /dev/stderr) # Capture and display output
           echo "$OUTPUT"
           PUBLIC_IP=$(echo "$OUTPUT" | grep "PUBLIC_IP=" | cut -d'=' -f2)
@@ -68,9 +78,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: us-east-2
           AWS_LAUNCH_TEMPLATE_ID: lt-00d02f31cfc602f27
-          AMI_ID: ami-085f872ca0cd80fed
-          RESOLUTION_WIDTH: 1920
-          RESOLUTION_HEIGHT: 1080
+          AMI_ID: ami-055cd47506a2f39bb
+          RESOLUTION: 1920x1080
       - name: Run TestDriver
         run: node bin/testdriverai.js run testdriver/acceptance/${{ matrix.test }} --ip="${{ steps.aws-setup.outputs.public-ip }}" --junit=out.xml
         env:

package/setup/aws/cloudformation.yaml

@@ -168,8 +168,15 @@ Resources:
             IpProtocol: tcp,
             FromPort: 8765,
             ToPort: 8765,
-            CidrIp: !Ref AllowedIngressCidr,
-            Description: "pyautogui-cli WebSockets",
+            CidrIp: 35.171.123.200/32,
+            Description: "pyautogui-cli WebSockets - Static IP 1",
+          }
+        - {
+            IpProtocol: tcp,
+            FromPort: 8765,
+            ToPort: 8765,
+            CidrIp: 52.201.199.222/32,
+            Description: "pyautogui-cli WebSockets - Static IP 2",
           }
         - {
             IpProtocol: tcp,

package/test/mcp-example-test.yaml

@@ -0,0 +1,27 @@
+name: MCP Example Test
+description: Test created using TestDriver MCP to navigate example.com
+
+steps:
+  # Open Chrome browser and navigate to example.com
+  - exec:
+      code: Start-Process chrome "https://example.com"
+      language: pwsh
+      timeout: 5000
+
+  # Wait for the page to load
+  - wait-for-text:
+      text: Example Domain
+      timeout: 10000
+
+  # Assert that the Example Domain heading is visible
+  - assert:
+      expect: The text "Example Domain" is visible on the page
+
+  # Click on the "Learn more" link
+  - hover-text:
+      text: Learn more
+      action: click
+
+  # Wait for navigation to complete
+  - wait:
+      timeout: 2000

package/test-find-api.js

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+
+/**
+ * Quick test of the new find() API
+ * This is a simple smoke test to verify the Element class works
+ */
+
+const TestDriver = require("../sdk");
+
+async function testFindAPI() {
+  console.log("Testing new find() API...\n");
+
+  const client = new TestDriver("test-key", {
+    logging: false,
+  });
+
+  // Test 1: Create element without connecting
+  console.log("✓ Test 1: Creating Element instance");
+  const element = client.find("test element");
+  console.log("  Element description:", element.description);
+  console.log("  Element found():", element.found());
+  console.log("  Element coordinates:", element.getCoordinates());
+
+  // Test 2: Verify element methods exist
+  console.log("\n✓ Test 2: Verifying Element methods");
+  console.log("  Has find():", typeof element.find === "function");
+  console.log("  Has click():", typeof element.click === "function");
+  console.log("  Has hover():", typeof element.hover === "function");
+  console.log(
+    "  Has doubleClick():",
+    typeof element.doubleClick === "function",
+  );
+  console.log("  Has rightClick():", typeof element.rightClick === "function");
+  console.log("  Has mouseDown():", typeof element.mouseDown === "function");
+  console.log("  Has mouseUp():", typeof element.mouseUp === "function");
+  console.log("  Has found():", typeof element.found === "function");
+  console.log(
+    "  Has getCoordinates():",
+    typeof element.getCoordinates === "function",
+  );
+
+  // Test 3: Verify error handling for clicking unfound element
+  console.log("\n✓ Test 3: Error handling for unfound element");
+  try {
+    await element.click();
+    console.log("  ❌ Should have thrown error");
+  } catch (error) {
+    console.log("  ✓ Correctly throws error:", error.message);
+  }
+
+  // Test 4: Verify TypeScript types exist (if running from TypeScript)
+  console.log("\n✓ Test 4: SDK methods");
+  console.log("  Has find():", typeof client.find === "function");
+  console.log(
+    "  Has deprecated hoverText():",
+    typeof client.hoverText === "undefined" ? "not yet connected" : "exists",
+  );
+  console.log(
+    "  Has deprecated waitForText():",
+    typeof client.waitForText === "undefined" ? "not yet connected" : "exists",
+  );
+
+  console.log("\n✅ All basic tests passed!");
+  console.log(
+    "\nNote: Full integration tests require connection to TestDriver sandbox.",
+  );
+  console.log("See examples/sdk-find-example.js for complete usage examples.");
+}
+
+testFindAPI().catch((error) => {
+  console.error("Test failed:", error);
+  process.exit(1);
+});

package/test-prompt-cache.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+/**
+ * Simple test to verify prompt caching functionality
+ *
+ * This test demonstrates that:
+ * 1. First .prompt() call makes an API request and caches the YAML response
+ * 2. Second .prompt() call with the same prompt uses the cached YAML
+ * 3. Cache can be disabled with TD_NO_PROMPT_CACHE=true
+ */
+
+const TestDriver = require("./sdk.js");
+const promptCache = require("./agent/lib/cache.js");
+
+async function testPromptCache() {
+  console.log("Testing prompt caching functionality...\n");
+
+  const client = new TestDriver(process.env.TD_API_KEY, {
+    os: "linux",
+    logging: true,
+  });
+
+  try {
+    // Connect to sandbox
+    console.log("Connecting to sandbox...");
+    await client.connect();
+    console.log("Connected!\n");
+
+    const testPrompt = "click the search button";
+
+    // Clear cache for this prompt to start fresh
+    const cachePath = promptCache.getCachePath(testPrompt);
+    console.log(`Cache path for "${testPrompt}": ${cachePath}\n`);
+
+    // Test 1: First call (should make API request and cache)
+    console.log("Test 1: First .ai() call (should cache the response)");
+    const stats1 = promptCache.getCacheStats();
+    console.log(`Cache before: ${stats1.count} files`);
+
+    await client.ai(testPrompt);
+
+    const stats2 = promptCache.getCacheStats();
+    console.log(`Cache after: ${stats2.count} files`);
+    console.log(
+      `Cache hit: ${promptCache.hasCache(testPrompt) ? "YES" : "NO"}\n`,
+    );
+
+    // Test 2: Second call (should use cache)
+    console.log(
+      "Test 2: Second .ai() call with same prompt (should use cache)",
+    );
+    console.log('Look for "(using cached response)" message above...\n');
+    await client.ai(testPrompt);
+
+    // Test 3: Third call with cache disabled (should make API call)
+    console.log(
+      "\nTest 3: Third .ai() call with cache=false (should bypass cache)",
+    );
+    await client.ai(testPrompt, false);
+
+    // Test 4: Show cache contents
+    console.log("\nTest 4: Cache contents");
+    const cachedYaml = promptCache.readCache(testPrompt);
+    if (cachedYaml) {
+      console.log("Cached YAML preview (first 500 chars):");
+      console.log(cachedYaml.substring(0, 500));
+      console.log("...\n");
+    }
+
+    // Test 5: Cache statistics
+    console.log("Test 5: Cache statistics");
+    const finalStats = promptCache.getCacheStats();
+    console.log(`Total cached prompts: ${finalStats.count}`);
+    console.log(`Cache files:`, finalStats.files.slice(0, 5));
+
+    console.log("\n✅ Prompt caching test completed!");
+    console.log("\nTo disable caching, pass false: client.ai(prompt, false)");
+    console.log("To clear cache, delete .testdriver/.cache/*.yaml files");
+  } catch (error) {
+    console.error("❌ Test failed:", error.message);
+    throw error;
+  } finally {
+    // Disconnect
+    await client.disconnect();
+  }
+}
+
+// Run the test
+if (require.main === module) {
+  testPromptCache().catch((error) => {
+    console.error("Test error:", error);
+    process.exit(1);
+  });
+}
+
+module.exports = testPromptCache;

package/test-sandbox-render.js

@@ -0,0 +1,28 @@
+const TestDriver = require("./sdk.js");
+
+async function test() {
+  console.log("Testing sandbox rendering...");
+
+  const client = new TestDriver(process.env.TD_API_KEY, {
+    os: process.env.TEST_PLATFORM || "linux",
+    headless: false, // Should open browser
+    logging: true,
+  });
+
+  try {
+    console.log("Connecting to sandbox...");
+    const instance = await client.connect();
+    console.log("Connected to instance:", instance);
+
+    // Wait a bit to see if browser opens
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+
+    await client.disconnect();
+    console.log("Test completed successfully");
+  } catch (error) {
+    console.error("Test failed:", error);
+    process.exit(1);
+  }
+}
+
+test();

package/test-sdk-methods.js

@@ -0,0 +1,15 @@
+const SDK = require("./sdk.js");
+
+const client = new SDK("test-key");
+
+// Get all public methods (non-private, non-constructor)
+const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(client))
+  .filter((m) => !m.startsWith("_") && m !== "constructor")
+  .sort();
+
+console.log("Public SDK Methods:");
+console.log(methods.join(", "));
+console.log("\nTotal:", methods.length, "methods");
+
+// Check if commands will be set up after connect
+console.log("\nCommands before connect:", client.commands);

package/test-sdk-refactor.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+/**
+ * Quick test to verify SDK refactoring works correctly
+ */
+
+const TestDriver = require("./sdk.js");
+
+async function test() {
+  console.log("Testing SDK refactor...\n");
+
+  // Test 1: SDK construction
+  console.log("✓ Test 1: Creating SDK instance");
+  const client = new TestDriver(process.env.TD_API_KEY || "test-key", {
+    logging: false,
+  });
+
+  console.log("  - agent exists:", !!client.agent);
+  console.log("  - emitter exists:", !!client.emitter);
+  console.log("  - config exists:", !!client.config);
+  console.log("  - session exists:", !!client.session);
+  console.log("  - apiClient exists:", !!client.apiClient);
+  console.log("  - analytics exists:", !!client.analytics);
+  console.log("  - sandbox exists:", !!client.sandbox);
+  console.log("  - system exists:", !!client.system);
+
+  // Test 2: Check agent methods are accessible
+  console.log("\n✓ Test 2: Checking agent methods");
+  console.log(
+    "  - agent.exploratoryLoop exists:",
+    typeof client.agent.exploratoryLoop,
+  );
+  console.log("  - agent.buildEnv exists:", typeof client.agent.buildEnv);
+  console.log(
+    "  - agent.getRecentSandboxId exists:",
+    typeof client.agent.getRecentSandboxId,
+  );
+
+  // Test 3: Check SDK methods
+  console.log("\n✓ Test 3: Checking SDK methods");
+  console.log("  - ai() exists:", typeof client.ai);
+  console.log("  - auth() exists:", typeof client.auth);
+  console.log("  - connect() exists:", typeof client.connect);
+  console.log("  - disconnect() exists:", typeof client.disconnect);
+
+  console.log("\n✅ All basic tests passed!");
+}
+
+test().catch((error) => {
+  console.error("\n❌ Test failed:", error.message);
+  console.error(error.stack);
+  process.exit(1);
+});

package/test-stack-trace.mjs

@@ -0,0 +1,57 @@
+/**
+ * Quick test to verify stack trace filtering works
+ */
+
+// Mock the MatchError similar to commands.js
+class MatchError extends Error {
+  constructor(message, fatal = false) {
+    super(message);
+    this.fatal = fatal;
+    this.attachScreenshot = true;
+  }
+}
+
+// Simulate SDK wrapper with stack filtering (improved version)
+class TestSDK {
+  constructor() {
+    const command = async (message) => {
+      // Simulate the command throwing an error
+      throw new MatchError(`AI Assertion failed: ${message}`, true);
+    };
+
+    // Wrap the method with proper stack trace handling
+    this.assert = async function (...args) {
+      // Capture the call site for better error reporting
+      const callSite = {};
+      Error.captureStackTrace(callSite, this.assert);
+
+      try {
+        return await command(...args);
+      } catch (error) {
+        // Replace the stack trace to point to the actual caller
+        if (Error.captureStackTrace && callSite.stack) {
+          const errorMessage = error.stack?.split("\n")[0];
+          const callerStack = callSite.stack?.split("\n").slice(1);
+          error.stack = errorMessage + "\n" + callerStack.join("\n");
+        }
+        throw error;
+      }
+    }.bind(this);
+  }
+}
+
+// Test it
+async function runTest() {
+  const client = new TestSDK();
+
+  try {
+    console.log("Testing stack trace...\n");
+    await client.assert("home page appears"); // Line 42 - this should show in stack
+  } catch (error) {
+    console.log("Error caught!");
+    console.log("Stack trace:");
+    console.log(error.stack);
+  }
+}
+
+runTest();

package/testdriver/acceptance-sdk/QUICK_REFERENCE.md

@@ -0,0 +1,61 @@
+# SDK Test Reporting - Quick Reference
+
+## 🚀 Commands
+
+```bash
+# Run all tests
+npm run test:sdk
+
+# View terminal summary
+npm run test:sdk:results
+
+# Open HTML report
+npm run test:sdk:report
+
+# Watch mode (dev)
+npm run test:sdk:watch
+
+# Interactive UI
+npm run test:sdk:ui
+```
+
+## 📊 What You Get
+
+### Locally
+
+1. **Console**: Verbose logs with full test output
+2. **Terminal Summary**: `npm run test:sdk:results` - Quick pass/fail counts
+3. **HTML Report**: `npm run test:sdk:report` - Interactive browser viewer
+
+### GitHub Actions
+
+1. **Step Summary**: Markdown tables in workflow summary page
+2. **Test Summary Action**: Badge counts and annotations
+3. **Artifacts**: Download junit.xml, results.json, and index.html
+
+## 📁 Output Files
+
+```
+test-results/
+├── junit.xml       # For CI/CD tools
+├── results.json    # Machine-readable
+└── index.html      # Interactive report
+```
+
+## ⚡ Quick Tips
+
+- **Debugging failures?** → `npm run test:sdk:report` (HTML has best error context)
+- **Quick status check?** → `npm run test:sdk:results` (terminal summary)
+- **PR review?** → Check GitHub Actions summary tab
+- **Need history?** → Download artifacts from GitHub Actions runs
+
+## 🔍 GitHub Summary Preview
+
+Every test run creates a summary with:
+
+- ✅ Pass/fail counts table
+- ❌ Failed test details with errors
+- ✅ List of all passing tests
+- ⏱️ Duration metrics
+
+Find it: Actions → Your workflow run → Summary tab

package/testdriver/acceptance-sdk/README.md

@@ -0,0 +1,128 @@
+# TestDriver SDK Acceptance Tests
+
+This directory contains acceptance tests for the TestDriver SDK using Vitest.
+
+## Running Tests
+
+### Run All Tests (Cross-Platform)
+
+```bash
+npm run test:sdk
+```
+
+### Run Platform-Specific Tests
+
+Use the `TEST_PLATFORM` environment variable to run tests for a specific platform:
+
+```bash
+# Run Windows-only tests
+npm run test:sdk:windows
+
+# Run macOS-only tests
+npm run test:sdk:mac
+
+# Run Linux-only tests
+npm run test:sdk:linux
+```
+
+Or set the environment variable directly:
+
+```bash
+TEST_PLATFORM=windows npm run test:sdk
+TEST_PLATFORM=mac npm run test:sdk
+```
+
+## Test Organization
+
+### Cross-Platform Tests
+
+Tests without a platform suffix run on all platforms:
+
+- `hover-text.test.mjs` - Runs everywhere
+- `scroll.test.mjs` - Runs everywhere
+- `screenshot.test.mjs` - Runs everywhere
+
+### Platform-Specific Tests
+
+Platform-specific tests use naming conventions:
+
+- `*.windows.test.mjs` - Windows-only tests (e.g., `exec-pwsh.windows.test.mjs`)
+- `*.mac.test.mjs` - macOS-only tests
+- `*.linux.test.mjs` - Linux-only tests
+
+### Conditional Test Skipping
+
+Some tests use `skipIf` to conditionally skip based on the platform:
+
+```javascript
+it.skipIf(() => testdriver.os === "linux")(
+  "should run on Windows/Mac",
+  async () => {
+    // This test will be skipped on Linux
+  },
+);
+```
+
+## Environment Variables
+
+- `TEST_PLATFORM` - Filter tests by platform (`windows`, `mac`, `linux`)
+- `TD_OS` - Override the sandbox OS (defaults to `linux`)
+- `TD_API_KEY` - Your TestDriver API key (required)
+- `TD_API_ROOT` - API endpoint (optional)
+- `DEBUG_ENV` - Show environment variable loading (optional)
+- `DEBUG_EVENTS` - Enable detailed event logging (optional)
+
+## Examples
+
+```bash
+# Run only Windows tests on a Windows sandbox
+TEST_PLATFORM=windows npm run test:sdk
+
+# Run all tests but use a Windows sandbox
+TD_OS=windows npm run test:sdk
+
+# Run with debugging enabled
+DEBUG_ENV=true DEBUG_EVENTS=true npm run test:sdk
+
+# Watch mode for development
+npm run test:sdk:watch
+
+# Generate coverage report
+npm run test:sdk:coverage
+```
+
+## Test Structure
+
+Each test follows this pattern:
+
+```javascript
+import { afterEach, beforeEach, describe, it } from "vitest";
+import {
+  createTestClient,
+  setupTest,
+  teardownTest,
+} from "./setup/testHelpers.mjs";
+
+describe("My Test", () => {
+  let testdriver;
+
+  beforeEach(async () => {
+    testdriver = createTestClient();
+    await setupTest(testdriver);
+  });
+
+  afterEach(async () => {
+    await teardownTest(testdriver);
+  });
+
+  it("should do something", async () => {
+    // Your test logic
+  });
+});
+```
+
+## See Also
+
+- [SDK README](../../SDK_README.md) - Full SDK documentation
+- [Quick Reference](./QUICK_REFERENCE.md) - SDK method quick reference
+- [Test Reporting](./TEST_REPORTING.md) - Test recording and reporting docs