testdriverai 7.3.12 → 7.3.13

package/agent/lib/sdk.js

@@ -5,6 +5,115 @@ const crypto = require("crypto");
 const { version } = require("../../package.json");
 const axios = require("axios");
 
+/**
+ * Default retry configuration
+ */
+const DEFAULT_RETRY_CONFIG = {
+  maxRetries: 10,
+  baseDelayMs: 3000,
+  maxDelayMs: 30000,
+  // Error codes that should trigger a retry
+  retryableNetworkCodes: [
+    'ECONNRESET',
+    'ECONNREFUSED', 
+    'ETIMEDOUT',
+    'ENOTFOUND',
+    'ENETUNREACH',
+    'ERR_NETWORK',
+    'ECONNABORTED',
+    'EPIPE',
+    'EAI_AGAIN',
+  ],
+  // HTTP status codes that should trigger a retry
+  retryableStatusCodes: [429, 500, 502, 503, 504],
+};
+
+/**
+ * Determines if an error is retryable
+ * @param {Error} error - The axios error
+ * @param {Object} config - Retry configuration
+ * @returns {boolean} Whether the request should be retried
+ */
+function isRetryableError(error, config = DEFAULT_RETRY_CONFIG) {
+  // Network-level errors (no response received)
+  if (!error.response) {
+    return config.retryableNetworkCodes.includes(error.code);
+  }
+  
+  // HTTP status code based retries
+  const status = error.response?.status;
+  return config.retryableStatusCodes.includes(status);
+}
+
+/**
+ * Calculate delay for next retry using exponential backoff with jitter
+ * @param {number} attempt - Current attempt number (0-indexed)
+ * @param {Error} error - The error that triggered the retry
+ * @param {Object} config - Retry configuration
+ * @returns {number} Delay in milliseconds
+ */
+function calculateRetryDelay(attempt, error, config = DEFAULT_RETRY_CONFIG) {
+  // Respect Retry-After header for rate limiting
+  if (error.response?.status === 429) {
+    const retryAfter = error.response.headers?.['retry-after'];
+    if (retryAfter) {
+      const retryAfterMs = parseInt(retryAfter, 10) * 1000;
+      if (!isNaN(retryAfterMs) && retryAfterMs > 0) {
+        return Math.min(retryAfterMs, config.maxDelayMs);
+      }
+    }
+  }
+  
+  // Exponential backoff: baseDelay * 2^attempt + random jitter
+  const exponentialDelay = config.baseDelayMs * Math.pow(2, attempt);
+  const jitter = Math.random() * config.baseDelayMs * 0.5;
+  return Math.min(exponentialDelay + jitter, config.maxDelayMs);
+}
+
+/**
+ * Sleep for a specified duration
+ * @param {number} ms - Milliseconds to sleep
+ * @returns {Promise<void>}
+ */
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Execute an async function with retry logic
+ * @param {Function} fn - Async function to execute
+ * @param {Object} options - Options
+ * @param {Object} options.retryConfig - Retry configuration (uses defaults if not provided)
+ * @param {Function} options.onRetry - Callback called before each retry (attempt, error, delayMs)
+ * @returns {Promise<*>} Result of the function
+ */
+async function withRetry(fn, options = {}) {
+  const config = { ...DEFAULT_RETRY_CONFIG, ...options.retryConfig };
+  let lastError;
+  
+  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      
+      // Don't retry if we've exhausted attempts or error isn't retryable
+      if (attempt >= config.maxRetries || !isRetryableError(error, config)) {
+        throw error;
+      }
+      
+      const delayMs = calculateRetryDelay(attempt, error, config);
+      
+      // Call onRetry callback if provided
+      if (options.onRetry) {
+        options.onRetry(attempt + 1, error, delayMs);
+      }
+      
+      await sleep(delayMs);
+    }
+  }
+  
+  throw lastError;
+}
+
 /**
  * Generate Sentry trace headers for distributed tracing
  * Uses the same trace ID derivation as the API (MD5 hash of session ID)
@@ -118,7 +227,19 @@ const createSDK = (emitter, config, sessionInstance) => {
     };
 
     try {
-      let res = await axios(url, c);
+      let res = await withRetry(
+        () => axios(url, c),
+        {
+          onRetry: (attempt, error, delayMs) => {
+            emitter.emit(events.sdk.retry, {
+              path: 'auth/exchange-api-key',
+              attempt,
+              error: error.message || error.code,
+              delayMs,
+            });
+          },
+        }
+      );
 
       token = res.data.token;
       return token;
@@ -243,7 +364,7 @@ const createSDK = (emitter, config, sessionInstance) => {
         ...sentryHeaders, // Add Sentry distributed tracing headers
       },
       responseType: typeof onChunk === "function" ? "stream" : "json",
-      timeout: 60000, // 60 second timeout to prevent hanging requests
+      timeout: 120000, // 120 second timeout to prevent hanging requests
       data: {
         ...data,
         session: sessionInstance.get(),
@@ -254,7 +375,25 @@ const createSDK = (emitter, config, sessionInstance) => {
     try {
       let response;
 
-      response = await axios(url, c);
+      // Use retry logic for non-streaming requests
+      // Streaming requests are not retried as they involve ongoing data transfer
+      if (typeof onChunk !== "function") {
+        response = await withRetry(
+          () => axios(url, c),
+          {
+            onRetry: (attempt, error, delayMs) => {
+              emitter.emit(events.sdk.retry, {
+                path,
+                attempt,
+                error: error.message || error.code,
+                delayMs,
+              });
+            },
+          }
+        );
+      } else {
+        response = await axios(url, c);
+      }
 
       emitter.emit(events.sdk.response, {
         path,

package/agent/lib/system.js

@@ -56,11 +56,6 @@ const createSystem = (emitter, sandbox, config) => {
 
       await screenshot({ filename: step1, format: "png" });
 
-      // Location of cursor image
-      const cursorPath = path.join(__dirname, "resources", "cursor-2.png");
-
-      const mousePos = await getMousePosition();
-
       // Load the screenshot image with Jimp
       let image = await Jimp.read(step1);
       
@@ -76,9 +71,12 @@ const createSystem = (emitter, sandbox, config) => {
       );
 
       if (mouse) {
+        // Only get mouse position when needed to avoid unnecessary websocket calls
+        const cursorPath = path.join(__dirname, "resources", "cursor-2.png");
+        const mousePos = await getMousePosition();
+        
         // Load and composite the mouse cursor image
         const cursorImage = await Jimp.read(cursorPath);
-
         image.composite(cursorImage, mousePos.x, mousePos.y);
       }
 

package/ai/skills/testdriver:ai/SKILL.md

@@ -0,0 +1,204 @@
+---
+name: testdriver:ai
+description: Execute natural language tasks using AI
+---
+<!-- Generated from ai.mdx. DO NOT EDIT. -->
+
+## Overview
+
+The `ai()` method allows you to execute complex tasks using natural language descriptions. TestDriver's AI will figure out the steps needed to accomplish the task.
+
+## Syntax
+
+```javascript
+await testdriver.ai(task, options)
+```
+
+## Parameters
+
+<ParamField path="task" type="string" required>
+  Natural language description of what to do
+</ParamField>
+
+<ParamField path="options" type="object">
+  Execution options
+  
+  <Expandable title="properties">
+    <ParamField path="validateAndLoop" type="boolean" default="false">
+      Whether to validate completion and retry if incomplete
+    </ParamField>
+  </Expandable>
+</ParamField>
+
+## Returns
+
+`Promise<string | void>` - Final AI response if `validateAndLoop` is true
+
+## Examples
+
+### Basic Usage
+
+```javascript
+// Simple task execution
+await testdriver.ai('Click the submit button');
+
+// Complex multi-step task
+await testdriver.ai('Fill out the contact form and submit it');
+
+// Navigation task
+await testdriver.ai('Go to the settings page and enable notifications');
+```
+
+### With Validation
+
+```javascript
+// AI will verify the task completed successfully
+const result = await testdriver.ai('Complete the checkout process', { 
+  validateAndLoop: true 
+});
+
+console.log('Task result:', result);
+```
+
+### Multi-Step Workflows
+
+```javascript
+// The AI will break down complex tasks
+await testdriver.ai('Search for "laptop", add the first result to cart, and proceed to checkout');
+
+// UI exploration
+await testdriver.ai('Find and click all menu items to explore the application');
+```
+
+## Use Cases
+
+<AccordionGroup>
+  <Accordion title="Exploratory Testing">
+    Use AI to explore unfamiliar applications:
+    
+    ```javascript
+    await testdriver.ai('Explore the main navigation menu');
+    await testdriver.ai('Try to find the user profile settings');
+    ```
+  </Accordion>
+  
+  <Accordion title="Complex Workflows">
+    Let AI handle multi-step processes:
+    
+    ```javascript
+    await testdriver.ai('Complete the multi-step registration form');
+    await testdriver.ai('Configure all the advanced settings to default values');
+    ```
+  </Accordion>
+  
+  <Accordion title="Flexible Interactions">
+    When exact element locations aren't critical:
+    
+    ```javascript
+    await testdriver.ai('Close any popup dialogs');
+    await testdriver.ai('Accept the cookie consent if it appears');
+    ```
+  </Accordion>
+</AccordionGroup>
+
+## Best Practices
+
+<Check>
+  **Be specific but flexible**: Give enough context without over-constraining the AI
+  
+  ```javascript
+  // ✅ Good
+  await testdriver.ai('Add the first product to the shopping cart');
+  
+  // ❌ Too vague
+  await testdriver.ai('do something');
+  
+  // ❌ Too specific (defeats the purpose)
+  await testdriver.ai('click at coordinates 500, 300');
+  ```
+</Check>
+
+<Check>
+  **Use for exploration, not precision**: For critical assertions, use explicit methods
+  
+  ```javascript
+  // Use AI for setup
+  await testdriver.ai('Navigate to the login page');
+  
+  // Use explicit methods for critical steps
+  const usernameField = await testdriver.find('username input');
+  await usernameField.click();
+  await testdriver.type('testuser');
+  
+  await testdriver.assert('login page is displayed');
+  ```
+</Check>
+
+<Warning>
+  **AI tasks may be slower**: The AI needs to analyze the screen and plan actions. For performance-critical tests, use explicit methods.
+</Warning>
+
+## When to Use AI vs Explicit Methods
+
+### Use `ai()` when:
+- Exploring unfamiliar applications
+- Handling optional UI elements (popups, cookies, etc.)
+- Prototyping tests quickly
+- Tasks where exact steps may vary
+
+### Use explicit methods when:
+- Performance is critical
+- You need precise control
+- Making assertions
+- Debugging test failures
+- Repetitive, predictable actions
+
+## Complete Example
+
+```javascript
+import { beforeAll, afterAll, describe, it } from 'vitest';
+import TestDriver from 'testdriverai';
+
+describe('E-commerce Flow with AI', () => {
+  let testdriver;
+
+  beforeAll(async () => {
+    client = new TestDriver(process.env.TD_API_KEY);
+    await testdriver.auth();
+    await testdriver.connect();
+  });
+
+  afterAll(async () => {
+    await testdriver.disconnect();
+  });
+
+  it('should complete shopping flow using AI assistance', async () => {
+    await testdriver.focusApplication('Google Chrome');
+    
+    // Use AI for navigation and exploration
+    await testdriver.ai('Browse to the electronics section');
+    await testdriver.ai('Find and add a laptop to the cart');
+    
+    // Use explicit methods for critical steps
+    const cartIcon = await testdriver.find('shopping cart icon');
+    await cartIcon.click();
+    
+    const total = await testdriver.extract('the cart total amount');
+    console.log('Cart total:', total);
+    
+    // Use AI for checkout flow
+    await testdriver.ai('Proceed to checkout and fill in shipping details', {
+      validateAndLoop: true
+    });
+    
+    // Explicit assertion
+    await testdriver.assert('order confirmation page is displayed');
+  });
+});
+```
+
+## Related Methods
+
+- [`find()`](/v7/find) - Locate specific elements
+- [`assert()`](/v7/assert) - Make assertions
+- [`extract()`](/v7/extract) - Extract information

package/ai/skills/testdriver:assert/SKILL.md

@@ -0,0 +1,315 @@
+---
+name: testdriver:assert
+description: Make AI-powered assertions about screen state
+---
+<!-- Generated from assert.mdx. DO NOT EDIT. -->
+
+## Overview
+
+Make AI-powered assertions about the current screen state using natural language. The AI analyzes the screen and verifies that your assertion is true.
+
+## Syntax
+
+```javascript
+await testdriver.assert(assertion)
+await testdriver.assert(assertion, options)
+```
+
+## Parameters
+
+<ParamField path="assertion" type="string" required>
+  Natural language description of what should be true
+</ParamField>
+
+<ParamField path="options" type="object">
+  Optional configuration
+  
+  <Expandable title="properties">
+    <ParamField path="ai" type="object">
+      AI sampling configuration for this assert call (overrides global `ai` config from constructor).
+      
+      <Expandable title="properties">
+        <ParamField path="temperature" type="number">
+          Controls randomness. `0` = deterministic, higher = more creative. Default: model default.
+        </ParamField>
+        
+        <ParamField path="top" type="object">
+          Sampling parameters
+          
+          <Expandable title="properties">
+            <ParamField path="p" type="number">
+              Top-P (nucleus sampling). Range: 0-1.
+            </ParamField>
+            
+            <ParamField path="k" type="number">
+              Top-K sampling. `1` = most deterministic.
+            </ParamField>
+          </Expandable>
+        </ParamField>
+      </Expandable>
+    </ParamField>
+  </Expandable>
+</ParamField>
+
+## Returns
+
+`Promise<boolean>` - `true` if assertion passes, throws error if assertion fails
+
+## Examples
+
+### Basic Assertions
+
+```javascript
+// Verify page elements
+await testdriver.assert('the login page is displayed');
+await testdriver.assert('submit button is visible');
+await testdriver.assert('error message is shown');
+
+// Verify text content
+await testdriver.assert('the page title is "Welcome"');
+await testdriver.assert('username field contains "john.doe"');
+await testdriver.assert('success message says "Account created"');
+
+// Verify states
+await testdriver.assert('the form is empty');
+await testdriver.assert('the checkbox is checked');
+await testdriver.assert('the dropdown shows "United States"');
+
+// Verify visual appearance
+await testdriver.assert('the button is blue');
+await testdriver.assert('the loading spinner is displayed');
+await testdriver.assert('the modal dialog is open');
+```
+
+## Best Practices
+
+<Check>
+  **Be specific in assertions**
+  
+  More specific assertions are more reliable:
+  
+  ```javascript
+  // ❌ Too vague
+  await testdriver.assert('button is visible');
+  
+  // ✅ Specific
+  await testdriver.assert('blue submit button is visible below the form');
+  ```
+</Check>
+
+<Check>
+  **Assert state changes**
+  
+  Verify state before and after actions:
+  
+  ```javascript
+  // Before
+  await testdriver.assert('cart is empty');
+  
+  // Action
+  const addBtn = await testdriver.find('add to cart');
+  await addBtn.click();
+  
+  // After
+  await testdriver.assert('cart contains 1 item');
+  ```
+</Check>
+
+<Check>
+  **Use with test framework assertions**
+  
+  Combine AI assertions with traditional test assertions:
+  
+  ```javascript
+  // AI assertion
+  const result = await testdriver.assert('success message is displayed');
+  
+  // Framework assertion
+  expect(result).toBeTruthy();
+  
+  // Extract for detailed comparison
+  const message = await testdriver.extract('the success message text');
+  expect(message).toContain('successfully');
+  ```
+</Check>
+
+## Polling Assertions
+
+For conditions that may take time to become true:
+
+```javascript
+async function waitForAssertion(testdriver, assertion, timeout = 30000) {
+  const startTime = Date.now();
+  
+  while (Date.now() - startTime < timeout) {
+    try {
+      await testdriver.assert(assertion);
+      return true; // Assertion passed
+    } catch (error) {
+      // Assertion failed, wait and retry
+      await new Promise(r => setTimeout(r, 1000));
+    }
+  }
+  
+  throw new Error(`Assertion timeout: "${assertion}"`);
+}
+
+// Usage
+await waitForAssertion(testdriver, 'page has finished loading', 30000);
+await waitForAssertion(testdriver, 'results are displayed', 10000);
+```
+
+## Use Cases
+
+<AccordionGroup>
+  <Accordion title="Form Validation">
+    ```javascript
+    // Try to submit empty form
+    const submitBtn = await testdriver.find('submit button');
+    await submitBtn.click();
+    
+    // Verify validation errors
+    await testdriver.assert('email field shows "required" error');
+    await testdriver.assert('password field shows "required" error');
+    
+    // Verify form not submitted
+    await testdriver.assert('still on the form page');
+    ```
+  </Accordion>
+  
+  <Accordion title="Page Navigation">
+    ```javascript
+    const loginBtn = await testdriver.find('login button');
+    await loginBtn.click();
+    
+    // Verify navigation
+    await testdriver.assert('user dashboard is displayed');
+    await testdriver.assert('welcome message shows user name');
+    await testdriver.assert('logout button is visible');
+    ```
+  </Accordion>
+  
+  <Accordion title="Dynamic Content">
+    ```javascript
+    const loadBtn = await testdriver.find('load more button');
+    await loadBtn.click();
+    
+    // Poll for content using helper
+    await waitForAssertion(testdriver, 'more than 10 items are shown', 10000);
+    await testdriver.assert('load more button is still visible');
+    ```
+  </Accordion>
+  
+  <Accordion title="Visual States">
+    ```javascript
+    // Verify hover effect
+    const button = await testdriver.find('primary button');
+    await button.hover();
+    
+    await testdriver.assert('button background is darker');
+    
+    // Verify button is enabled
+    await testdriver.assert('submit button is enabled');
+    ```
+  </Accordion>
+  
+  <Accordion title="Multi-Step Workflows">
+    ```javascript
+    // Step 1
+    await testdriver.assert('step 1 is active');
+    const nextBtn = await testdriver.find('next button');
+    await nextBtn.click();
+    
+    // Step 2
+    await testdriver.assert('step 2 is active');
+    await testdriver.assert('step 1 is completed');
+    await nextBtn.click();
+    
+    // Step 3
+    await testdriver.assert('step 3 is active');
+    await testdriver.assert('step 2 is completed');
+    ```
+  </Accordion>
+</AccordionGroup>
+
+## Complete Example
+
+```javascript
+import { beforeAll, afterAll, describe, it, expect } from 'vitest';
+import TestDriver from 'testdriverai';
+
+describe('Assertions', () => {
+  let testdriver;
+
+  beforeAll(async () => {
+    client = new TestDriver(process.env.TD_API_KEY);
+    await testdriver.auth();
+    await testdriver.connect();
+  });
+
+  afterAll(async () => {
+    await testdriver.disconnect();
+  });
+
+  it('should validate login flow', async () => {
+    await testdriver.focusApplication('Google Chrome');
+    
+    // Verify initial state
+    await testdriver.assert('the login page is displayed');
+    await testdriver.assert('username field is empty');
+    await testdriver.assert('password field is empty');
+    
+    // Fill form
+    const usernameField = await testdriver.find('username input');
+    await usernameField.click();
+    await testdriver.type('testuser');
+    
+    await testdriver.pressKeys(['tab']);
+    await testdriver.type('password123');
+    
+    // Verify fields filled
+    await testdriver.assert('username field contains "testuser"');
+    await testdriver.assert('password field is not empty');
+    
+    // Submit
+    await testdriver.pressKeys(['enter']);
+    
+    // Poll for success
+    await waitForAssertion(testdriver, 'user dashboard is displayed', 10000);
+    
+    // Verify logged in state
+    await testdriver.assert('welcome message is shown');
+    await testdriver.assert('logout button is visible');
+    
+    // Verify login page is gone
+    await testdriver.assert('login form is displayed', false, true);
+  });
+
+  it('should show validation errors', async () => {
+    // Try empty submission
+    const submitBtn = await testdriver.find('submit button');
+    await submitBtn.click();
+    
+    // Verify multiple errors
+    const errors = [
+      'email field shows error',
+      'name field shows error',
+      'password field shows error'
+    ];
+    
+    for (const error of errors) {
+      const result = await testdriver.assert(error);
+      expect(result).toBeTruthy();
+    }
+    
+    // Verify not submitted
+    await testdriver.assert('confirmation page is shown', false, true);
+  });
+});
+```
+
+## Related Methods
+
+- [`extract()`](/v7/extract) - Extract information for detailed assertions
+- [`find()`](/v7/find) - Locate elements to verify
+- [`ai()`](/v7/ai) - Complex AI-driven tasks