testdriverai 7.1.0 → 7.1.2

package/docs/v7/features/ai-native.mdx

@@ -0,0 +1,427 @@
+---
+title: "AI-Native"
+description: "Built specifically for AI agents and autonomous testing"
+icon: "robot"
+---
+
+TestDriver is designed from the ground up to be used by AI agents, enabling autonomous test creation, execution, and debugging.
+
+## AI Agent Support
+
+Complete guide for AI agents with autonomous operation patterns:
+
+<Card title="agents.md - AI Agent Guide" icon="book" href="/agents.md">
+  Comprehensive documentation specifically for AI agents, including:
+
+  - Quick setup and authentication
+  - Provision methods (chrome, vscode, electron)
+  - Core API methods with examples
+  - Sandbox reconnection patterns
+  - Debugging failed finds
+  - Iterative selector refinement
+  - Best practices for autonomous operation
+</Card>
+
+```javascript
+// AI agents can autonomously write and run tests
+import { test } from 'vitest';
+import { chrome } from 'testdriverai/presets';
+
+test('ai-generated test', async (context) => {
+  const { testdriver } = await chrome(context, {
+    url: 'https://example.com'
+  });
+
+  await testdriver.find('More information link').click();
+  await testdriver.assert('IANA page is visible');
+});
+```
+
+## Natural Language Everything
+
+AI-powered element location, assertions, and instructions:
+
+<Tabs>
+  <Tab title="Element Location">
+    ```javascript
+    // Describe elements naturally
+    await testdriver.find('submit button');
+    await testdriver.find('email input in the login form');
+    await testdriver.find('delete button in the top right corner');
+    await testdriver.find('first product card in the grid');
+    ```
+  </Tab>
+
+  <Tab title="Assertions">
+    ```javascript
+    // Assert application state naturally
+    await testdriver.assert('the login page is displayed');
+    await testdriver.assert('success message says "Welcome back"');
+    await testdriver.assert('the form is empty');
+    await testdriver.assert('the button is blue and enabled');
+    await testdriver.assert('loading spinner is not visible');
+    ```
+  </Tab>
+
+  <Tab title="Instructions">
+    ```javascript
+    // Complex interactions described naturally
+    await testdriver.find('Products menu').hover();
+    await testdriver.find('Laptops submenu item in the Products dropdown').click();
+    await testdriver.find('Sort by dropdown').click();
+    await testdriver.find('Price: Low to High option').click();
+    ```
+  </Tab>
+</Tabs>
+
+## Iterative Selector Refinement
+
+AI agents can iteratively refine selectors when initial attempts fail:
+
+```javascript
+test('iterative debugging', async (context) => {
+  const testdriver = new TestDriver({
+    apiKey: process.env.TD_API_KEY
+  });
+
+  const instance = await testdriver.connect();
+  const sandboxId = instance.instanceId;
+
+  // Try multiple selector variations
+  const selectors = [
+    'submit button',
+    'blue submit button',
+    'submit button in bottom right',
+    'button with "Submit" text',
+    'primary action button'
+  ];
+
+  for (const selector of selectors) {
+    try {
+      console.log(`Trying: ${selector}`);
+      const element = await testdriver.find(selector);
+      console.log(`✓ Found with: ${selector}`);
+      await element.click();
+      break; // Success!
+    } catch (error) {
+      console.log(`✗ Failed with: ${selector}`);
+      console.log(`  Similarity: ${error.similarity}`);
+      console.log(`  Confidence: ${error.confidence}`);
+    }
+  }
+
+  await testdriver.disconnect();
+});
+```
+
+<Tip>
+  AI agents can analyze error similarity scores to refine subsequent selector attempts intelligently.
+</Tip>
+
+## Sandbox Reconnection
+
+AI agents can reconnect to sandboxes for debugging:
+
+```javascript
+// Initial test attempt
+test('initial attempt', async () => {
+  const testdriver = new TestDriver({
+    apiKey: process.env.TD_API_KEY
+  });
+
+  const instance = await testdriver.connect();
+  console.log('Sandbox ID:', instance.instanceId);
+  // Save this ID for reconnection
+
+  try {
+    await testdriver.find('difficult selector').click();
+  } catch (error) {
+    console.error('Failed:', error);
+    // Keep sandbox alive for debugging
+  }
+});
+
+// Reconnect to same sandbox
+test('reconnect attempt', async () => {
+  const testdriver = new TestDriver({
+    apiKey: process.env.TD_API_KEY
+  });
+
+  // Read last sandbox ID
+  const lastSandboxId = testdriver.getLastSandboxId();
+
+  // Reconnect
+  await testdriver.connect({
+    sandboxId: lastSandboxId,
+    newSandbox: false
+  });
+
+  // Try alternative selector
+  await testdriver.find('alternative selector').click();
+
+  await testdriver.disconnect();
+});
+```
+
+<Card title="Automatic Sandbox Tracking" icon="bookmark">
+  TestDriver saves the last sandbox ID to `.testdriver/last-sandbox` automatically. AI agents can read this file to reconnect without manual ID tracking.
+</Card>
+
+## Error Analysis
+
+Rich error information for AI agents to analyze and adapt:
+
+```javascript
+try {
+  await testdriver.find('non-existent button').click();
+} catch (error) {
+  // Rich error information for AI analysis
+  console.log('Error type:', error.name); // 'ElementNotFoundError'
+  console.log('Message:', error.message);
+  console.log('Similarity score:', error.similarity); // 0.72
+  console.log('Confidence:', error.confidence); // 0.81
+  console.log('Debug screenshot:', error.debugScreenshot); // Base64
+  console.log('Cache info:', error.cacheInfo);
+  console.log('Attempted prompt:', error.prompt);
+
+  // AI can analyze and adapt
+  if (error.similarity > 0.7) {
+    // Close match, try more specific selector
+    await testdriver.find('blue submit button in bottom right').click();
+  } else {
+    // No close match, try completely different approach
+    await testdriver.find('primary action button').click();
+  }
+}
+```
+
+## Autonomous Test Generation
+
+AI agents can generate entire test suites:
+
+```javascript
+// AI agent generates tests from requirements
+const requirements = `
+  Test the login flow:
+  1. Navigate to login page
+  2. Enter valid credentials
+  3. Click submit
+  4. Verify dashboard appears
+`;
+
+// Agent generates:
+test('user can login', async (context) => {
+  const { testdriver } = await chrome(context, {
+    url: 'https://myapp.com/login'
+  });
+
+  await testdriver.find('email input').type('user@example.com');
+  await testdriver.find('password input').type('password', { secret: true });
+  await testdriver.find('Login button').click();
+
+  await testdriver.assert('Dashboard is visible');
+});
+```
+
+## MCP Integration (Coming Soon)
+
+Query test infrastructure using natural language:
+
+```javascript
+// Future MCP interactivity
+await testdriver.mcp.query("What is the CPU usage of my app?");
+await testdriver.mcp.query("Show me memory leaks in the last test run");
+await testdriver.mcp.query("What tests are flakiest this week?");
+await testdriver.mcp.query("Compare performance between branches");
+```
+
+<Note>
+  MCP (Model Context Protocol) integration is planned for Q1 2025.
+</Note>
+
+## AI-Powered Debugging
+
+TestDriver provides intelligent debugging assistance:
+
+<AccordionGroup>
+  <Accordion title="Similarity Scoring">
+    ```javascript
+    // AI analyzes how close the match was
+    try {
+      await testdriver.find('submit button');
+    } catch (error) {
+      if (error.similarity > 0.8) {
+        console.log('Very close match - likely wrong selector specificity');
+      } else if (error.similarity > 0.5) {
+        console.log('Partial match - element might be present but different');
+      } else {
+        console.log('No match - element likely not on screen');
+      }
+    }
+    ```
+  </Accordion>
+
+  <Accordion title="Visual Debugging">
+    ```javascript
+    // Get debug screenshot for AI analysis
+    try {
+      await testdriver.find('button');
+    } catch (error) {
+      // AI can analyze the debug screenshot
+      const screenshot = error.debugScreenshot; // Base64 image
+      // Send to vision model for analysis
+      const analysis = await visionModel.analyze(screenshot,
+        "Why can't we find the submit button?"
+      );
+    }
+    ```
+  </Accordion>
+
+  <Accordion title="Cache Diagnostics">
+    ```javascript
+    // Understand cache behavior
+    try {
+      await testdriver.find('element');
+    } catch (error) {
+      console.log('Cache info:', error.cacheInfo);
+      // {
+      //   cacheKey: 'abc123',
+      //   cacheHit: false,
+      //   similarEntries: 3,
+      //   bestSimilarity: 0.87,
+      //   reason: 'threshold_not_met'
+      // }
+    }
+    ```
+  </Accordion>
+</AccordionGroup>
+
+## Programmatic Control
+
+AI agents have full programmatic control:
+
+```javascript
+// Dynamic test generation
+const testCases = [
+  { email: 'user1@example.com', password: 'pass1', shouldSucceed: true },
+  { email: 'invalid', password: 'pass2', shouldSucceed: false },
+  { email: 'user3@example.com', password: 'wrong', shouldSucceed: false },
+];
+
+for (const testCase of testCases) {
+  test(`login with ${testCase.email}`, async (context) => {
+    const { testdriver } = await chrome(context, { url });
+
+    await testdriver.find('email input').type(testCase.email);
+    await testdriver.find('password input').type(testCase.password, { secret: true });
+    await testdriver.find('submit').click();
+
+    if (testCase.shouldSucceed) {
+      await testdriver.assert('dashboard is visible');
+    } else {
+      await testdriver.assert('error message is displayed');
+    }
+  });
+}
+```
+
+## Learning & Adaptation
+
+TestDriver learns from test executions:
+
+<Card title="AI Understanding of Your App" icon="brain">
+  The cache system builds a visual understanding over time:
+
+  - Element location patterns
+  - UI component relationships
+  - Common interaction flows
+  - Stability characteristics
+  - Performance baselines
+
+  This enables:
+  - Faster subsequent executions
+  - Better error messages
+  - Intelligent suggestions
+  - Anomaly detection
+</Card>
+
+## Best Practices for AI Agents
+
+<AccordionGroup>
+  <Accordion title="1. Use Descriptive Selectors">
+    ```javascript
+    // ✅ Good - specific and contextual
+    await testdriver.find('blue submit button below the login form');
+
+    // ❌ Bad - too vague
+    await testdriver.find('button');
+    ```
+  </Accordion>
+
+  <Accordion title="2. Always Use secret: true for Passwords">
+    ```javascript
+    // ✅ Protected from logging
+    await testdriver.type('MyPassword123', { secret: true });
+
+    // ❌ Password will be logged
+    await testdriver.type('MyPassword123');
+    ```
+  </Accordion>
+
+  <Accordion title="3. Handle Errors Gracefully">
+    ```javascript
+    // ✅ Try alternative approaches
+    try {
+      await testdriver.find('optional element').click();
+    } catch (error) {
+      console.log('Element not found, trying alternative...');
+      await testdriver.find('alternative element').click();
+    }
+    ```
+  </Accordion>
+
+  <Accordion title="4. Save Sandbox IDs">
+    ```javascript
+    // ✅ Enable reconnection
+    const instance = await testdriver.connect();
+    console.log('Sandbox ID:', instance.instanceId);
+    // Save for debugging later
+    ```
+  </Accordion>
+</AccordionGroup>
+
+## Learn More
+
+<CardGroup cols={2}>
+  <Card
+    title="AI Agent Guide"
+    icon="robot"
+    href="/agents.md"
+  >
+    Complete guide for AI agents
+  </Card>
+
+  <Card
+    title="API Reference"
+    icon="code"
+    href="/v7/api/client"
+  >
+    Full API documentation
+  </Card>
+
+  <Card
+    title="Error Handling"
+    icon="triangle-exclamation"
+    href="/v7/guides/error-handling"
+  >
+    Handle errors effectively
+  </Card>
+
+  <Card
+    title="Debugging Guide"
+    icon="bug"
+    href="/v7/guides/debugging"
+  >
+    Debug failing tests
+  </Card>
+</CardGroup>