testdriverai 6.2.2 → 7.0.0

package/agent/lib/validation.js

@@ -62,33 +62,27 @@ const types = () =>
       },
       // - command: scroll # Scroll up or down. Make sure the correct portion of the page is focused before scrolling.
       //   direction: down # Available directions are: up, down, left, right
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
-      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300 for keyboard and 200 for mouse.
+      //   amount: 300 # Optional. The amount of pixels to scroll. Defaults to 300.
       ScrollCommand: {
         command: '"scroll"',
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
         "amount?": "number",
       },
       // - command: scroll-until-text # Scroll until text is found
       //   text: Sign Up # The text to find on screen. The longer and more unique the better.
       //   direction: down # Available directions are: up, down, left, right
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
       ScrollUntilTextCommand: {
         command: '"scroll-until-text"',
         text: "string",
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
       },
       // - command: scroll-until-image # Scroll until icon or image is found
       //   description: Submit at the bottom of the form
       //   direction: down # Available directions are: up, down, left, rights
-      //   method: keyboard # Optional. Available methods are: keyboard (default), mouse. Use mouse only if the prompt explicitly asks for it.
       ScrollUntilImageCommand: {
         command: '"scroll-until-image"',
         description: "string",
         direction: '"up" | "down" | "left" | "right"',
-        "method?": '"keyboard" | "mouse"',
       },
       // - command: wait-for-text # Wait until text is seen on screen. Not recommended unless explicitly requested by user.
       //   text: Copyright 2024 # The text to find on screen.

package/debug-locate-response.js

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+
+/**
+ * Debug script to inspect the full locate API response
+ * Run this with: TD_API_KEY=your_key node debug-locate-response.js
+ */
+
+const TestDriverSDK = require("./sdk.js");
+
+async function debugLocateResponse() {
+  const client = new TestDriverSDK(process.env.TD_API_KEY);
+
+  try {
+    console.log("Connecting to sandbox (Linux)...");
+    await client.connect({ headless: true });
+
+    console.log("Opening a test page...");
+    await client.focusApplication("Google Chrome");
+    await client.type("https://example.com");
+    await client.pressKeys(["enter"]);
+
+    // Wait for page to load
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+
+    console.log("\nFinding an element to inspect the response...");
+    const element = await client.find("the heading that says Example Domain");
+
+    console.log("\n=".repeat(60));
+    console.log("FULL LOCATE API RESPONSE:");
+    console.log("=".repeat(60));
+
+    const response = element.getResponse();
+    console.log(JSON.stringify(response, null, 2));
+
+    console.log("\n=".repeat(60));
+    console.log("RESPONSE KEYS:");
+    console.log("=".repeat(60));
+
+    if (response) {
+      Object.keys(response).forEach((key) => {
+        const value = response[key];
+        const type = Array.isArray(value) ? "array" : typeof value;
+        const preview =
+          typeof value === "string" && value.length > 100
+            ? `${value.substring(0, 100)}... (${value.length} chars)`
+            : typeof value === "object"
+              ? JSON.stringify(value)
+              : value;
+
+        console.log(`  ${key} (${type}): ${preview}`);
+      });
+    }
+
+    console.log("\n=".repeat(60));
+    console.log("ELEMENT PROPERTIES:");
+    console.log("=".repeat(60));
+    console.log("  found:", element.found());
+    console.log("  x:", element.x);
+    console.log("  y:", element.y);
+    console.log("  centerX:", element.centerX);
+    console.log("  centerY:", element.centerY);
+    console.log("  width:", element.width);
+    console.log("  height:", element.height);
+    console.log("  confidence:", element.confidence);
+    console.log("  text:", element.text);
+    console.log("  label:", element.label);
+    console.log(
+      "  screenshot:",
+      element.screenshot ? `${element.screenshot.length} chars` : null,
+    );
+    console.log("  boundingBox:", element.boundingBox);
+
+    await client.disconnect();
+  } catch (error) {
+    console.error("Error:", error.message);
+    console.error(error.stack);
+    await client.disconnect();
+    process.exit(1);
+  }
+}
+
+debugLocateResponse();

package/debugger/index.html

@@ -34,10 +34,14 @@
         top: 0;
         left: 0;
         transform-origin: top left;
+        overflow: hidden;
       }
 
+      html,
       body {
         overflow: hidden;
+        width: 100%;
+        height: 100%;
       }
 
       *::-webkit-scrollbar {
@@ -66,6 +70,7 @@
 
       .overlay {
         position: relative;
+        overflow: hidden;
       }
 
       .screenshot {
@@ -179,6 +184,7 @@
         width: 100%;
         height: 100%;
         z-index: 1;
+        overflow: hidden;
       }
 
       /* Loading screen styles */
@@ -354,6 +360,10 @@
 
       const iframe = document.querySelector("#vm-iframe");
 
+      // Detect if OS is Linux
+      const isLinux = parsedData.os === "linux";
+      const topBarOffset = isLinux ? 14 : 0;
+
       // set overlay width and height to match the given resolution
       const overlayWidth = parsedData.resolution[0];
       const overlayHeight = parsedData.resolution[1];
@@ -361,7 +371,8 @@
       iframe.style.display = "block";
       iframe.src = parsedData.url;
       iframe.style.width = overlayWidth + "px";
-      iframe.style.height = overlayHeight + "px";
+      // Increase iframe height by 14px for Linux to account for top bar
+      iframe.style.height = overlayHeight + topBarOffset + "px";
 
       // Calculate scale factor to fit within window if needed
       const windowWidth = window.innerWidth;
@@ -513,7 +524,7 @@
           const boxElement = document.createElement("div");
           boxElement.className = "bounding-box";
           boxElement.style.left = toCss(box.x);
-          boxElement.style.top = toCss(box.y);
+          boxElement.style.top = toCss(box.y + topBarOffset);
           boxElement.style.width = toCss(box.width);
           boxElement.style.height = toCss(box.height);
           effects.appendChild(boxElement);
@@ -541,14 +552,14 @@
       // Mouse event handlers
       addEventHandler(events.mouseMove, (event, { x, y } = {}) => {
         mouse.style.marginLeft = toCss(x);
-        mouse.style.marginTop = toCss(y);
+        mouse.style.marginTop = toCss(y + topBarOffset);
       });
 
       addEventHandler(
         events.mouseClick,
         (event, { x, y, click = "single" } = {}) => {
           mouse.style.marginLeft = toCss(x);
-          mouse.style.marginTop = toCss(y);
+          mouse.style.marginTop = toCss(y + topBarOffset);
           // Reset class so animation can restart
           mouse.setAttribute("class", "mouse");
           // Force reflow

package/docs/ARCHITECTURE.md

@@ -0,0 +1,424 @@
+# TestDriver Test Recording Architecture
+
+## Overview
+
+This system provides comprehensive test execution tracking, linking test runs with dashcam screen recordings and CI/CD pipelines in the TestDriver dashboard.
+
+## Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Developer's Machine / CI                  │
+├─────────────────────────────────────────────────────────────┤
+│                                                               │
+│  ┌──────────────┐         ┌──────────────┐                  │
+│  │   Vitest     │────────▶│  TD Vitest   │                  │
+│  │  Test Runner │         │   Plugin     │                  │
+│  └──────────────┘         └──────┬───────┘                  │
+│                                   │                           │
+│  ┌──────────────┐                 │                          │
+│  │   Dashcam    │                 │                          │
+│  │  Recording   │                 │                          │
+│  └──────┬───────┘                 │                          │
+│         │                         │                           │
+│         │ (records screen)        │ (reports results)        │
+│         │                         │                           │
+└─────────┼─────────────────────────┼───────────────────────────┘
+          │                         │
+          │                         │
+          ▼                         ▼
+┌─────────────────────────────────────────────────────────────┐
+│                   TestDriver API Server                      │
+├─────────────────────────────────────────────────────────────┤
+│                                                               │
+│  ┌──────────────┐         ┌──────────────┐                  │
+│  │ Replay API   │         │ Test Run API │                  │
+│  │ (dashcam)    │         │ (new)        │                  │
+│  └──────┬───────┘         └──────┬───────┘                  │
+│         │                        │                           │
+│         └────────┬───────────────┘                           │
+│                  │                                            │
+│                  ▼                                            │
+│         ┌────────────────┐                                   │
+│         │   MongoDB      │                                   │
+│         │                │                                   │
+│         │ • TdTestRun    │                                   │
+│         │ • TdTestCase   │                                   │
+│         │ • TdSandbox    │                                   │
+│         │ • Replay       │                                   │
+│         └────────────────┘                                   │
+│                                                               │
+└───────────────────────────┬───────────────────────────────────┘
+                            │
+                            │
+                            ▼
+┌─────────────────────────────────────────────────────────────┐
+│                  TestDriver Web Dashboard                    │
+├─────────────────────────────────────────────────────────────┤
+│                                                               │
+│  ┌─────────────────────────────────────────┐                │
+│  │        Test Runs View (NEW)             │                │
+│  │                                          │                │
+│  │  • List all test runs                   │                │
+│  │  • Filter by status, date, CI           │                │
+│  │  • Show pass/fail statistics            │                │
+│  │  • Link to CI/CD runs                   │                │
+│  └─────────────────────────────────────────┘                │
+│                                                               │
+│  ┌─────────────────────────────────────────┐                │
+│  │      Test Run Detail View (NEW)         │                │
+│  │                                          │                │
+│  │  • Test case list with status           │                │
+│  │  • Dashcam replay player (embedded)     │                │
+│  │  • Error messages and stack traces      │                │
+│  │  • Sandbox details                      │                │
+│  │  • Git commit info                      │                │
+│  └─────────────────────────────────────────┘                │
+│                                                               │
+└───────────────────────────────────────────────────────────────┘
+```
+
+## Data Models
+
+### TdTestRun
+Represents a complete test suite execution (e.g., `npx vitest run`).
+
+**Key Fields:**
+- `runId`: Unique identifier
+- `suiteName`: Name of the test suite
+- `status`: running | passed | failed | cancelled
+- `totalTests`, `passedTests`, `failedTests`: Statistics
+- `platform`: windows | mac | linux
+- CI/CD metadata (provider, runId, url)
+- Git metadata (repo, branch, commit)
+- `dashcamSessionId`: Links to dashcam recordings
+
+**Relationships:**
+- `team`: Owner team
+- `sandbox`: TdSandbox where tests ran
+- `testCases`: Collection of TdTestCase
+- `replays`: Associated Replay records
+
+### TdTestCase
+Represents an individual test within a test run.
+
+**Key Fields:**
+- `testName`: Test name from `test('...')`
+- `testFile`: Path to test file
+- `suiteName`: Describe block name
+- `status`: passed | failed | skipped | pending
+- `duration`: Test duration in ms
+- `errorMessage`, `errorStack`: Failure details
+- `replayUrl`: Direct link to dashcam replay
+- `replayStartTime`, `replayEndTime`: Timestamps within replay
+
+**Relationships:**
+- `testRun`: Parent TdTestRun
+- `replay`: Associated Replay record
+
+### TdSandbox
+Represents a spawned VM/sandbox instance.
+
+**Key Fields:**
+- `sandboxId`: Unique identifier
+- `platform`: windows | mac | linux
+- `status`: provisioning | ready | running | stopped | terminated
+- `instanceId`, `instanceType`: AWS EC2 details
+- `ipAddress`, `vncUrl`, `wsUrl`: Connection details
+- `spawnTime`, `readyTime`, `terminateTime`: Lifecycle timestamps
+- `dashcamAuth`: Whether dashcam was authenticated
+- `dashcamProjectId`: Dashcam project for replays
+
+**Relationships:**
+- `team`: Owner team
+- `user`: User who spawned it
+- `testRuns`: Tests that ran on this sandbox
+- `replays`: Dashcam recordings from this sandbox
+
+**Note:** Sandbox creation/updates happen via WebSocket (not REST API) as part of the sandbox provisioning flow.
+
+### Replay (Extended)
+Existing model extended with test run associations.
+
+**New Fields:**
+- `tdTestRun`: Associated test run
+- `tdTestCase`: Associated test case
+- `tdSandbox`: Sandbox where recorded
+
+## API Endpoints
+
+### POST /api/v1/testdriver/test-run-create
+Create a new test run.
+
+**Auth:** Required (Bearer token)
+
+**Request:**
+```json
+{
+  "runId": "vitest-1234567890-abc123",
+  "suiteName": "Integration Tests",
+  "platform": "windows",
+  "sandboxId": "sandbox-xyz",
+  "ciProvider": "GitHub Actions",
+  "ciRunId": "12345",
+  "repo": "myorg/myrepo",
+  "branch": "main",
+  "commit": "abc123def456"
+}
+```
+
+**Response:**
+```json
+{
+  "data": {
+    "id": "...",
+    "runId": "vitest-1234567890-abc123",
+    "status": "running",
+    "startTime": 1700000000000
+  }
+}
+```
+
+### POST /api/v1/testdriver/test-run-complete
+Mark a test run as complete.
+
+**Auth:** Required
+
+**Request:**
+```json
+{
+  "runId": "vitest-1234567890-abc123",
+  "status": "passed",
+  "totalTests": 25,
+  "passedTests": 24,
+  "failedTests": 1,
+  "skippedTests": 0
+}
+```
+
+### POST /api/v1/testdriver/test-case-create
+Record a test case result (create or update).
+
+**Auth:** Required
+
+**Request:**
+```json
+{
+  "runId": "vitest-1234567890-abc123",
+  "testName": "should login successfully",
+  "testFile": "tests/auth/login.test.js",
+  "suiteName": "Authentication Tests",
+  "status": "passed",
+  "startTime": 1700000001000,
+  "endTime": 1700000002500,
+  "duration": 1500,
+  "replayUrl": "https://app.dashcam.io/replay/abc123"
+}
+```
+
+## Components
+
+### Vitest Plugin (`interfaces/vitest-plugin.mjs`)
+Automatically integrates with Vitest test runs.
+
+**Features:**
+- Auto-detects CI/CD environment (GitHub Actions, GitLab, etc.)
+- Extracts Git metadata from environment or git commands
+- Creates test run at start
+- Records each test case result
+- Associates with dashcam session if `DASHCAM_SESSION_ID` is set
+- Completes test run with statistics
+- Uses plugin architecture for better global state management
+
+**Usage:**
+```javascript
+// vitest.config.mjs
+import testDriverPlugin from './interfaces/vitest-plugin.mjs';
+
+export default {
+  plugins: [
+    testDriverPlugin({
+      apiKey: process.env.TD_API_KEY,
+      apiRoot: process.env.TD_API_ROOT || 'https://testdriver-api.onrender.com',
+    }),
+  ],
+}
+```
+
+### SDK Methods (`sdk.js`)
+
+#### `client.createTestRun(options)`
+Create a test run programmatically.
+
+#### `client.recordTestCase(options)`
+Record a test case result.
+
+#### `client.completeTestRun(options)`
+Mark test run as complete.
+
+## Integration Flows
+
+### Flow 1: Automated with Vitest Reporter
+
+```
+1. Developer runs: npx vitest run
+2. Vitest starts, reporter initializes
+3. Reporter creates TdTestRun
+4. For each test:
+   - Vitest runs test
+   - Reporter records TdTestCase (passed/failed)
+5. All tests complete
+6. Reporter calls completeTestRun()
+7. Results visible in dashboard
+```
+
+### Flow 2: With Dashcam Recording
+
+```
+1. Start dashcam: dashcam start
+2. Set session ID: export DASHCAM_SESSION_ID=$(dashcam session-id)
+3. Run tests: npx vitest run
+4. Reporter creates test run with dashcamSessionId
+5. Tests execute, dashcam records
+6. Stop dashcam: dashcam stop
+7. Publish: dashcam publish -p PROJECT_ID
+8. Replay URL returned
+9. Dashboard shows test results + replay link
+```
+
+### Flow 3: CI/CD Pipeline (GitHub Actions)
+
+```yaml
+jobs:
+  test:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: npm install
+      - name: Start Dashcam
+        run: |
+          dashcam start
+          echo "DASHCAM_SESSION_ID=$(dashcam session-id)" >> $GITHUB_ENV
+      - name: Run Tests
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+        run: npx vitest run
+      - name: Publish Recording
+        if: always()
+        run: |
+          dashcam stop
+          dashcam publish -p ${{ secrets.DASHCAM_PROJECT_ID }}
+```
+
+Reporter auto-detects:
+- GitHub repo, branch, commit
+- Workflow run ID and URL
+- Job ID
+- Actor (who triggered)
+
+## Dashcam Association Strategies
+
+### Strategy 1: Session ID (Implemented)
+- Set `DASHCAM_SESSION_ID` environment variable
+- Reporter includes in test run creation
+- Dashboard queries replays by session ID
+- Shows all replays from that session
+
+### Strategy 2: Explicit URL (Implemented)
+- Dashcam publishes, returns URL
+- Pass URL to `recordTestCase()`
+- Direct 1:1 link between test and replay
+
+### Strategy 3: Timestamp Matching (Future)
+- Parse dashcam logs for replay timestamps
+- Match test start/end times with replay markers
+- Automatically associate without manual linking
+- Allows seeking to exact test within long replay
+
+### Strategy 4: Log Parsing (Future)
+- Dashcam logs test names/files during recording
+- Parse logs to extract test-to-timestamp mapping
+- Generate replay URLs with timestamp seek parameters
+- Example: `https://app.dashcam.io/replay/abc123?t=45000` (seek to 45s)
+
+## Dashboard Views (To Be Built)
+
+### Test Runs List
+- Table of all test runs
+- Columns: Suite Name, Status, Tests (passed/failed), Duration, Date, CI Link
+- Filters: Status, Date range, CI provider, Platform
+- Search: By suite name, repo, branch
+
+### Test Run Detail
+- Header: Suite name, status, duration, platform
+- Statistics card: Total/passed/failed/skipped
+- Test cases table: Name, Status, Duration, Replay link
+- Sidebar: Git info, CI info, Sandbox details
+- Dashcam replay player (embedded iframe)
+- Click test case → seek replay to that test's time range
+
+### Sandbox Management
+- List of active/terminated sandboxes
+- Lifecycle timeline visualization
+- Cost tracking (duration × instance type)
+- Associated test runs
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `TD_API_KEY` | Yes | TestDriver API key for authentication |
+| `DASHCAM_SESSION_ID` | No | Links test run to dashcam session |
+| `TD_SANDBOX_ID` | No | Sandbox ID if running in TestDriver sandbox |
+| `GITHUB_ACTIONS` | Auto | Detected for GitHub Actions integration |
+| `GITLAB_CI` | Auto | Detected for GitLab CI integration |
+| `CIRCLECI` | Auto | Detected for CircleCI integration |
+
+## Future Enhancements
+
+1. **Real-time Test Streaming**
+   - WebSocket connection from reporter
+   - Live test progress in dashboard
+   - See tests pass/fail as they run
+
+2. **Flaky Test Detection**
+   - Track test history across runs
+   - Identify tests that intermittently fail
+   - Suggest fixes based on error patterns
+
+3. **Performance Regression Detection**
+   - Compare test durations across runs
+   - Alert on significant slowdowns
+   - Visualize performance trends
+
+4. **Advanced Dashcam Integration**
+   - Automatic timestamp extraction from logs
+   - AI-powered test failure analysis from replays
+   - Highlight exact moment of failure in replay
+
+5. **Multi-Framework Support**
+   - Jest reporter
+   - Mocha reporter
+   - Playwright reporter
+   - Cypress plugin
+
+6. **Cost Analytics**
+   - Track sandbox costs per test run
+   - Optimize instance types
+   - Budget alerts
+
+## Security Considerations
+
+- API keys stored securely (environment variables)
+- Bearer token authentication for all API calls
+- Team-based access control (tests only visible to team members)
+- Replay access control (dashcam's existing permissions)
+- No sensitive data in test metadata (sanitize error messages)
+
+## Performance Considerations
+
+- Async test case recording (doesn't slow tests)
+- Batch updates for large test suites
+- Efficient database indexing (runId, testFile, status)
+- Replay association is lazy (doesn't block test recording)
+- Optional reporter (disable in local development)