testdriverai 7.3.34 → 7.3.36

package/docs/v7/customizing-devices.mdx

@@ -10,7 +10,64 @@ Configure TestDriver behavior with options passed to the `TestDriver()` function
 
 ```javascript
 const testdriver = TestDriver(context, {
-  reconnect: false,
+  // === Sandbox & Connection ===
+  newSandbox: true,          // Force creation of a new sandbox (default: true)
+  reconnect: false,          // Reconnect to last sandbox (default: false)
+  keepAlive: 60000,          // Keep sandbox alive after disconnect in ms (default: 60000)
+  os: "linux",               // 'linux' | 'windows' (default: 'linux')
+  resolution: "1366x768",    // Sandbox resolution (e.g., '1920x1080')
+  ip: "203.0.113.42",        // Direct IP for self-hosted sandbox
+  sandboxAmi: "ami-1234",    // Custom AMI ID (AWS deployments)
+  sandboxInstance: "i3.metal", // EC2 instance type (AWS deployments)
+
+  // === Preview & Debugging ===
+  preview: "browser",        // "browser" | "ide" | "none" (default: "browser")
+  headless: false,           // @deprecated - use preview: "none" instead
+  debugOnFailure: false,     // Keep sandbox alive on test failure for debugging
+
+  // === Caching ===
+  cache: true,               // Enable element caching (default: true)
+  // Or use advanced caching config:
+  // cache: {
+  //   enabled: true,
+  //   thresholds: {
+  //     find: { screen: 0.05, element: 0.8 },
+  //     assert: 0.05
+  //   }
+  // },
+  cacheKey: "my-test",       // Cache key for element finding operations
+
+  // === Recording & Screenshots ===
+  dashcam: true,             // Enable/disable Dashcam video recording (default: true)
+  autoScreenshots: true,     // Capture screenshots before/after each command (default: true)
+
+  // === AI Configuration ===
+  ai: {                      // Global AI sampling configuration
+    temperature: 0,          // 0 = deterministic, higher = more creative
+    top: {
+      p: 0.9,               // Top-P nucleus sampling (0-1)
+      k: 40,                // Top-K sampling (1 = most likely, 0 = disabled)
+    },
+  },
+
+  // === Screen Change Detection ===
+  redraw: true,              // Enable redraw detection (default: true)
+  // Or use advanced redraw config:
+  // redraw: {
+  //   enabled: true,
+  //   thresholds: {
+  //     screen: 0.05,       // Pixel diff threshold (0-1), false to disable
+  //     network: false,     // Monitor network activity (default: false)
+  //   }
+  // },
+
+  // === Logging & Analytics ===
+  logging: true,             // Enable console logging output (default: true)
+  analytics: true,           // Enable analytics tracking (default: true)
+
+  // === Advanced ===
+  apiRoot: "https://...",    // API endpoint URL (for self-hosted deployments)
+  environment: {},           // Additional environment variables for the sandbox
 });
 ```
 
@@ -54,6 +111,16 @@ const testdriver = TestDriver(context, {
   The legacy `headless: true` option still works for backward compatibility and maps to `preview: "none"`.
 </Note>
 
+### Debug on Failure
+
+Keep the sandbox alive when a test fails so you can reconnect and debug interactively. The sandbox ID is printed to the console along with instructions for reconnecting via MCP.
+
+```javascript
+const testdriver = TestDriver(context, {
+  debugOnFailure: true,
+});
+```
+
 ### IP Target
 
 If self-hosting TestDriver, use `ip` to specify the device IP. See [Self-Hosting TestDriver](../self-hosting.md) for details.
@@ -105,11 +172,102 @@ steps:
   - run: TD_OS=${{ matrix.os }} vitest run
 ```
 
-## Keepalive
+### Dashcam Recording
+
+Dashcam video recording is enabled by default. Disable it to skip recording:
+
+```javascript
+const testdriver = TestDriver(context, {
+  dashcam: false,
+});
+```
+
+### Automatic Screenshots
+
+Screenshots are automatically captured before and after every command (click, type, find, assert, etc.) by default. Each screenshot filename includes the line number from your test file.
+
+Disable automatic screenshots:
+
+```javascript
+const testdriver = TestDriver(context, {
+  autoScreenshots: false,
+});
+```
+
+### Caching
+
+Element caching speeds up repeated `find()` and `assert()` calls. Enabled by default.
+
+```javascript
+// Disable caching
+const testdriver = TestDriver(context, {
+  cache: false,
+});
+
+// Advanced: custom thresholds
+const testdriver = TestDriver(context, {
+  cache: {
+    enabled: true,
+    thresholds: {
+      find: { screen: 0.05, element: 0.8 },
+      assert: 0.05,
+    },
+  },
+  cacheKey: "my-test",
+});
+```
 
-By default, sandboxes terminate immediately when the test finishes. Set this value to keep the sandbox alive for reconnection.
+### Redraw Detection
 
-The `keepAlive` param enables you to keep the sandbox running after the test completes for debugging or reconnection.  This will allow you to use the debugger to inspect the state of the device after the test has finished.
+Redraw detection waits for the screen to stabilize before taking actions. Enabled by default.
+
+```javascript
+// Disable redraw detection
+const testdriver = TestDriver(context, {
+  redraw: false,
+});
+
+// Advanced: custom thresholds with network monitoring
+const testdriver = TestDriver(context, {
+  redraw: {
+    enabled: true,
+    thresholds: {
+      screen: 0.05,
+      network: true,
+    },
+  },
+});
+```
+
+### AI Configuration
+
+Control how the AI model generates responses for `find()` verification and `assert()` calls:
+
+```javascript
+const testdriver = TestDriver(context, {
+  ai: {
+    temperature: 0,       // 0 = deterministic
+    top: { p: 0.9, k: 40 },
+  },
+});
+```
+
+### Environment Variables
+
+Pass additional environment variables to the sandbox:
+
+```javascript
+const testdriver = TestDriver(context, {
+  environment: {
+    MY_VAR: "value",
+    DEBUG: "true",
+  },
+});
+```
+
+## Keepalive
+
+By default, sandboxes stay alive for 60 seconds after disconnect. Customize this with `keepAlive`:
 
 ```javascript
 const testdriver = TestDriver(context, {
@@ -117,6 +275,14 @@ const testdriver = TestDriver(context, {
 });
 ```
 
+Set to `0` to terminate immediately:
+
+```javascript
+const testdriver = TestDriver(context, {
+  keepAlive: 0,  // Terminate sandbox immediately on disconnect
+});
+```
+
 ### Reconnecting to Existing Sandbox
 
 Speed up test development by reconnecting to an existing sandbox instead of starting fresh each time. This lets you iterate quickly on failing steps without re-running the entire test from the beginning.

package/examples/chrome-extension.test.mjs

@@ -65,31 +65,31 @@ describe("Chrome Extension Test", () => {
     expect(popupResult).toBeTruthy();
   });
 
-  it("should load Loom from Chrome Web Store by extensionId", async (context) => {
-    const testdriver = TestDriver(context, { ...getDefaults(context) });
-
-    // Launch Chrome with Loom loaded by its Chrome Web Store ID
-    // Loom ID: liecbddmkiiihnedobmlmillhodjkdmb
-    await testdriver.provision.chromeExtension({
-      extensionId: 'liecbddmkiiihnedobmlmillhodjkdmb'
-    });
-
-    // Navigate to testdriver.ai (extensions don't load on New Tab)
-    const addressBar = await testdriver.find("Chrome address bar");
-    await addressBar.click();
-    await testdriver.type("testdriver.ai");
-    await testdriver.pressKeys(["enter"]);
-
-    // Wait for page to load
-    const pageResult = await testdriver.assert("I can see testdriver.ai");
-    expect(pageResult).toBeTruthy();
-
-    // Click on the extensions button (puzzle piece icon) in Chrome toolbar
-    const extensionsButton = await testdriver.find("The puzzle-shaped icon in the Chrome toolbar.", {zoom: true});
-    await extensionsButton.click();
-
-    // Look for Loom in the extensions menu
-    const loomExtension = await testdriver.find("Loom extension in the extensions dropdown");
-    expect(loomExtension.found()).toBeTruthy();
-  });
+  // it("should load Loom from Chrome Web Store by extensionId", async (context) => {
+  //   const testdriver = TestDriver(context, { ...getDefaults(context) });
+
+  //   // Launch Chrome with Loom loaded by its Chrome Web Store ID
+  //   // Loom ID: liecbddmkiiihnedobmlmillhodjkdmb
+  //   await testdriver.provision.chromeExtension({
+  //     extensionId: 'liecbddmkiiihnedobmlmillhodjkdmb'
+  //   });
+
+  //   // Navigate to testdriver.ai (extensions don't load on New Tab)
+  //   const addressBar = await testdriver.find("Chrome address bar");
+  //   await addressBar.click();
+  //   await testdriver.type("testdriver.ai");
+  //   await testdriver.pressKeys(["enter"]);
+
+  //   // Wait for page to load
+  //   const pageResult = await testdriver.assert("I can see testdriver.ai");
+  //   expect(pageResult).toBeTruthy();
+
+  //   // Click on the extensions button (puzzle piece icon) in Chrome toolbar
+  //   const extensionsButton = await testdriver.find("The puzzle-shaped icon in the Chrome toolbar.", {zoom: true});
+  //   await extensionsButton.click();
+
+  //   // Look for Loom in the extensions menu
+  //   const loomExtension = await testdriver.find("Loom extension in the extensions dropdown");
+  //   expect(loomExtension.found()).toBeTruthy();
+  // });
 });

package/interfaces/vitest-plugin.mjs

@@ -1142,7 +1142,10 @@ class TestDriverReporter {
 
       const suiteName = test.suite?.name;
       const startTime = Date.now() - duration; // Calculate start time from duration
-      const retryCount = result.retryCount || 0;
+      // In Vitest v4, retryCount is on diagnostic(), not result()
+      // result() only returns { state, errors }, while diagnostic() has retryCount, duration, etc.
+      const diagnostic = test.diagnostic?.();
+      const retryCount = diagnostic?.retryCount || 0;
       const testRunDbId = process.env.TD_TEST_RUN_DB_ID;
       const consoleUrl = getConsoleUrl(pluginState.apiRoot);
       const hasRetries = retryCount > 0 && dashcamUrls.length > 1;

package/lib/sentry.js

@@ -16,7 +16,11 @@ const os = require("os");
 const { version } = require("../package.json");
 const logger = require("../agent/lib/logger");
 
-// Store the current session's trace context
+// Store trace contexts per session so concurrent tests don't overwrite each other.
+// Keys are sessionIds, values are { traceId, sessionId }.
+const _traceContexts = new Map();
+
+// For backward compatibility, track the most recently set session
 let currentTraceId = null;
 let currentSessionId = null;
 
@@ -177,7 +181,13 @@ function setSessionTraceContext(sessionId) {
   if (!isEnabled() || !sessionId) return;
 
   // Derive trace ID from session ID (same algorithm as API)
-  currentTraceId = crypto.createHash("md5").update(sessionId).digest("hex");
+  const traceId = crypto.createHash("md5").update(sessionId).digest("hex");
+
+  // Store per-session trace context for concurrent safety
+  _traceContexts.set(sessionId, { traceId, sessionId });
+
+  // Also update the module-level "latest" for backward compatibility
+  currentTraceId = traceId;
   currentSessionId = sessionId;
 
   // Set as global tag so all events include it
@@ -203,9 +213,26 @@ function setSessionTraceContext(sessionId) {
 /**
  * Clear the session trace context
  */
-function clearSessionTraceContext() {
-  currentTraceId = null;
-  currentSessionId = null;
+function clearSessionTraceContext(sessionId) {
+  if (sessionId) {
+    _traceContexts.delete(sessionId);
+    // If the cleared session was the "latest", pick another or null
+    if (currentSessionId === sessionId) {
+      if (_traceContexts.size > 0) {
+        const last = Array.from(_traceContexts.values()).pop();
+        currentTraceId = last.traceId;
+        currentSessionId = last.sessionId;
+      } else {
+        currentTraceId = null;
+        currentSessionId = null;
+      }
+    }
+  } else {
+    // Clear all (backward compatibility)
+    _traceContexts.clear();
+    currentTraceId = null;
+    currentSessionId = null;
+  }
 }
 
 /**

package/lib/vitest/hooks.mjs

@@ -131,7 +131,10 @@ function forwardToAllSandboxes(args) {
  * reporter output).
  */
 function installConsoleSpy() {
-  if (_consoleSpy.installed) return;
+  // Check both installed flag AND that spies are still valid.
+  // Guards against a race where cleanupConsoleSpy restores mocks (setting
+  // installed=false) while a new test is starting up concurrently.
+  if (_consoleSpy.installed && _consoleSpy.spies) return;
   _consoleSpy.installed = true;
 
   // Capture originals once — these are whatever console methods look like
@@ -211,33 +214,6 @@ function cleanupConsoleSpy(client) {
 const testDriverInstances = new WeakMap();
 const lifecycleHandlers = new WeakMap();
 
-// Set to track all active TestDriver instances for signal-based cleanup
-const activeInstances = new Set();
-
-// Register signal handlers once to clean up all active instances on forced exit
-let signalHandlersRegistered = false;
-function registerSignalHandlers() {
-  if (signalHandlersRegistered) return;
-  signalHandlersRegistered = true;
-
-  const cleanup = async () => {
-    const instances = Array.from(activeInstances);
-    activeInstances.clear();
-    await Promise.race([
-      Promise.all(instances.map((inst) => inst.disconnect().catch(() => {}))),
-      new Promise((resolve) => setTimeout(resolve, 5000)), // 5s max for cleanup
-    ]);
-  };
-
-  process.on("SIGINT", () => {
-    cleanup().finally(() => process.exit(130));
-  });
-
-  process.on("SIGTERM", () => {
-    cleanup().finally(() => process.exit(143));
-  });
-}
-
 /**
  * Create a TestDriver client in a Vitest test with automatic lifecycle management
  *
@@ -315,8 +291,6 @@ export function TestDriver(context, options = {}) {
   testdriver.__vitestContext = context.task;
   testdriver._debugOnFailure = mergedOptions.debugOnFailure || false;
   testDriverInstances.set(context.task, testdriver);
-  activeInstances.add(testdriver);
-  registerSignalHandlers();
 
   // Set platform metadata early so the reporter can show the correct OS from the start
   if (!context.task.meta) {
@@ -452,9 +426,6 @@ export function TestDriver(context, options = {}) {
       // Clean up console spies
       cleanupConsoleSpy(currentInstance);
 
-      // Remove from active instances tracking (even in debug mode we clean up tracking)
-      activeInstances.delete(currentInstance);
-
       // DO NOT disconnect or terminate - keep sandbox alive for debugging
       return;
     }
@@ -581,8 +552,6 @@ export function TestDriver(context, options = {}) {
     } catch (error) {
       console.error("Error disconnecting client:", error);
     } finally {
-      // Remove from active instances tracking
-      activeInstances.delete(currentInstance);
       // Terminate AWS instance if one was spawned for this test
       // This must happen AFTER dashcam.stop() to ensure recording is saved
       // AND it must happen even if disconnect() fails

package/lib/vitest/setup-aws.mjs

@@ -115,16 +115,16 @@ function cleanupAllInstances() {
 process.on("exit", cleanupAllInstances);
 process.on("SIGINT", () => {
   cleanupAllInstances();
-  process.exit(130); // Restore default SIGINT exit behavior (128 + signal 2)
+  // Don't call process.exit here - let the signal handler do its job
 });
 process.on("SIGTERM", () => {
   cleanupAllInstances();
-  process.exit(143); // Restore default SIGTERM exit behavior (128 + signal 15)
+  // Don't call process.exit here - let the signal handler do its job
 });
 process.on("uncaughtException", (error) => {
   console.error("[TestDriver] Uncaught exception:", error);
   cleanupAllInstances();
-  process.exit(1); // Exit after uncaught exception cleanup
+  // Don't call process.exit here - let Node.js handle the exception
 });
 
 beforeEach(async (context) => {
@@ -140,6 +140,14 @@ beforeEach(async (context) => {
     return;
   }
 
+  // If ip is provided via plugin options, skip spawning
+  const pluginIp = globalThis.__testdriverPlugin?.state?.testDriverOptions?.ip;
+  if (pluginIp) {
+    console.log(`[TestDriver] Using ip from plugin options: ${pluginIp}`);
+    context.ip = pluginIp;
+    return;
+  }
+
   if (!process.env.AWS_LAUNCH_TEMPLATE_ID || !process.env.AMI_ID) {
     throw new Error(
       "[TestDriver] TD_OS=windows requires AWS_LAUNCH_TEMPLATE_ID and AMI_ID environment variables",

package/lib/vitest/setup-self-hosted.mjs

@@ -81,6 +81,21 @@ beforeEach(async (context) => {
     return;
   }
 
+  // If TD_IP is already set, use it and skip spawning
+  if (process.env.TD_IP) {
+    console.log(`[TestDriver] Using existing instance at ${process.env.TD_IP}`);
+    context.ip = process.env.TD_IP;
+    return;
+  }
+
+  // If ip is provided via plugin options, skip spawning
+  const pluginIp = globalThis.__testdriverPlugin?.state?.testDriverOptions?.ip;
+  if (pluginIp) {
+    console.log(`[TestDriver] Using ip from plugin options: ${pluginIp}`);
+    context.ip = pluginIp;
+    return;
+  }
+
   // Verify AWS credentials are available
   if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_LAUNCH_TEMPLATE_ID || !process.env.AMI_ID) {
     throw new Error('[TestDriver] TD_OS=windows requires AWS credentials (AWS_ACCESS_KEY_ID, AWS_LAUNCH_TEMPLATE_ID, AMI_ID)');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.3.34",
+  "version": "7.3.36",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",
@@ -23,7 +23,6 @@
     },
     "./vitest/setup": "./lib/vitest/setup.mjs",
     "./vitest/setup-aws": "./lib/vitest/setup-aws.mjs",
-    "./vitest/setup-disable-defender": "./lib/vitest/setup-disable-defender.mjs",
     "./vitest/hooks": {
       "types": "./lib/vitest/hooks.d.ts",
       "default": "./lib/vitest/hooks.mjs"