testdriverai 7.3.8 → 7.3.10

package/examples/z_flake-shared.mjs

@@ -9,6 +9,7 @@ export function popupLoadingTest(label, options = {}) {
   describe(`Popup with Loading (${label})`, () => {
     it("should accept cookies and wait for completion", async (context) => {
       const testdriver = TestDriver(context, {
+        preview: 'ide',
         ip: context.ip || process.env.TD_IP,
         ...options,
       });

package/interfaces/vitest-plugin.d.ts

@@ -5,11 +5,20 @@
 
 import TestDriverSDK, { TestDriverOptions } from '../sdk';
 
+/**
+ * A single dashcam URL entry for one retry attempt
+ */
+export interface DashcamUrlEntry {
+  url: string | null;
+  platform: string;
+  attempt: number;
+}
+
 /**
  * Plugin state object
  */
 export interface PluginState {
-  dashcamUrls: Map<string, { url: string; platform: string }>;
+  dashcamUrls: Map<string, DashcamUrlEntry[]>;
   suiteTestRuns: Map<string, any>;
   testDriverOptions: TestDriverOptions;
 }
@@ -20,14 +29,19 @@ export interface PluginState {
 export const pluginState: PluginState;
 
 /**
- * Register a Dashcam URL for a test
+ * Register a Dashcam URL for a test attempt
+ */
+export function registerDashcamUrl(testId: string, url: string, platform: string, attempt?: number): void;
+
+/**
+ * Get the latest Dashcam URL entry for a test (backward compatible)
  */
-export function registerDashcamUrl(testId: string, url: string, platform: string): void;
+export function getDashcamUrl(testId: string): DashcamUrlEntry | undefined;
 
 /**
- * Get Dashcam URL for a test
+ * Get all Dashcam URL entries for a test (all retry attempts)
  */
-export function getDashcamUrl(testId: string): { url: string; platform: string } | undefined;
+export function getAllDashcamUrls(testId: string): DashcamUrlEntry[];
 
 /**
  * Clear all Dashcam URLs

package/interfaces/vitest-plugin.mjs

@@ -9,6 +9,29 @@ import { setTestRunInfo } from "./shared-test-state.mjs";
 // Use createRequire to import CommonJS modules without esbuild processing
 const require = createRequire(import.meta.url);
 
+/**
+ * Resolve the TestDriver SDK version using multiple strategies.
+ * Similar to resolveVitestVersion(), guards against import.meta.url rewriting.
+ * @returns {string|null}
+ */
+function resolveTestDriverVersion() {
+  try {
+    return require("../package.json").version;
+  } catch {}
+
+  try {
+    const cwdRequire = createRequire(path.join(process.cwd(), "package.json"));
+    return cwdRequire("testdriverai/package.json").version;
+  } catch {}
+
+  try {
+    const pkgPath = path.join(process.cwd(), "node_modules", "testdriverai", "package.json");
+    return JSON.parse(fs.readFileSync(pkgPath, "utf8")).version;
+  } catch {}
+
+  return null;
+}
+
 /**
  * Minimum required Vitest major version
  */
@@ -170,21 +193,32 @@ export const pluginState = {
   // TestDriver options to pass to all instances
   testDriverOptions: {},
   // Dashcam URL tracking (in-memory, no files needed!)
-  dashcamUrls: new Map(), // testId -> dashcamUrl
+  dashcamUrls: new Map(), // testId -> [{url, platform, attempt}]
   lastDashcamUrl: null, // Fallback for when test ID isn't available
   // Suite-level test run tracking
   suiteTestRuns: new Map(), // suiteId -> { runId, testRunDbId, token }
 };
 
 // Export functions that can be used by the reporter or tests
-export function registerDashcamUrl(testId, url, platform) {
-  logger.debug(`Registering dashcam URL for test ${testId}:`, url);
-  pluginState.dashcamUrls.set(testId, { url, platform });
+export function registerDashcamUrl(testId, url, platform, attempt) {
+  logger.debug(`Registering dashcam URL for test ${testId} (attempt ${attempt || 1}):`, url);
+  // Support multiple attempts per test - store as array
+  if (!pluginState.dashcamUrls.has(testId)) {
+    pluginState.dashcamUrls.set(testId, []);
+  }
+  pluginState.dashcamUrls.get(testId).push({ url, platform, attempt: attempt || 1 });
   pluginState.lastDashcamUrl = url;
 }
 
 export function getDashcamUrl(testId) {
-  return pluginState.dashcamUrls.get(testId);
+  const entries = pluginState.dashcamUrls.get(testId);
+  if (!entries) return undefined;
+  // Return the last entry for backward compatibility (single URL callers)
+  return entries[entries.length - 1];
+}
+
+export function getAllDashcamUrls(testId) {
+  return pluginState.dashcamUrls.get(testId) || [];
 }
 
 export function clearDashcamUrls() {
@@ -743,6 +777,17 @@ class TestDriverReporter {
       // Default to linux if no tests write platform info
       testRunData.platform = "linux";
 
+      // Send version metadata
+      testRunData.nodeVersion = process.version;
+      const tdVer = resolveTestDriverVersion();
+      if (tdVer) {
+        testRunData.testDriverVersion = tdVer;
+      }
+      const vitestVer = resolveVitestVersion();
+      if (vitestVer) {
+        testRunData.vitestVersion = vitestVer;
+      }
+
       logger.debug("Creating test run with data:", JSON.stringify(testRunData));
       pluginState.testRun = await createTestRun(testRunData);
       logger.debug("Test run created:", JSON.stringify(pluginState.testRun));
@@ -929,6 +974,7 @@ class TestDriverReporter {
     logger.debug(`Test meta for ${test.id}:`, meta);
 
     const dashcamUrl = meta.dashcamUrl || null;
+    const dashcamUrls = meta.dashcamUrls || []; // Per-attempt URLs
     const sessionId = meta.sessionId || null;
     const platform = meta.platform || null;
     const sandboxId = meta.sandboxId || null;
@@ -986,8 +1032,12 @@ class TestDriverReporter {
 
       const suiteName = test.suite?.name;
       const startTime = Date.now() - duration; // Calculate start time from duration
+      const retryCount = result.retryCount || 0;
+      const testRunDbId = process.env.TD_TEST_RUN_DB_ID;
+      const consoleUrl = getConsoleUrl(pluginState.apiRoot);
+      const hasRetries = retryCount > 0 && dashcamUrls.length > 1;
 
-      // Record test case with all metadata
+      // Record a single test case with all metadata
       const testCaseData = {
         runId: testRunId,
         testName: test.name,
@@ -997,7 +1047,7 @@ class TestDriverReporter {
         startTime: startTime,
         endTime: Date.now(),
         duration: duration,
-        retries: result.retryCount || 0,
+        retries: retryCount,
       };
 
       // Add sessionId if available
@@ -1010,6 +1060,13 @@ class TestDriverReporter {
         testCaseData.replayUrl = dashcamUrl;
       }
 
+      // Include per-attempt replay URLs for retry visibility
+      if (dashcamUrls.length > 0) {
+        const attemptUrls = dashcamUrls
+          .map(a => ({ attempt: a.attempt, url: a.url || null, sessionId: a.sessionId || null }));
+        testCaseData.replayUrls = attemptUrls;
+      }
+
       if (suiteName) testCaseData.suiteName = suiteName;
       if (errorMessage) testCaseData.errorMessage = errorMessage;
       if (errorStack) testCaseData.errorStack = errorStack;
@@ -1025,7 +1082,6 @@ class TestDriverReporter {
       );
 
       const testCaseDbId = testCaseResponse.data?.id;
-      const testRunDbId = process.env.TD_TEST_RUN_DB_ID;
 
       // Store test case data for GitHub comment generation
       pluginState.recordedTestCases.push({
@@ -1035,14 +1091,25 @@ class TestDriverReporter {
 
       console.log("");
       console.log(
-        `🔗 Test Report: ${getConsoleUrl(pluginState.apiRoot)}/runs/${testRunDbId}/${testCaseDbId}`,
+        `🔗 Test Report: ${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
       );
+
+      // If there were retries, list all per-attempt dashcam URLs for debugging
+      if (hasRetries) {
+        const validAttempts = dashcamUrls.filter(a => a.url);
+        if (validAttempts.length > 0) {
+          console.log(`📋 Retry attempts (${dashcamUrls.length} total):`);
+          for (const attempt of validAttempts) {
+            console.log(`   Attempt ${attempt.attempt}: ${attempt.url}`);
+          }
+        }
+      }
       
       // Output parseable format for docs generation (examples only)
       if (testFile.startsWith("examples/")) {
         const testFileName = path.basename(testFile);
         console.log(
-          `TESTDRIVER_EXAMPLE_URL::${testFileName}::${getConsoleUrl(pluginState.apiRoot)}/runs/${testRunDbId}/${testCaseDbId}`,
+          `TESTDRIVER_EXAMPLE_URL::${testFileName}::${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
         );
       }
     } catch (error) {
@@ -1099,12 +1166,16 @@ function getPlatform() {
   }
 
   // Try to get platform from dashcam URLs (registered during test cleanup)
-  for (const [, data] of pluginState.dashcamUrls) {
-    if (data.platform) {
-      logger.debug(
-        `Using platform from dashcam URL registration: ${data.platform}`,
-      );
-      return data.platform;
+  for (const [, entries] of pluginState.dashcamUrls) {
+    // entries is now an array of {url, platform, attempt}
+    const arr = Array.isArray(entries) ? entries : [entries];
+    for (const data of arr) {
+      if (data.platform) {
+        logger.debug(
+          `Using platform from dashcam URL registration: ${data.platform}`,
+        );
+        return data.platform;
+      }
     }
   }
 

package/lib/vitest/hooks.mjs

@@ -415,11 +415,29 @@ export function TestDriver(context, options = {}) {
       context.task.meta.testOrder = 0;
       context.task.meta.sessionId = currentInstance.getSessionId?.() || null;
 
+      // Initialize dashcamUrls array for tracking per-attempt URLs (persists across retries)
+      if (!context.task.meta.dashcamUrls) {
+        context.task.meta.dashcamUrls = [];
+      }
+
+      // Determine the current attempt number (1-based)
+      const attemptNumber = context.task.meta.dashcamUrls.length + 1;
+      const isRetry = attemptNumber > 1;
+      const attemptLabel = isRetry ? ` (attempt ${attemptNumber})` : "";
+
       // Stop dashcam if it was started - with timeout to prevent hanging
       if (currentInstance._dashcam && currentInstance._dashcam.recording) {
         try {
           const dashcamUrl = await currentInstance.dashcam.stop();
-          // Add dashcam URL to metadata
+
+          // Track this attempt's URL in the per-attempt array
+          context.task.meta.dashcamUrls.push({
+            attempt: attemptNumber,
+            url: dashcamUrl || null,
+            sessionId: currentInstance.getSessionId?.() || null,
+          });
+
+          // Keep backward compatibility - last attempt's URL
           context.task.meta.dashcamUrl = dashcamUrl || null;
 
           // Also register in memory if plugin is available (for cross-process scenarios)
@@ -428,16 +446,15 @@ export function TestDriver(context, options = {}) {
               context.task.id,
               dashcamUrl,
               platform,
+              attemptNumber,
             );
           }
 
-          const debugMode =
-            process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG;
-
-          if (debugMode) {
+          // Always print the dashcam URL for each attempt so it's visible in logs
+          if (dashcamUrl) {
             console.log("");
             console.log(
-              "🎥" + chalk.yellow(` Dashcam URL`) + `: ${dashcamUrl}`,
+              "🎥" + chalk.yellow(` Dashcam URL${attemptLabel}`) + `: ${dashcamUrl}`,
             );
             console.log("");
           }
@@ -461,11 +478,23 @@ export function TestDriver(context, options = {}) {
           if (currentInstance._dashcam) {
             currentInstance._dashcam.recording = false;
           }
+          // Track failed attempt
+          context.task.meta.dashcamUrls.push({
+            attempt: attemptNumber,
+            url: null,
+            sessionId: currentInstance.getSessionId?.() || null,
+            error: error.message,
+          });
           // Ensure dashcamUrl is set to null if stop failed
           context.task.meta.dashcamUrl = null;
         }
       } else {
-        // No dashcam recording, set URL to null explicitly
+        // No dashcam recording - still track the attempt
+        context.task.meta.dashcamUrls.push({
+          attempt: attemptNumber,
+          url: null,
+          sessionId: currentInstance.getSessionId?.() || null,
+        });
         context.task.meta.dashcamUrl = null;
       }
 

package/lib/vitest/setup.mjs

@@ -15,14 +15,15 @@
  */
 
 import {
-  clearDashcamUrls,
-  clearSuiteTestRun,
-  getDashcamUrl,
-  getPluginState,
-  getSuiteTestRun,
-  pluginState,
-  registerDashcamUrl,
-  setSuiteTestRun,
+    clearDashcamUrls,
+    clearSuiteTestRun,
+    getAllDashcamUrls,
+    getDashcamUrl,
+    getPluginState,
+    getSuiteTestRun,
+    pluginState,
+    registerDashcamUrl,
+    setSuiteTestRun,
 } from '../../interfaces/vitest-plugin.mjs';
 
 // Set up global TestDriver plugin interface
@@ -31,6 +32,7 @@ globalThis.__testdriverPlugin = {
   state: pluginState,
   registerDashcamUrl,
   getDashcamUrl,
+  getAllDashcamUrls,
   clearDashcamUrls,
   getPluginState,
   getSuiteTestRun,

package/mcp-server/dist/server.mjs

@@ -401,8 +401,8 @@ Debug mode (connect to existing sandbox):
         const TestDriverSDK = (await import("../../sdk.js")).default;
         // Determine preview mode from environment variable
         // TD_PREVIEW can be "ide", "browser", or "none"
-        // Default to "none" for MCP server (headless) unless explicitly set
-        const previewMode = process.env.TD_PREVIEW || "none";
+        // Default to "ide" so the live preview shows within the IDE
+        const previewMode = process.env.TD_PREVIEW || "ide";
         logger.debug("session_start: Preview mode", { preview: previewMode });
         // Get IP from params or environment (for self-hosted instances)
         const instanceIp = params.ip || process.env.TD_IP;

package/mcp-server/src/server.ts

@@ -509,8 +509,8 @@ Debug mode (connect to existing sandbox):
       
       // Determine preview mode from environment variable
       // TD_PREVIEW can be "ide", "browser", or "none"
-      // Default to "none" for MCP server (headless) unless explicitly set
-      const previewMode = process.env.TD_PREVIEW || "none";
+      // Default to "ide" so the live preview shows within the IDE
+      const previewMode = process.env.TD_PREVIEW || "ide";
       logger.debug("session_start: Preview mode", { preview: previewMode });
       
       // Get IP from params or environment (for self-hosted instances)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.3.8",
+  "version": "7.3.10",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",

package/sdk.d.ts

@@ -230,6 +230,8 @@ export interface TestDriverOptions {
   logging?: boolean;
   /** Enable/disable cache (default: true). Set to false to force regeneration on all find operations */
   cache?: boolean;
+  /** Global AI sampling configuration. Can be overridden per find() or assert() call. */
+  ai?: AIConfig;
   /** Cache threshold configuration for different methods */
   cacheThreshold?: {
     /** Threshold for find operations (default: 0.05 = 5% difference, 95% similarity) */
@@ -546,6 +548,19 @@ export interface FocusApplicationOptions {
   name: string;
 }
 
+/** AI sampling configuration for controlling model behavior */
+export interface AIConfig {
+  /** Temperature for AI sampling (0 = deterministic, higher = more creative). Default: 0 for find verification, model default for assert. */
+  temperature?: number;
+  /** Top-P and Top-K sampling parameters */
+  top?: {
+    /** Top-P (nucleus sampling). Controls diversity by limiting to top P probability mass. Range: 0-1. */
+    p?: number;
+    /** Top-K sampling. Limits choices to top K tokens. 1 = always pick most likely. 0 = disabled. */
+    k?: number;
+  };
+}
+
 /** Options for extract command */
 export interface ExtractOptions {
   /** What to extract */
@@ -564,6 +579,8 @@ export interface AssertOptions {
   os?: string;
   /** Screen resolution for cache partitioning */
   resolution?: string;
+  /** AI sampling configuration (overrides global ai config) */
+  ai?: AIConfig;
 }
 
 /** Options for exec command */
@@ -1028,7 +1045,7 @@ export default class TestDriverSDK {
   find(description: string, cacheThreshold?: number): ChainableElementPromise;
   find(
     description: string,
-    options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number },
+    options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; ai?: AIConfig },
   ): ChainableElementPromise;
 
   /**
@@ -1267,7 +1284,7 @@ export default class TestDriverSDK {
    * // With custom threshold
    * await client.assert('the page loaded', { threshold: 0.01, cacheKey: 'login-test' });
    */
-  assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string }): Promise<boolean>;
+  assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string; ai?: AIConfig }): Promise<boolean>;
 
   /**
    * Extract information from the screen using AI