testdriverai 7.8.0 → 7.9.0-test.1

package/interfaces/vitest-plugin.mjs

@@ -13,6 +13,7 @@ const channelConfig = require("../lib/resolve-channel.js");
 
 // Import Sentry for error reporting
 const Sentry = require("@sentry/node");
+const chalk = require("chalk");
 
 // Track if Sentry has been initialized
 let sentryInitialized = false;
@@ -23,7 +24,7 @@ let sentryInitialized = false;
  */
 function initializeSentry() {
   if (sentryInitialized) return;
-  
+
   // Respect telemetry opt-out
   if (process.env.TD_TELEMETRY === "false") {
     return;
@@ -31,12 +32,12 @@ function initializeSentry() {
 
   try {
     const version = resolveTestDriverVersion() || "unknown";
-    
+
     Sentry.init({
       dsn:
         process.env.SENTRY_DSN ||
         "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
-      environment: "vitest",
+      environment: channelConfig.sentryEnvironment,
       release: version,
       sampleRate: 1.0,
       tracesSampleRate: 1.0,
@@ -53,12 +54,12 @@ function initializeSentry() {
       // Filter out events that should not be reported to Sentry
       beforeSend(event, hint) {
         const error = hint.originalException;
-        
+
         // Don't send user-cancelled errors
         if (error && error.message && error.message.includes("User cancelled")) {
           return null;
         }
-        
+
         // Don't send test failures - these are expected behavior, not bugs in the SDK
         // Test failures indicate the test found a problem, which is the intended use case
         if (event.exception?.values) {
@@ -67,28 +68,33 @@ function initializeSentry() {
             if (exception.type === "TestFailure") {
               return null;
             }
-            
+
             // Filter out common user code errors (ReferenceError, TypeError from user tests)
             // Only report if the error originates from TestDriver SDK code, not user test code
             const isUserCodeError = exception.stacktrace?.frames?.some(frame => {
               const filename = frame.filename || frame.abs_path || "";
               // Check if error is from user test files (not from SDK internals)
-              return filename.includes("/tests/") || 
-                     filename.includes("/test/") || 
-                     filename.includes(".test.") ||
-                     filename.includes(".spec.");
+              return filename.includes("/tests/") ||
+                filename.includes("/test/") ||
+                filename.includes(".test.") ||
+                filename.includes(".spec.");
             });
-            
+
             if (isUserCodeError && (exception.type === "ReferenceError" || exception.type === "TypeError")) {
               return null;
             }
+
+            // Filter out ElementNotFoundError - expected test outcome, not a crash
+            if (exception.type === "ElementNotFoundError") {
+              return null;
+            }
           }
         }
-        
+
         return event;
       },
     });
-    
+
     sentryInitialized = true;
     logger.debug("Sentry initialized for vitest");
   } catch (err) {
@@ -118,17 +124,17 @@ async function flushSentry(timeout = 2000) {
 function resolveTestDriverVersion() {
   try {
     return require("../package.json").version;
-  } catch {}
+  } catch { }
 
   try {
     const cwdRequire = createRequire(path.join(process.cwd(), "package.json"));
     return cwdRequire("testdriverai/package.json").version;
-  } catch {}
+  } catch { }
 
   try {
     const pkgPath = path.join(process.cwd(), "node_modules", "testdriverai", "package.json");
     return JSON.parse(fs.readFileSync(pkgPath, "utf8")).version;
-  } catch {}
+  } catch { }
 
   return null;
 }
@@ -149,19 +155,19 @@ function resolveVitestVersion() {
   // Strategy 1: createRequire from import.meta.url (standard CJS interop)
   try {
     return require("vitest/package.json").version;
-  } catch {}
+  } catch { }
 
   // Strategy 2: createRequire from process.cwd() (works when import.meta.url is rewritten)
   try {
     const cwdRequire = createRequire(path.join(process.cwd(), "package.json"));
     return cwdRequire("vitest/package.json").version;
-  } catch {}
+  } catch { }
 
   // Strategy 3: read directly from node_modules on disk
   try {
     const vitestPkgPath = path.join(process.cwd(), "node_modules", "vitest", "package.json");
     return JSON.parse(fs.readFileSync(vitestPkgPath, "utf8")).version;
-  } catch {}
+  } catch { }
 
   return null;
 }
@@ -176,7 +182,7 @@ function checkVitestVersion() {
   if (!version) {
     throw new Error(
       "TestDriver requires Vitest to be installed. " +
-        "Please install it: npm install vitest@latest",
+      "Please install it: npm install vitest@latest",
     );
   }
 
@@ -184,7 +190,7 @@ function checkVitestVersion() {
   if (major < MINIMUM_VITEST_VERSION) {
     throw new Error(
       `TestDriver requires Vitest >= ${MINIMUM_VITEST_VERSION}.0.0, but found ${version}. ` +
-        `Please upgrade Vitest: npm install vitest@latest`,
+      `Please upgrade Vitest: npm install vitest@latest`,
     );
   }
 }
@@ -374,7 +380,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
     // Network-level error (fetch failed entirely)
     const networkError = new Error(
       `Unable to reach TestDriver API at ${apiRoot}. ` +
-        "Check your internet connection and try again.",
+      "Check your internet connection and try again.",
     );
     networkError.code = "NETWORK_ERROR";
     networkError.isNetworkError = true;
@@ -394,8 +400,8 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
     if (response.status === 401) {
       const authError = new Error(
         data.message ||
-          "Invalid API key. Please check your TD_API_KEY and try again. " +
-            "Get your API key at https://console.testdriver.ai/team",
+        "Invalid API key. Please check your TD_API_KEY and try again. " +
+        "Get your API key at https://console.testdriver.ai/team",
       );
       authError.code = data.error || "INVALID_API_KEY";
       authError.isAuthError = true;
@@ -406,7 +412,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
     if (response.status >= 500) {
       const serverError = new Error(
         data.message ||
-          `TestDriver API is currently unavailable (HTTP ${response.status}). Please try again later.`,
+        `TestDriver API is currently unavailable (HTTP ${response.status}). Please try again later.`,
       );
       serverError.code = data.error || "API_UNAVAILABLE";
       serverError.isServerError = true;
@@ -426,7 +432,7 @@ export async function authenticateWithApiKey(apiKey, apiRoot) {
     // Other HTTP errors
     throw new Error(
       `Authentication failed: ${response.status} ${response.statusText}` +
-        (data.message ? ` - ${data.message}` : ""),
+      (data.message ? ` - ${data.message}` : ""),
     );
   }
 
@@ -1138,7 +1144,7 @@ class TestDriverReporter {
         const error = result.errors[0];
         errorMessage = error.message;
         errorStack = error.stack;
-        
+
         // Note: We do NOT report test failures to Sentry.
         // Test failures are expected behavior (they indicate a test found a bug).
         // We only want actual SDK crashes and exceptions reported to Sentry.
@@ -1208,8 +1214,125 @@ class TestDriverReporter {
 
       console.log("");
       console.log(
-        `🔗 Test Report: ${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
+        chalk.cyan(`🔗 Test Report: ${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`),
       );
+      console.log("");
+
+      // Write per-test-case JSON result file
+      {
+        const testResult = meta.testResult || {};
+
+        // Parse replay URL to extract replayId and shareKey for embed links
+        let replayUrl = dashcamUrl || null;
+        let replayGifUrl = null;
+        let replayEmbedUrl = null;
+        let replayMarkdown = null;
+        const replayMatch = dashcamUrl && dashcamUrl.match(/\/replay\/([a-f0-9]+)\?share=([^&\s]+)/);
+        if (replayMatch) {
+          const [, replayId, shareKey] = replayMatch;
+          const apiRoot = pluginState.apiRoot;
+          replayGifUrl = `${apiRoot}/replay/${replayId}/gif?shareKey=${shareKey}`;
+          replayEmbedUrl = `${consoleUrl}/replay/${replayId}?share=${shareKey}&embed=true`;
+          replayMarkdown = `[![Test Recording](${replayGifUrl})](${replayUrl})`;
+        }
+
+        const resultData = {
+          // Versions
+          versions: {
+            sdk: testResult.sdkVersion || null,
+            vitest: resolveVitestVersion() || null,
+            api: testResult.apiVersion || null,
+            runnerBefore: testResult.runnerVersionBefore || null,
+            runnerAfter: testResult.runnerVersionAfter || null,
+            runnerWasUpdated: testResult.wasUpdated || false,
+          },
+
+          // Test info
+          test: {
+            file: testResult.testFile || null,
+            name: testResult.testName || null,
+            suite: testResult.suiteName || null,
+            passed: status === "passed",
+            caseId: testCaseDbId || null,
+            runId: testRunDbId || null,
+            error: errorMessage || testResult.error || null,
+            errorStack: errorStack || testResult.errorStack || null,
+          },
+
+          // URLs
+          urls: {
+            api: testResult.apiUrl || null,
+            console: consoleUrl || null,
+            vnc: testResult.vncUrl || null,
+            testRun: testCaseDbId ? `${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}` : null,
+          },
+
+          // Recording replay
+          replay: {
+            url: replayUrl,
+            gifUrl: replayGifUrl,
+            embedUrl: replayEmbedUrl,
+            markdown: replayMarkdown,
+          },
+
+          // Timing
+          date: testResult.date || new Date().toISOString(),
+
+          // Team & session
+          team: {
+            id: testResult.teamId || null,
+            sessionId: testResult.sessionId || null,
+          },
+
+          // Infrastructure
+          infrastructure: {
+            sandboxId: testResult.sandboxId || null,
+            instanceId: testResult.instanceId || null,
+            os: testResult.os || null,
+            amiId: testResult.amiId || null,
+            e2bTemplateId: testResult.e2bTemplateId || null,
+            imageVersion: testResult.imageVersion || null,
+          },
+
+          // Realtime
+          realtime: {
+            channel: testResult.realtimeChannel || null,
+            messageCount: testResult.realtimeMessageCount || 0,
+          },
+
+          // Interactions
+          interactions: testResult.interactions || { total: 0, cached: 0, byType: {} },
+        };
+
+        // Sanitize testName for filesystem use
+        const safeName = (test.name || "unknown").replace(/[^a-zA-Z0-9_.-]/g, "_").substring(0, 200);
+        const resultDir = path.join(process.cwd(), ".testdriver", "results", testFile);
+        fs.mkdirSync(resultDir, { recursive: true });
+
+        // Include a stable unique suffix in the filename to avoid collisions
+        // when multiple tests in the same file share the same name.
+        const hashSourceParts = [];
+        if (test.id) {
+          hashSourceParts.push(String(test.id));
+        }
+        if (Array.isArray(test.suitePath)) {
+          hashSourceParts.push(test.suitePath.join(" > "));
+        }
+        if (test.file && (test.file.name || test.file.path)) {
+          hashSourceParts.push(test.file.name || test.file.path);
+        }
+        // Fallback to the test name if no other identifiers are available.
+        if (hashSourceParts.length === 0) {
+          hashSourceParts.push(test.name || "unknown");
+        }
+        const hashSource = hashSourceParts.join(" | ");
+        const uniqueHash = crypto.createHash("sha256").update(hashSource).digest("hex").slice(0, 8);
+
+        fs.writeFileSync(
+          path.join(resultDir, `${safeName}-${uniqueHash}.json`),
+          JSON.stringify(resultData, null, 2),
+        );
+      }
 
       // If there were retries, list all per-attempt dashcam URLs for debugging
       if (hasRetries) {
@@ -1221,14 +1344,7 @@ class TestDriverReporter {
           }
         }
       }
-      
-      // Output parseable format for docs generation (examples only)
-      if (testFile.startsWith("examples/")) {
-        const testFileName = path.basename(testFile);
-        console.log(
-          `TESTDRIVER_EXAMPLE_URL::${testFileName}::${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
-        );
-      }
+
     } catch (error) {
       logger.error("Failed to report test case:", error.message);
     }
@@ -1247,12 +1363,20 @@ class TestDriverReporter {
  * @returns {string} The corresponding web console URL
  */
 function getConsoleUrl(apiRoot) {
-  // Allow explicit override via env (e.g. VITE_DOMAIN from .env)
-  if (process.env.VITE_DOMAIN) return process.env.VITE_DOMAIN;
+  // Explicit override — use TD_CONSOLE_URL when deliberately set
+  if (process.env.TD_CONSOLE_URL) return process.env.TD_CONSOLE_URL;
 
   if (!apiRoot) return "https://console.testdriver.ai";
 
-  // Map known channel API URLs to their console equivalents
+  // Fly.io: swap "-api" for "-web" in the hostname
+  // e.g. preview-138-api.fly.dev -> preview-138-web.fly.dev
+  //      td-test-api.fly.dev     -> td-test-web.fly.dev
+  const flyMatch = apiRoot.match(/https:\/\/([\w-]+)-api\.fly\.dev/);
+  if (flyMatch) {
+    return `https://${flyMatch[1]}-web.fly.dev`;
+  }
+
+  // Known channel API URLs -> console equivalents
   // e.g. https://api-canary.testdriver.ai -> https://console-canary.testdriver.ai
   for (const url of Object.values(channelConfig.channels)) {
     if (url === apiRoot) {
@@ -1260,27 +1384,19 @@ function getConsoleUrl(apiRoot) {
     }
   }
 
-  // Local development: ngrok/cloudflare tunnels -> localhost:3001
+  // Local development
   if (apiRoot.includes("ngrok.io") || apiRoot.includes("trycloudflare.com") || apiRoot.includes("localhost")) {
-    return `http://localhost:3001`;
+    return "http://localhost:3001";
   }
 
-  // Render PR previews: map API service to Web service
-  // canary-api-pr-123.onrender.com -> canary-web-pr-123.onrender.com
-  // testdriver-api-i4m4-pr-123.onrender.com -> web-i4m4-pr-123.onrender.com
+  // Render PR previews (legacy)
   const renderPrMatch = apiRoot.match(/https:\/\/([\w-]+)-api(-[\w]+)?(-pr-\d+)?\.onrender\.com/);
   if (renderPrMatch) {
     const [, prefix, suffix, prSuffix] = renderPrMatch;
-    let webPrefix;
-    if (prefix === 'testdriver' && suffix) {
-      webPrefix = 'web' + suffix;
-    } else {
-      webPrefix = prefix + '-web';
-    }
+    const webPrefix = (prefix === 'testdriver' && suffix) ? 'web' + suffix : prefix + '-web';
     return `https://${webPrefix}${prSuffix || ''}.onrender.com`;
   }
 
-  // Other tunnels or unknown hosts: return as-is
   return apiRoot;
 }
 

package/lib/core/Dashcam.js

@@ -31,8 +31,7 @@ class Dashcam {
     this.apiKey =
       options.apiKey ||
       client.apiKey ||
-      client.config?.TD_API_KEY ||
-      "4e93d8bf-3886-4d26-a144-116c4063522d";
+      client.config?.TD_API_KEY;
     this.autoStart = options.autoStart ?? false;
     this.logs = options.logs || [];
     this.recording = false;
@@ -104,7 +103,7 @@ class Dashcam {
       return `https://console-${envMatch[1]}.testdriver.ai`;
     }
 
-    // Production: API on render.com or v6 -> Console on testdriver.ai
+    // Production: API on custom domain or v6 -> Console on testdriver.ai
     if (
       apiRoot.includes("api.testdriver.ai") ||
       apiRoot.includes("v6.testdriver.ai")
@@ -117,24 +116,19 @@ class Dashcam {
       return "http://localhost:3001";
     }
 
-    // Render PR previews: map API service to Web service
-    // canary-api-pr-123.onrender.com -> canary-web-pr-123.onrender.com
-    // testdriver-api-i4m4-pr-123.onrender.com -> web-i4m4-pr-123.onrender.com
-    const renderPrMatch = apiRoot.match(/https:\/\/([\w-]+)-api(-[\w]+)?(-pr-\d+)?\.onrender\.com/);
-    if (renderPrMatch) {
-      const [, prefix, suffix, prSuffix] = renderPrMatch;
-      // Map API naming to Web naming:
-      // canary-api -> canary-web
-      // testdriver-api-i4m4 -> web-i4m4
-      let webPrefix;
-      if (prefix === 'testdriver' && suffix) {
-        // testdriver-api-i4m4 -> web-i4m4
-        webPrefix = 'web' + suffix;
-      } else {
-        // canary-api -> canary-web
-        webPrefix = prefix + '-web';
-      }
-      return `https://${webPrefix}${prSuffix || ''}.onrender.com`;
+    // Fly.io PR previews: map API app to Web app
+    // pr-123-api.fly.dev -> pr-123-web.fly.dev
+    const flyPrMatch = apiRoot.match(/https:\/\/(pr-\d+)-api\.fly\.dev/);
+    if (flyPrMatch) {
+      const [, prPrefix] = flyPrMatch;
+      return `https://${prPrefix}-web.fly.dev`;
+    }
+
+    // Fly.io environment apps: test-api.fly.dev -> test-web.fly.dev
+    const flyEnvMatch = apiRoot.match(/https:\/\/([\w-]+)-api\.fly\.dev/);
+    if (flyEnvMatch) {
+      const [, prefix] = flyEnvMatch;
+      return `https://${prefix}-web.fly.dev`;
     }
 
     // Cloudflare tunnels, custom domains, etc.: the web console is served
@@ -446,7 +440,7 @@ class Dashcam {
     this.recording = false;
 
     // Extract the /replay/... path from CLI output and reconstruct the URL
-    // using getConsoleUrl(). The CLI may return a wrong domain (e.g. canary-web.onrender.com)
+    // using getConsoleUrl(). The CLI may return a wrong domain
     // so we always rewrite the base URL to match the current environment.
     if (output) {
       // Match /replay/{id} with optional query params from any URL or broken prefix

package/lib/environments.json

@@ -1,18 +1,22 @@
 {
   "dev": {
     "apiRoot": "https://api-dev.testdriver.ai",
-    "consoleUrl": "https://console-dev.testdriver.ai"
+    "consoleUrl": "https://console-dev.testdriver.ai",
+    "tdEnv": "dev"
   },
   "test": {
     "apiRoot": "https://api-test.testdriver.ai",
-    "consoleUrl": "https://console-test.testdriver.ai"
+    "consoleUrl": "https://console-test.testdriver.ai",
+    "tdEnv": "staging"
   },
   "canary": {
     "apiRoot": "https://api-canary.testdriver.ai",
-    "consoleUrl": "https://console-canary.testdriver.ai"
+    "consoleUrl": "https://console-canary.testdriver.ai",
+    "tdEnv": "production"
   },
   "stable": {
     "apiRoot": "https://api.testdriver.ai",
-    "consoleUrl": "https://console.testdriver.ai"
+    "consoleUrl": "https://console.testdriver.ai",
+    "tdEnv": "production"
   }
 }