testdriverai 7.2.3 → 7.2.10

package/interfaces/vitest-plugin.mjs

@@ -1,8 +1,6 @@
 import { execSync } from "child_process";
 import crypto from "crypto";
-import fs from "fs";
 import { createRequire } from "module";
-import os from "os";
 import path from "path";
 import { setTestRunInfo } from "./shared-test-state.mjs";
 
@@ -662,76 +660,19 @@ class TestDriverReporter {
     
     logger.debug(`Calculated duration: ${duration}ms (startTime: ${testCase?.startTime}, now: ${Date.now()})`);
 
-    // Read test metadata from file (cross-process communication)
-    let dashcamUrl = null;
-    let sessionId = null;
-    let testFile = "unknown";
-    let testOrder = 0;
+    // Read test metadata from Vitest's task.meta (set in test hooks)
+    const meta = test.meta();
+    logger.debug(`Test meta for ${test.id}:`, meta);
 
-    const testResultFile = path.join(
-      os.tmpdir(),
-      "testdriver-results",
-      `${test.id}.json`,
-    );
-
-    logger.debug(`Looking for test result file with test.id: ${test.id}`);
-    logger.debug(`Test result file path: ${testResultFile}`);
-
-    try {
-      if (fs.existsSync(testResultFile)) {
-        const testResult = JSON.parse(fs.readFileSync(testResultFile, "utf-8"));
-        dashcamUrl = testResult.dashcamUrl || null;
-        const platform = testResult.platform || null;
-        sessionId = testResult.sessionId || null;
-        const absolutePath =
-          testResult.testFile ||
-          test.file?.filepath ||
-          test.file?.name ||
-          "unknown";
-        // Make path relative to project root
-        testFile = pluginState.projectRoot && absolutePath !== "unknown"
-          ? path.relative(pluginState.projectRoot, absolutePath)
-          : absolutePath;
-        testOrder =
-          testResult.testOrder !== undefined ? testResult.testOrder : 0;
-        // Don't override duration from file - use Vitest's result.duration
-        // duration is already set above from result.duration
-
-        // Update test run platform from first test that reports it
-        if (platform && !pluginState.detectedPlatform) {
-          pluginState.detectedPlatform = platform;
-        }
+    const dashcamUrl = meta.dashcamUrl || null;
+    const sessionId = meta.sessionId || null;
+    const platform = meta.platform || null;
+    const sandboxId = meta.sandboxId || null;
+    let testFile = meta.testFile || "unknown";
+    const testOrder = meta.testOrder !== undefined ? meta.testOrder : 0;
 
-        // Clean up the file after reading
-        try {
-          fs.unlinkSync(testResultFile);
-        } catch {
-          // Ignore cleanup errors
-        }
-      } else {
-        logger.debug(`No result file found for test: ${test.id}`);
-        // Fallback to test object properties - try multiple sources
-        // In Vitest, the file path is on test.module.task.filepath
-        const absolutePath =
-          test.module?.task?.filepath ||
-          test.module?.file?.filepath ||
-          test.module?.file?.name ||
-          test.file?.filepath ||
-          test.file?.name ||
-          test.suite?.file?.filepath ||
-          test.suite?.file?.name ||
-          test.location?.file ||
-          "unknown";
-        // Make path relative to project root
-        testFile = pluginState.projectRoot && absolutePath !== "unknown"
-          ? path.relative(pluginState.projectRoot, absolutePath)
-          : absolutePath;
-        logger.debug(`Resolved testFile: ${testFile}`);
-      }
-    } catch (error) {
-      logger.error("Failed to read test result file:", error.message);
-      // Fallback to test object properties - try multiple sources
-      // In Vitest, the file path is on test.module.task.filepath
+    // If testFile not in meta, fallback to test object properties
+    if (testFile === "unknown") {
       const absolutePath =
         test.module?.task?.filepath ||
         test.module?.file?.filepath ||
@@ -742,13 +683,17 @@ class TestDriverReporter {
         test.suite?.file?.name ||
         test.location?.file ||
         "unknown";
-      // Make path relative to project root
       testFile = pluginState.projectRoot && absolutePath !== "unknown"
         ? path.relative(pluginState.projectRoot, absolutePath)
         : absolutePath;
       logger.debug(`Resolved testFile from fallback: ${testFile}`);
     }
 
+    // Update test run platform from first test that reports it
+    if (platform && !pluginState.detectedPlatform) {
+      pluginState.detectedPlatform = platform;
+    }
+
     // Get test run info from environment variables
     const testRunId = process.env.TD_TEST_RUN_ID;
     const token = process.env.TD_TEST_RUN_TOKEN;

package/lib/sentry.js

@@ -0,0 +1,343 @@
+/**
+ * Sentry initialization for TestDriver CLI
+ *
+ * This module initializes Sentry for error tracking and performance monitoring.
+ * It should be required at the very beginning of the CLI entry point.
+ * 
+ * Distributed Tracing:
+ * The CLI uses session-based trace IDs (MD5 hash of session ID) to link
+ * CLI traces with API traces. Call setSessionTraceContext() after establishing
+ * a session to ensure all CLI errors/logs are linked to the same trace.
+ */
+
+const Sentry = require("@sentry/node");
+const crypto = require("crypto");
+const os = require("os");
+const { version } = require("../package.json");
+
+// Store the current session's trace context
+let currentTraceId = null;
+let currentSessionId = null;
+
+// Track if we've attached listeners to avoid duplicates
+let emitterAttached = false;
+
+const isEnabled = () => {
+
+  // Disable if explicitly disabled
+  if (process.env.TD_TELEMETRY === "false") {
+    return false;
+  }
+  return true;
+};
+
+if (isEnabled()) {
+  Sentry.init({
+    dsn:
+      process.env.SENTRY_DSN ||
+      "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
+    environment: process.env.NODE_ENV || "development",
+    release: `testdriverai@${version}`,
+    sampleRate: 1.0,
+    tracesSampleRate: 1.0, // Sample 20% of transactions for performance
+    enableLogs: true,
+    integrations: [
+      Sentry.httpIntegration(),
+      Sentry.nodeContextIntegration(),
+    ],
+    // Set initial context
+    initialScope: {
+      tags: {
+        platform: os.platform(),
+        arch: os.arch(),
+        nodeVersion: process.version,
+      },
+    },
+    // Filter out common non-errors
+    beforeSend(event, hint) {
+
+      console.log('sending sentry event', event);
+
+      const error = hint.originalException;
+
+      // Don't send user-initiated exits
+      if (error && error.message && error.message.includes("User cancelled")) {
+        return null;
+      }
+
+      return event;
+    },
+  });
+}
+
+/**
+ * Set user context for Sentry
+ * @param {Object} user - User object with id, email, etc.
+ */
+function setUser(user) {
+  if (!isEnabled()) return;
+  Sentry.setUser(user);
+}
+
+/**
+ * Set additional context
+ * @param {string} name - Context name
+ * @param {Object} context - Context data
+ */
+function setContext(name, context) {
+  if (!isEnabled()) return;
+  Sentry.setContext(name, context);
+}
+
+/**
+ * Set a tag
+ * @param {string} key - Tag key
+ * @param {string} value - Tag value
+ */
+function setTag(key, value) {
+  if (!isEnabled()) return;
+  Sentry.setTag(key, value);
+}
+
+/**
+ * Capture an exception
+ * @param {Error} error - The error to capture
+ * @param {Object} context - Additional context
+ */
+function captureException(error, context = {}) {
+  if (!isEnabled()) return;
+
+  Sentry.withScope((scope) => {
+    // Link to session trace if available
+    if (currentTraceId && currentSessionId) {
+      scope.setTag("session", currentSessionId);
+      scope.setContext("trace", {
+        trace_id: currentTraceId,
+        session_id: currentSessionId,
+      });
+    }
+    
+    if (context.tags) {
+      Object.entries(context.tags).forEach(([key, value]) => {
+        scope.setTag(key, value);
+      });
+    }
+    if (context.extra) {
+      Object.entries(context.extra).forEach(([key, value]) => {
+        scope.setExtra(key, value);
+      });
+    }
+    Sentry.captureException(error);
+  });
+}
+
+/**
+ * Capture a message
+ * @param {string} message - The message to capture
+ * @param {string} level - Severity level (info, warning, error)
+ */
+function captureMessage(message, level = "info") {
+  if (!isEnabled()) return;
+  
+  Sentry.withScope((scope) => {
+    // Link to session trace if available
+    if (currentTraceId && currentSessionId) {
+      scope.setTag("session", currentSessionId);
+      scope.setContext("trace", {
+        trace_id: currentTraceId,
+        session_id: currentSessionId,
+      });
+    }
+    Sentry.captureMessage(message, level);
+  });
+}
+
+/**
+ * Set the session trace context for distributed tracing
+ * This links CLI errors/logs to the same trace as API calls
+ * @param {string} sessionId - The session ID
+ */
+function setSessionTraceContext(sessionId) {
+  if (!isEnabled() || !sessionId) return;
+  
+  // Derive trace ID from session ID (same algorithm as API)
+  currentTraceId = crypto.createHash("md5").update(sessionId).digest("hex");
+  currentSessionId = sessionId;
+  
+  // Set as global tag so all events include it
+  Sentry.setTag("session", sessionId);
+  Sentry.setTag("trace_id", currentTraceId);
+  
+  // Try to set propagation context for trace linking (may not be available in all versions)
+  try {
+    const scope = Sentry.getCurrentScope();
+    if (scope && typeof scope.setPropagationContext === 'function') {
+      scope.setPropagationContext({
+        traceId: currentTraceId,
+        spanId: currentTraceId.substring(0, 16),
+        sampled: true,
+      });
+    }
+  } catch (e) {
+    // Ignore errors - propagation context may not be supported
+    console.log('Could not set propagation context:', e.message);
+  }
+}
+
+/**
+ * Clear the session trace context
+ */
+function clearSessionTraceContext() {
+  currentTraceId = null;
+  currentSessionId = null;
+}
+
+/**
+ * Get the current trace ID (for debugging)
+ * @returns {string|null} Current trace ID or null
+ */
+function getTraceId() {
+  return currentTraceId;
+}
+
+/**
+ * Attach log listeners to an emitter to capture CLI logs as Sentry breadcrumbs
+ * @param {EventEmitter} emitter - The event emitter to listen to
+ */
+function attachLogListeners(emitter) {
+
+  if (!isEnabled() || !emitter || emitterAttached) return;
+  
+  // Check if Sentry.logger is available
+  if (!Sentry.logger) {
+    console.log('Sentry.logger not available, skipping log listeners');
+    return;
+  }
+  
+  emitterAttached = true;
+
+  // Helper to strip ANSI codes for cleaner logs
+  const stripAnsi = (str) => {
+    if (typeof str !== 'string') return String(str);
+    return str.replace(/\x1B[[(?);]{0,2}(;?\d)*./g, '');
+  };
+
+  // Helper to get current log attributes with trace context
+  const getLogAttributes = (extra = {}) => {
+    const attrs = { ...extra };
+    if (currentSessionId) {
+      attrs['session.id'] = currentSessionId;
+    }
+    if (currentTraceId) {
+      attrs['sentry.trace.trace_id'] = currentTraceId;
+    }
+    // Get current user from Sentry scope
+    try {
+      const user = Sentry.getCurrentScope().getUser();
+      if (user) {
+        if (user.id) attrs['user.id'] = user.id;
+        if (user.email) attrs['user.email'] = user.email;
+        if (user.username) attrs['user.name'] = user.username;
+      }
+    } catch (e) {
+      // Ignore errors getting user
+    }
+    return attrs;
+  };
+  
+  // Capture log:log as info logs
+  emitter.on('log:log', (message) => {
+    Sentry.logger.info(stripAnsi(message), getLogAttributes({ category: 'cli.log' }));
+  });
+  
+  // Capture log:warn as warning logs
+  emitter.on('log:warn', (message) => {
+    Sentry.logger.warn(stripAnsi(message), getLogAttributes({ category: 'cli.warn' }));
+  });
+  
+  // Capture log:debug as debug logs (only in verbose mode)
+  if (process.env.VERBOSE || process.env.DEBUG || process.env.TD_DEBUG) {
+    emitter.on('log:debug', (message) => {
+      Sentry.logger.debug(stripAnsi(message), getLogAttributes({ category: 'cli.debug' }));
+    });
+  }
+  
+  // Capture command events
+  emitter.on('command:start', (data) => {
+    Sentry.logger.info(`Command started: ${data?.command || data?.name || 'unknown'}`, getLogAttributes({
+      category: 'cli.command',
+      ...data,
+    }));
+  });
+  
+  emitter.on('command:error', (data) => {
+    Sentry.logger.error(`Command error: ${data?.message || data?.error || 'unknown'}`, getLogAttributes({
+      category: 'cli.command',
+      ...data,
+    }));
+  });
+  
+  // Capture step events
+  emitter.on('step:start', (data) => {
+    Sentry.logger.info(`Step started: ${data?.step || data?.name || 'unknown'}`, getLogAttributes({
+      category: 'cli.step',
+    }));
+  });
+  
+  emitter.on('step:error', (data) => {
+    Sentry.logger.error(`Step error: ${data?.message || data?.error || 'unknown'}`, getLogAttributes({
+      category: 'cli.step',
+      ...data,
+    }));
+  });
+  
+  // Capture test events
+  emitter.on('test:start', (data) => {
+    Sentry.logger.info(`Test started: ${data?.name || 'unknown'}`, getLogAttributes({
+      category: 'cli.test',
+    }));
+  });
+  
+  emitter.on('test:error', (data) => {
+    Sentry.logger.error(`Test error: ${data?.message || data?.error || 'unknown'}`, getLogAttributes({
+      category: 'cli.test',
+      ...data,
+    }));
+  });
+}
+
+/**
+ * Start a new transaction for performance monitoring
+ * @param {string} name - Transaction name
+ * @param {string} op - Operation type
+ * @returns {Object} Transaction object
+ */
+function startTransaction(name, op = "cli") {
+  if (!isEnabled()) return null;
+  return Sentry.startSpan({ name, op });
+}
+
+/**
+ * Flush pending events before process exit
+ * @param {number} timeout - Timeout in milliseconds
+ */
+async function flush(timeout = 2000) {
+  if (!isEnabled()) return;
+  await Sentry.flush(timeout);
+}
+
+module.exports = {
+  Sentry,
+  isEnabled,
+  setUser,
+  setContext,
+  setTag,
+  captureException,
+  captureMessage,
+  setSessionTraceContext,
+  clearSessionTraceContext,
+  getTraceId,
+  attachLogListeners,
+  startTransaction,
+  flush,
+};

package/lib/vitest/hooks.mjs

@@ -16,8 +16,6 @@
  */
 
 import chalk from 'chalk';
-import fs from 'fs';
-import os from 'os';
 import path from 'path';
 import { vi } from 'vitest';
 import TestDriverSDK from '../../sdk.js';
@@ -137,6 +135,7 @@ const lifecycleHandlers = new WeakMap();
  * @param {string} [options.apiKey] - TestDriver API key (defaults to process.env.TD_API_KEY)
  * @param {boolean} [options.headless] - Run sandbox in headless mode
  * @param {boolean} [options.newSandbox] - Create new sandbox
+ * @param {number} [options.timeout=0] - Sandbox timeout (TTL) in milliseconds. 0 = use provider default (5 min for E2B Linux)
  * @param {boolean} [options.autoConnect=true] - Automatically connect to sandbox
  * @returns {TestDriver} TestDriver client instance
  * 
@@ -155,9 +154,16 @@ export function TestDriver(context, options = {}) {
     throw new Error('TestDriver() requires Vitest context. Pass the context parameter from your test function: test("name", async (context) => { ... })');
   }
   
-  // Return existing instance if already created for this test
+  // Return existing instance if already created for this test AND it's still connected
+  // On retry, the previous instance will be disconnected, so we need to create a new one
   if (testDriverInstances.has(context.task)) {
-    return testDriverInstances.get(context.task);
+    const existingInstance = testDriverInstances.get(context.task);
+    if (existingInstance.connected) {
+      return existingInstance;
+    }
+    // Instance exists but is disconnected (likely a retry) - remove it and create fresh
+    testDriverInstances.delete(context.task);
+    lifecycleHandlers.delete(context.task);
   }
   
   // Get global plugin options if available
@@ -195,7 +201,6 @@ export function TestDriver(context, options = {}) {
   
   if (autoConnect) {
     testdriver.__connectionPromise = (async () => {
-        console.log('[testdriver] Connecting to sandbox...');
         if (debugConsoleSpy) {
           console.log('[DEBUG] Before auth - sandbox.instanceSocketConnected:', testdriver.sandbox?.instanceSocketConnected);
         }
@@ -232,85 +237,80 @@ export function TestDriver(context, options = {}) {
   }
   
   // Register cleanup handler with dashcam.stop()
-  if (!lifecycleHandlers.has(context.task)) {
-    const cleanup = async () => {
-      try {
-        // Stop dashcam if it was started - with timeout to prevent hanging
-        if (testdriver._dashcam && testdriver._dashcam.recording) {
-          try {
-            const dashcamUrl = await testdriver.dashcam.stop();
-            console.log('');
-            console.log('🎥' + chalk.yellow(` Dashcam URL`) + `: ${dashcamUrl}`);
-            console.log('');
-            // Write test result to file for the reporter (cross-process communication)
-            // This should happen regardless of whether dashcam succeeded, to ensure platform info is available
-            const testId = context.task.id;
-            const platform = testdriver.os || 'linux';
-            const absolutePath = context.task.file?.filepath || context.task.file?.name || 'unknown';
-            const projectRoot = process.cwd();
-            const testFile = absolutePath !== 'unknown'
-              ? path.relative(projectRoot, absolutePath)
-              : absolutePath;
-            
-            // Create results directory if it doesn't exist
-            const resultsDir = path.join(os.tmpdir(), 'testdriver-results');
-            if (!fs.existsSync(resultsDir)) {
-              fs.mkdirSync(resultsDir, { recursive: true });
-            }
-            
-            // Write test result file
-            const testResultFile = path.join(resultsDir, `${testId}.json`);
-            const testResult = {
-              dashcamUrl: dashcamUrl || null,
-              platform,
-              testFile,
-              testOrder: 0,
-              sessionId: testdriver.getSessionId(),
-            };
-            
-            fs.writeFileSync(testResultFile, JSON.stringify(testResult, null, 2));
-            
-            // Also register in memory if plugin is available
-            if (globalThis.__testdriverPlugin?.registerDashcamUrl) {
-              globalThis.__testdriverPlugin.registerDashcamUrl(testId, dashcamUrl, platform);
-            }
-          } catch (error) {
-            // Log more detailed error information for debugging
-            console.error('❌ Failed to stop dashcam:', error.name || error.constructor?.name || 'Error');
-            if (error.message) console.error('   Message:', error.message);
-            // NotFoundError during cleanup is expected if sandbox already terminated
-            if (error.name === 'NotFoundError' || error.responseData?.error === 'NotFoundError') {
-              console.log('   ℹ️  Sandbox session already terminated - dashcam stop skipped');
-            }
-            // Mark as not recording to prevent retries
-            if (testdriver._dashcam) {
-              testdriver._dashcam.recording = false;
-            }
+  // We always register a new cleanup handler because on retry we need to clean up the new instance
+  const cleanup = async () => {
+    // Get the current instance from the WeakMap (not from closure)
+    // This ensures we clean up the correct instance on retries
+    const currentInstance = testDriverInstances.get(context.task);
+    if (!currentInstance) {
+      return; // Already cleaned up
+    }
+    
+    try {
+      // Stop dashcam if it was started - with timeout to prevent hanging
+      if (currentInstance._dashcam && currentInstance._dashcam.recording) {
+        try {
+          const dashcamUrl = await currentInstance.dashcam.stop();
+          console.log('');
+          console.log('🎥' + chalk.yellow(` Dashcam URL`) + `: ${dashcamUrl}`);
+          console.log('');
+          
+          // Set test metadata directly on the Vitest task context
+          // This is the proper way to pass data from test to reporter
+          const platform = currentInstance.os || 'linux';
+          const absolutePath = context.task.file?.filepath || context.task.file?.name || 'unknown';
+          const projectRoot = process.cwd();
+          const testFile = absolutePath !== 'unknown'
+            ? path.relative(projectRoot, absolutePath)
+            : absolutePath;
+          
+          // Set metadata on the task for the reporter to read
+          context.task.meta.dashcamUrl = dashcamUrl || null;
+          context.task.meta.platform = platform;
+          context.task.meta.testFile = testFile;
+          context.task.meta.testOrder = 0;
+          context.task.meta.sessionId = currentInstance.getSessionId();
+          
+          // Also register in memory if plugin is available (for cross-process scenarios)
+          if (globalThis.__testdriverPlugin?.registerDashcamUrl) {
+            globalThis.__testdriverPlugin.registerDashcamUrl(context.task.id, dashcamUrl, platform);
+          }
+        } catch (error) {
+          // Log more detailed error information for debugging
+          console.error('❌ Failed to stop dashcam:', error.name || error.constructor?.name || 'Error');
+          if (error.message) console.error('   Message:', error.message);
+          // NotFoundError during cleanup is expected if sandbox already terminated
+          if (error.name === 'NotFoundError' || error.responseData?.error === 'NotFoundError') {
+            console.log('   ℹ️  Sandbox session already terminated - dashcam stop skipped');
+          }
+          // Mark as not recording to prevent retries
+          if (currentInstance._dashcam) {
+            currentInstance._dashcam.recording = false;
           }
         }
-        
-        // Clean up console spies
-        cleanupConsoleSpy(testdriver);
-        
-        // Wait for connection to finish if it was initiated
-        if (testdriver.__connectionPromise) {
-          await testdriver.__connectionPromise.catch(() => {}); // Ignore connection errors during cleanup
-        }
-        
-        // Disconnect with timeout
-        await Promise.race([
-          testdriver.disconnect(),
-          new Promise((resolve) => setTimeout(resolve, 5000)) // 5s timeout for disconnect
-        ]);
-      } catch (error) {
-        console.error('Error disconnecting client:', error);
       }
-    };
-    lifecycleHandlers.set(context.task, cleanup);
-    
-    // Vitest will call this automatically after the test
-    context.onTestFinished?.(cleanup);
-  }
+      
+      // Clean up console spies
+      cleanupConsoleSpy(currentInstance);
+      
+      // Wait for connection to finish if it was initiated
+      if (currentInstance.__connectionPromise) {
+        await currentInstance.__connectionPromise.catch(() => {}); // Ignore connection errors during cleanup
+      }
+      
+      // Disconnect with timeout
+      await Promise.race([
+        currentInstance.disconnect(),
+        new Promise((resolve) => setTimeout(resolve, 5000)) // 5s timeout for disconnect
+      ]);
+    } catch (error) {
+      console.error('Error disconnecting client:', error);
+    }
+  };
+  lifecycleHandlers.set(context.task, cleanup);
+  
+  // Vitest will call this automatically after the test (each retry attempt)
+  context.onTestFinished?.(cleanup);
   
   return testdriver;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.2.3",
+  "version": "7.2.10",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "exports": {
@@ -52,6 +52,7 @@
     "@oclif/plugin-help": "^6.2.30",
     "@oclif/plugin-not-found": "^3.2.59",
     "@oclif/plugin-warn-if-update-available": "^3.1.43",
+    "@sentry/node": "^9.47.1",
     "@stoplight/yaml-ast-parser": "^0.0.50",
     "ajv": "^8.17.1",
     "arktype": "^2.1.19",
@@ -97,8 +98,8 @@
     "mocha": "^10.8.2",
     "node-addon-api": "^8.0.0",
     "prettier": "3.3.3",
-    "testdriverai": "^6.1.11",
-    "vitest": "^4.0.15"
+    "testdriverai": "^7.2.3",
+    "vitest": "^4.0.16"
   },
   "optionalDependencies": {
     "@esbuild/linux-x64": "^0.21.5"