testdriverai 7.1.3 → 7.2.0

package/agent/lib/redraw.js

@@ -8,7 +8,6 @@ const DEFAULT_REDRAW_OPTIONS = {
   enabled: true,           // Master switch to enable/disable redraw detection
   screenRedraw: true,      // Enable screen redraw detection
   networkMonitor: true,    // Enable network activity monitoring
-  diffThreshold: 0.01,     // Percentage threshold for screen diff (0.01 = 0.01%)
 };
 
 // Factory function that creates redraw functionality with the provided system instance
@@ -20,10 +19,6 @@ const createRedraw = (
 ) => {
   // Merge default options with provided defaults
   const baseOptions = { ...DEFAULT_REDRAW_OPTIONS, ...defaultOptions };
-  // Support legacy redrawThresholdPercent number argument
-  if (typeof defaultOptions === 'number') {
-    baseOptions.diffThreshold = defaultOptions;
-  }
   
   const networkUpdateInterval = 15000;
 
@@ -35,7 +30,13 @@ const createRedraw = (
 
   let measurements = [];
   let networkSettled = true;
-  let screenHasRedrawn = null;
+  
+  // Screen stability tracking
+  let initialScreenImage = null;    // The image captured at start() - reference point
+  let lastScreenImage = null;       // Previous frame for consecutive comparison
+  let hasChangedFromInitial = false; // Has screen changed from initial state?
+  let consecutiveFramesStable = false; // Are consecutive frames now stable?
+  let screenMeasurements = [];      // Track consecutive frame diffs for stability detection
 
   // Track network interval to ensure only one exists
   let networkInterval = null;
@@ -45,7 +46,11 @@ const createRedraw = (
     lastRxBytes = null;
     measurements = [];
     networkSettled = true;
-    screenHasRedrawn = false;
+    initialScreenImage = null;
+    lastScreenImage = null;
+    hasChangedFromInitial = false;
+    consecutiveFramesStable = false;
+    screenMeasurements = [];
   };
 
   const parseNetworkStats = (thisRxBytes, thisTxBytes) => {
@@ -85,6 +90,40 @@ const createRedraw = (
     }
   };
 
+  // Parse screen diff stats for consecutive frame stability
+  // Detects when consecutive frames have stopped changing
+  const parseConsecutiveDiffStats = (diffPercent) => {
+    screenMeasurements.push(diffPercent);
+
+    // Keep last 10 measurements for stability detection
+    if (screenMeasurements.length > 10) {
+      screenMeasurements.shift();
+    }
+
+    // Need at least 2 measurements to determine stability
+    if (screenMeasurements.length < 2) {
+      consecutiveFramesStable = false;
+      return;
+    }
+
+    let avgDiff = screenMeasurements.reduce((acc, d) => acc + d, 0) / screenMeasurements.length;
+
+    let stdDevDiff = Math.sqrt(
+      screenMeasurements.reduce((acc, d) => acc + Math.pow(d - avgDiff, 2), 0) /
+        screenMeasurements.length,
+    );
+
+    let zIndexDiff = stdDevDiff !== 0 ? (diffPercent - avgDiff) / stdDevDiff : 0;
+
+    // Consecutive frames are stable when z-index is negative (current diff is below average)
+    // or diff is essentially zero (< 0.1% accounts for compression artifacts)
+    if (screenMeasurements.length >= 2 && (diffPercent < 0.1 || zIndexDiff < 0)) {
+      consecutiveFramesStable = true;
+    } else {
+      consecutiveFramesStable = false;
+    }
+  };
+
   async function updateNetwork() {
     if (sandbox && sandbox.instanceSocketConnected) {
       let network = await sandbox.send({
@@ -141,8 +180,6 @@ const createRedraw = (
     }
   }
 
-  let startImage = null;
-
   // Start network monitoring only when needed
   function startNetworkMonitoring() {
     if (!networkInterval) {
@@ -165,18 +202,18 @@ const createRedraw = (
     // Merge base options with per-call options
     currentOptions = { ...baseOptions, ...options };
     
-    console.log('[redraw] start() called with options:', JSON.stringify(currentOptions));
+    emitter.emit(events.log.debug, `[redraw] start() called with options: ${JSON.stringify(currentOptions)}`);
     
     // If redraw is completely disabled, return early
     if (!currentOptions.enabled) {
-      console.log('[redraw] start() - redraw disabled, returning null');
+      emitter.emit(events.log.debug, '[redraw] start() - redraw disabled, returning null');
       return null;
     }
     
     // If both screenRedraw and networkMonitor are disabled, disable redraw
     if (!currentOptions.screenRedraw && !currentOptions.networkMonitor) {
       currentOptions.enabled = false;
-      console.log('[redraw] start() - both screenRedraw and networkMonitor disabled, returning null');
+      emitter.emit(events.log.debug, '[redraw] start() - both screenRedraw and networkMonitor disabled, returning null');
       return null;
     }
     
@@ -187,17 +224,18 @@ const createRedraw = (
       startNetworkMonitoring();
     }
     
-    // Only capture start image if screen redraw is enabled
+    // Capture initial image for screen stability monitoring
     if (currentOptions.screenRedraw) {
-      startImage = await system.captureScreenPNG(0.25, true);
-      console.log('[redraw] start() - captured startImage:', startImage);
+      initialScreenImage = await system.captureScreenPNG(0.25, true);
+      lastScreenImage = initialScreenImage;
+      emitter.emit(events.log.debug, `[redraw] start() - captured initial image: ${initialScreenImage}`);
     }
     
-    return startImage;
+    return initialScreenImage;
   }
 
   async function checkCondition(resolve, startTime, timeoutMs, options) {
-    const { enabled, screenRedraw, networkMonitor, diffThreshold } = options;
+    const { enabled, screenRedraw, networkMonitor } = options;
     
     // If redraw is disabled, resolve immediately
     if (!enabled) {
@@ -207,33 +245,51 @@ const createRedraw = (
     
     let nowImage = screenRedraw ? await system.captureScreenPNG(0.25, true) : null;
     let timeElapsed = Date.now() - startTime;
-    let diffPercent = 0;
+    let diffFromInitial = 0;
+    let diffFromLast = 0;
     let isTimeout = timeElapsed > timeoutMs;
 
-    // Check screen redraw if enabled and we have a start image to compare against
-    if (screenRedraw && !screenHasRedrawn && startImage && nowImage) {
-      console.log('[redraw] checkCondition() - comparing images:', { startImage, nowImage });
-      diffPercent = await imageDiffPercent(startImage, nowImage);
-      console.log('[redraw] checkCondition() - diffPercent:', diffPercent, 'threshold:', diffThreshold);
-      screenHasRedrawn = diffPercent > diffThreshold;
-      console.log('[redraw] checkCondition() - screenHasRedrawn:', screenHasRedrawn);
-    } else if (screenRedraw && !startImage) {
-      // If no start image was captured, capture one now and wait for next check
-      console.log('[redraw] checkCondition() - no startImage, capturing now');
-      startImage = await system.captureScreenPNG(0.25, true);
+    // Screen stability detection:
+    // 1. Check if screen has changed from initial (detect transition)
+    // 2. Check if consecutive frames are stable (detect settling)
+    if (screenRedraw && nowImage) {
+      // Compare to initial image - has the screen changed at all?
+      if (initialScreenImage && !hasChangedFromInitial) {
+        diffFromInitial = await imageDiffPercent(initialScreenImage, nowImage);
+        emitter.emit(events.log.debug, `[redraw] checkCondition() - diffFromInitial: ${diffFromInitial}`);
+        // Consider changed if diff > 0.1% (accounts for compression artifacts)
+        if (diffFromInitial > 0.1) {
+          hasChangedFromInitial = true;
+          emitter.emit(events.log.debug, `[redraw] checkCondition() - screen has changed from initial!`);
+        }
+      }
+      
+      // Compare consecutive frames - has the screen stopped changing?
+      if (lastScreenImage && lastScreenImage !== initialScreenImage) {
+        diffFromLast = await imageDiffPercent(lastScreenImage, nowImage);
+        emitter.emit(events.log.debug, `[redraw] checkCondition() - diffFromLast: ${diffFromLast}`);
+        parseConsecutiveDiffStats(diffFromLast);
+        emitter.emit(events.log.debug, `[redraw] checkCondition() - consecutiveFramesStable: ${consecutiveFramesStable}, measurements: ${screenMeasurements.length}`);
+      }
+      
+      // Update last image for next comparison
+      lastScreenImage = nowImage;
     }
     
-    // If screen redraw is disabled, consider it as "redrawn"
-    const effectiveScreenRedrawn = screenRedraw ? screenHasRedrawn : true;
+    // Screen is settled when: it has changed from initial AND consecutive frames are now stable
+    const screenSettled = hasChangedFromInitial && consecutiveFramesStable;
+    
+    // If screen redraw is disabled, consider it as "settled"
+    const effectiveScreenSettled = screenRedraw ? screenSettled : true;
     // If network monitor is disabled, consider it as "settled"
     const effectiveNetworkSettled = networkMonitor ? networkSettled : true;
 
-    // Log redraw status
+    // Log redraw status - show both change detection and stability
     let redrawText = !screenRedraw
       ? theme.dim(`disabled`)
-      : effectiveScreenRedrawn
+      : effectiveScreenSettled
         ? theme.green(`y`)
-        : theme.dim(`${diffPercent}/${diffThreshold}%`);
+        : theme.dim(`${hasChangedFromInitial ? '✓' : '?'}→${consecutiveFramesStable ? '✓' : diffFromLast.toFixed(1)}%`);
     let networkText = !networkMonitor
       ? theme.dim(`disabled`)
       : effectiveNetworkSettled
@@ -248,9 +304,11 @@ const createRedraw = (
     emitter.emit(events.redraw.status, {
       redraw: {
         enabled: screenRedraw,
-        hasRedrawn: effectiveScreenRedrawn,
-        diffPercent,
-        threshold: diffThreshold,
+        settled: effectiveScreenSettled,
+        hasChangedFromInitial,
+        consecutiveFramesStable,
+        diffFromInitial,
+        diffFromLast,
         text: redrawText,
       },
       network: {
@@ -268,9 +326,11 @@ const createRedraw = (
       },
     });
 
-    if ((effectiveScreenRedrawn && effectiveNetworkSettled) || isTimeout) {
+    if ((effectiveScreenSettled && effectiveNetworkSettled) || isTimeout) {
       emitter.emit(events.redraw.complete, {
-        screenHasRedrawn: effectiveScreenRedrawn,
+        screenSettled: effectiveScreenSettled,
+        hasChangedFromInitial,
+        consecutiveFramesStable,
         networkSettled: effectiveNetworkSettled,
         isTimeout,
         timeElapsed,

package/agent/lib/sandbox.js

@@ -1,7 +1,27 @@
 const WebSocket = require("ws");
 const marky = require("marky");
+const crypto = require("crypto");
 const { events } = require("../events");
 
+/**
+ * Generate Sentry trace headers for distributed tracing
+ * Uses the same trace ID derivation as the API (MD5 hash of session ID)
+ * @param {string} sessionId - The session ID
+ * @returns {Object} Headers object with sentry-trace and baggage
+ */
+function getSentryTraceHeaders(sessionId) {
+  if (!sessionId) return {};
+  
+  // Same logic as API: derive trace ID from session ID
+  const traceId = crypto.createHash('md5').update(sessionId).digest('hex');
+  const spanId = crypto.randomBytes(8).toString('hex');
+  
+  return {
+    'sentry-trace': `${traceId}-${spanId}-1`,
+    'baggage': `sentry-trace_id=${traceId},sentry-sample_rate=1.0,sentry-sampled=true`
+  };
+}
+
 const createSandbox = (emitter, analytics, sessionInstance) => {
   class Sandbox {
     constructor() {
@@ -16,9 +36,28 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
       this.uniqueId = Math.random().toString(36).substring(7);
       this.os = null; // Store OS value to send with every message
       this.sessionInstance = sessionInstance; // Store session instance to include in messages
+      this.traceId = null; // Sentry trace ID for debugging
+    }
+
+    /**
+     * Get the Sentry trace ID for this session
+     * Useful for debugging with customers - they can share this ID to look up their traces
+     * @returns {string|null} The trace ID or null if not authenticated
+     */
+    getTraceId() {
+      return this.traceId;
     }
 
-    send(message) {
+    /**
+     * Get the Sentry trace URL for this session
+     * @returns {string|null} The full Sentry trace URL or null if no trace ID
+     */
+    getTraceUrl() {
+      if (!this.traceId) return null;
+      return `https://testdriver.sentry.io/explore/traces/trace/${this.traceId}`;
+    }
+
+    send(message, timeout = 300000) {
       let resolvePromise;
       let rejectPromise;
 
@@ -55,10 +94,33 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
           rejectPromise = reject;
         });
 
-        this.ps[message.requestId] = {
+        const requestId = message.requestId;
+        
+        // Set up timeout to prevent hanging requests
+        const timeoutId = setTimeout(() => {
+          if (this.ps[requestId]) {
+            const pendingMessage = this.ps[requestId];
+            // Stop the timing marker to prevent memory leak
+            try {
+              marky.stop(pendingMessage.timingKey);
+            } catch (e) {
+              // Ignore timing errors
+            }
+            delete this.ps[requestId];
+            rejectPromise(new Error(`Sandbox message '${message.type}' timed out after ${timeout}ms`));
+          }
+        }, timeout);
+
+        this.ps[requestId] = {
           promise: p,
-          resolve: resolvePromise,
-          reject: rejectPromise,
+          resolve: (result) => {
+            clearTimeout(timeoutId);
+            resolvePromise(result);
+          },
+          reject: (error) => {
+            clearTimeout(timeoutId);
+            rejectPromise(error);
+          },
           message,
           timingKey,
           startTime: Date.now(),
@@ -66,6 +128,9 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
 
         return p;
       }
+      
+      // Return a rejected promise if socket is not available
+      return Promise.reject(new Error('Sandbox socket not connected'));
     }
 
     async auth(apiKey) {
@@ -76,7 +141,16 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
 
       if (reply.success) {
         this.authenticated = true;
-        emitter.emit(events.sandbox.authenticated);
+        
+        // Log and store the Sentry trace ID for debugging
+        if (reply.traceId) {
+          this.traceId = reply.traceId;
+          console.log('');
+          console.log(`🔗 View Trace:`);
+          console.log(`https://testdriver.sentry.io/explore/traces/trace/${reply.traceId}`);
+        }
+        
+        emitter.emit(events.sandbox.authenticated, { traceId: reply.traceId });
         return true;
       }
     }
@@ -101,7 +175,25 @@ const createSandbox = (emitter, analytics, sessionInstance) => {
 
     async boot(apiRoot) {
       return new Promise((resolve, reject) => {
-        this.socket = new WebSocket(apiRoot.replace("https://", "wss://"));
+        // Get session ID for Sentry trace headers
+        const sessionId = this.sessionInstance?.get();
+        
+        if (!sessionId) {
+          console.warn('[Sandbox] No session ID available at boot time - Sentry tracing will not be available');
+        }
+        
+        const sentryHeaders = getSentryTraceHeaders(sessionId);
+
+        // Build WebSocket URL with Sentry trace headers as query params
+        const wsUrl = new URL(apiRoot.replace("https://", "wss://"));
+        if (sentryHeaders['sentry-trace']) {
+          wsUrl.searchParams.set('sentry-trace', sentryHeaders['sentry-trace']);
+        }
+        if (sentryHeaders['baggage']) {
+          wsUrl.searchParams.set('baggage', sentryHeaders['baggage']);
+        }
+
+        this.socket = new WebSocket(wsUrl.toString());
 
         // handle errors
         this.socket.on("close", () => {

package/agent/lib/sdk.js

@@ -1,9 +1,29 @@
 const { events } = require("../events");
+const crypto = require("crypto");
 
 // get the version from package.json
 const { version } = require("../../package.json");
 const axios = require("axios");
 
+/**
+ * Generate Sentry trace headers for distributed tracing
+ * Uses the same trace ID derivation as the API (MD5 hash of session ID)
+ * @param {string} sessionId - The session ID
+ * @returns {Object} Headers object with sentry-trace and baggage
+ */
+function getSentryTraceHeaders(sessionId) {
+  if (!sessionId) return {};
+  
+  // Same logic as API: derive trace ID from session ID
+  const traceId = crypto.createHash('md5').update(sessionId).digest('hex');
+  const spanId = crypto.randomBytes(8).toString('hex');
+  
+  return {
+    'sentry-trace': `${traceId}-${spanId}-1`,
+    'baggage': `sentry-trace_id=${traceId},sentry-sample_rate=1.0,sentry-sampled=true`
+  };
+}
+
 // Factory function that creates SDK with the provided emitter, config, and session
 let token = null;
 const createSDK = (emitter, config, sessionInstance) => {
@@ -116,11 +136,16 @@ const createSDK = (emitter, config, sessionInstance) => {
       ? [config["TD_API_ROOT"], path].join("")
       : [config["TD_API_ROOT"], "api", version, "testdriver", path].join("/");
 
+    // Get session ID for Sentry trace headers
+    const sessionId = sessionInstance.get();
+    const sentryHeaders = getSentryTraceHeaders(sessionId);
+
     const c = {
       method: "post",
       headers: {
         "Content-Type": "application/json",
         ...(token && { Authorization: `Bearer ${token}` }), // Add the authorization bearer token only if token is set
+        ...sentryHeaders, // Add Sentry distributed tracing headers
       },
       responseType: typeof onChunk === "function" ? "stream" : "json",
       timeout: 60000, // 60 second timeout to prevent hanging requests

package/agent/lib/system.js

@@ -2,6 +2,7 @@
 const fs = require("fs");
 const os = require("os");
 const path = require("path");
+const { randomUUID } = require("crypto");
 const Jimp = require("jimp");
 const { events } = require("../events.js");
 
@@ -31,7 +32,7 @@ const createSystem = (emitter, sandbox, config) => {
   let countImages = 0;
   const tmpFilename = () => {
     countImages = countImages + 1;
-    return path.join(os.tmpdir(), `${new Date().getTime() + countImages}.png`);
+    return path.join(os.tmpdir(), `td-${Date.now()}-${randomUUID().slice(0, 8)}-${countImages}.png`);
   };
 
   const captureAndResize = async (scale = 1, silent = false, mouse = false) => {

package/agent/lib/validation.js

@@ -45,11 +45,11 @@ const types = () =>
         command: '"focus-application"',
         name: "string",
       },
-      // - command: remember # Remember a string value without needing to interact with the desktop.
-      //   description: My dog's name # The key of the memory value to store.
-      //   value: Roofus # The value of the memory to store
-      RememberCommand: {
-        command: '"remember"',
+      // - command: extract # Extract a string value from the screen without needing to interact with the desktop.
+      //   description: My dog's name # The key of the value to extract.
+      //   value: Roofus # The value to extract
+      ExtractCommand: {
+        command: '"extract"',
         description: "string",
         value: "string",
       },
@@ -125,7 +125,7 @@ const types = () =>
       },
 
       Command:
-        "PressKeysCommand | HoverTextCommand | TypeCommand | WaitCommand | HoverImageCommand | FocusApplicationCommand | RememberCommand | GetEmailUrlCommand | ScrollCommand | ScrollUntilTextCommand | ScrollUntilImageCommand | WaitForTextCommand | WaitForImageCommand | AssertCommand | IfCommand",
+        "PressKeysCommand | HoverTextCommand | TypeCommand | WaitCommand | HoverImageCommand | FocusApplicationCommand | ExtractCommand | GetEmailUrlCommand | ScrollCommand | ScrollUntilTextCommand | ScrollUntilImageCommand | WaitForTextCommand | WaitForImageCommand | AssertCommand | IfCommand",
 
       CommandList: "Command[]",