testdriverai 7.3.44 → 7.4.1

package/.github/skills/testdriver:find/SKILL.md

@@ -33,8 +33,8 @@ const element = await testdriver.find(description, options)
       Similarity threshold (0-1) for cache matching. Lower values require more similarity. Set to -1 to disable cache.
     </ParamField>
     
-    <ParamField path="timeout" type="number">
-      Maximum time in milliseconds to poll for the element. Retries every 5 seconds until found or timeout expires.
+    <ParamField path="timeout" type="number" default={10000}>
+      Maximum time in milliseconds to poll for the element. Retries every 5 seconds until found or timeout expires. Defaults to `10000` (10 seconds). Set to `0` to disable polling and make a single attempt.
     </ParamField>
     
     <ParamField path="confidence" type="number">
@@ -309,19 +309,28 @@ const el = await testdriver.find('the blue submit button', { type: 'any' });
 </Tip>
 ## Polling for Dynamic Elements
 
-For elements that may not be immediately visible, use the `timeout` option to automatically poll:
+By default, `find()` polls for up to 10 seconds (retrying every 5 seconds) until the element is found. You can customize this with the `timeout` option:
 
 ```javascript
-// Poll for element (retries every 5 seconds until found or timeout)
+// Uses default 10s timeout - polls every 5 seconds
+const element = await testdriver.find('login button');
+await element.click();
+
+// Custom timeout - wait up to 30 seconds
 const element = await testdriver.find('login button', { timeout: 30000 });
 await element.click();
+
+// Disable polling - single attempt only
+const element = await testdriver.find('login button', { timeout: 0 });
 ```
 
 The `timeout` option:
+- Defaults to `10000` (10 seconds)
 - Retries finding the element every 5 seconds
 - Stops when the element is found or the timeout expires
 - Logs progress during polling
 - Returns the element (check `element.found()` if not throwing on failure)
+- Set to `0` to disable polling and make a single attempt
 
 ## Zoom Mode for Crowded UIs
 

package/.github/skills/testdriver:waiting-for-elements/SKILL.md

@@ -6,10 +6,16 @@ description: Handle async operations and prevent flaky tests
 
 ## Waiting for Elements
 
-Use the `timeout` option with `find()` to wait for elements that appear after async operations:
+By default, `find()` automatically polls for up to 10 seconds, retrying every 5 seconds until the element is found. This means most elements that appear after short async operations will be found without any extra configuration.
+
+For longer operations, increase the `timeout`:
 
 ```javascript
-// Wait up to 30 seconds for element to appear (polls every 5 seconds)
+// Default behavior - polls for up to 10 seconds automatically
+const element = await testdriver.find('Loading complete indicator');
+await element.click();
+
+// Wait up to 30 seconds for slower operations
 const element = await testdriver.find('Loading complete indicator', { timeout: 30000 });
 await element.click();
 
@@ -17,8 +23,8 @@ await element.click();
 await testdriver.find('submit button').click();
 await testdriver.find('success message', { timeout: 15000 });
 
-// Short timeout for quick checks
-const toast = await testdriver.find('notification toast', { timeout: 5000 });
+// Disable polling for instant checks
+const toast = await testdriver.find('notification toast', { timeout: 0 });
 ```
 
 ## Flake Prevention

package/agent/lib/commands.js

@@ -137,6 +137,54 @@ const createCommands = (
   
   const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
 
+  /**
+   * Track an interaction via HTTP API (fire-and-forget)
+   * @param {Object} data - Interaction data
+   * @param {string} data.interactionType - Type of interaction (click, type, assert, etc.)
+   * @param {string} [data.prompt] - Description/prompt for the interaction
+   * @param {Object} [data.input] - Input data (varies by interaction type)
+   * @param {Object} [data.coordinates] - Coordinates {x, y}
+   * @param {number} data.timestamp - Absolute epoch timestamp
+   * @param {number} [data.duration] - Duration in ms
+   * @param {boolean} data.success - Whether the interaction succeeded
+   * @param {string} [data.error] - Error message if failed
+   * @param {boolean} [data.cacheHit] - Whether cache was used
+   * @param {string} [data.selector] - Selector ID
+   * @param {boolean} [data.selectorUsed] - Whether selector was used
+   * @param {number} [data.confidence] - AI confidence score
+   * @param {string} [data.reasoning] - AI reasoning
+   * @param {number} [data.similarity] - Cache similarity score
+   * @param {string} [data.screenshotUrl] - S3 key for screenshot
+   * @param {boolean} [data.isSecret] - Whether interaction contains sensitive data
+   */
+  const trackInteraction = (data) => {
+    const sessionId = sessionInstance?.get();
+    if (!sessionId) return;
+
+    sdk.req("/api/v7.0.0/testdriver/interaction/track", {
+      session: sessionId,
+      type: data.interactionType,
+      coordinates: data.coordinates,
+      input: data.input,
+      prompt: data.prompt,
+      selectorUsed: data.selectorUsed,
+      selector: data.selector,
+      cacheHit: data.cacheHit,
+      status: "completed",
+      success: data.success,
+      error: data.error,
+      duration: data.duration,
+      timestamp: data.timestamp,
+      isSecret: data.isSecret,
+      confidence: data.confidence,
+      reasoning: data.reasoning,
+      similarity: data.similarity,
+      screenshotUrl: data.screenshotUrl,
+    }).catch((err) => {
+      console.warn(`Failed to track ${data.interactionType} interaction:`, err.message);
+    });
+  };
+
   const findImageOnScreen = async (
     relativePath,
     haystack,
@@ -296,26 +344,19 @@ const createCommands = (
     emitter.emit(events.log.narration, formatter.formatAssertResult(passed, responseText, assertDuration, cacheHit), true);
     
     // Track interaction with success/failure (fire-and-forget)
-    const sessionId = sessionInstance?.get();
-    if (sessionId) {
-      sandbox.send({
-        type: "trackInteraction",
-        interactionType: "assert",
-        session: sessionId,
-        prompt: assertion,
-        timestamp: assertTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-        duration: assertDuration,
-        success: passed,
-        error: passed ? undefined : responseText,
-        cacheHit: cacheHit,
-        confidence: confidence,
-        reasoning: reasoning,
-        similarity: similarity,
-        screenshotUrl: response?.screenshotKey ?? null,
-      }).catch((err) => {
-        console.warn("Failed to track assert interaction:", err.message);
-      });
-    }
+    trackInteraction({
+      interactionType: "assert",
+      prompt: assertion,
+      timestamp: assertTimestamp,
+      duration: assertDuration,
+      success: passed,
+      error: passed ? undefined : responseText,
+      cacheHit: cacheHit,
+      confidence: confidence,
+      reasoning: reasoning,
+      similarity: similarity,
+      screenshotUrl: response?.screenshotKey ?? null,
+    });
     
     if (passed) {
       return true;
@@ -430,40 +471,26 @@ const createCommands = (
       );
       
       // Track interaction success (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId) {
-        const scrollDuration = Date.now() - scrollStartTime;
-        sandbox.send({
-          type: "trackInteraction",
-          interactionType: "scroll",
-          session: sessionId,
-          input: { direction, amount },
-          timestamp: scrollTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-          duration: scrollDuration,
-          success: scrollSuccess,
-          error: scrollError,
-        }).catch((err) => {
-          console.warn("Failed to track scroll interaction:", err.message);
-        });
-      }
+      const scrollDuration = Date.now() - scrollStartTime;
+      trackInteraction({
+        interactionType: "scroll",
+        input: { direction, amount },
+        timestamp: scrollTimestamp,
+        duration: scrollDuration,
+        success: scrollSuccess,
+        error: scrollError,
+      });
     } catch (error) {
       // Track interaction failure (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId) {
-        const scrollDuration = Date.now() - scrollStartTime;
-        sandbox.send({
-          type: "trackInteraction",
-          interactionType: "scroll",
-          session: sessionId,
-          input: { direction, amount },
-          timestamp: scrollTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-          duration: scrollDuration,
-          success: false,
-          error: error.message,
-        }).catch((err) => {
-          console.warn("Failed to track scroll interaction:", err.message);
-        });
-      }
+      const scrollDuration = Date.now() - scrollStartTime;
+      trackInteraction({
+        interactionType: "scroll",
+        input: { direction, amount },
+        timestamp: scrollTimestamp,
+        duration: scrollDuration,
+        success: false,
+        error: error.message,
+      });
       throw error;
     }
   };
@@ -575,15 +602,12 @@ const createCommands = (
         const actionDuration = actionEndTime - clickStartTime;
         
         // Track interaction (fire-and-forget)
-        const sessionId = sessionInstance?.get();
-        if (sessionId && elementData.prompt) {
-          sandbox.send({
-            type: "trackInteraction",
+        if (elementData.prompt) {
+          trackInteraction({
             interactionType: "click",
-            session: sessionId,
             prompt: elementData.prompt,
             input: { x, y, action },
-            timestamp: clickTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
+            timestamp: clickTimestamp,
             duration: actionDuration,
             success: true,
             cacheHit: elementData.cacheHit,
@@ -593,8 +617,6 @@ const createCommands = (
             reasoning: elementData.reasoning ?? null,
             similarity: elementData.similarity ?? null,
             screenshotUrl: elementData.screenshotUrl ?? null,
-          }).catch((err) => {
-            console.warn("Failed to track click interaction:", err.message);
           });
         }
         
@@ -627,16 +649,13 @@ const createCommands = (
       return;
     } catch (error) {
       // Track interaction failure (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId && elementData.prompt) {
+      if (elementData.prompt) {
         const clickDuration = Date.now() - clickStartTime;
-        sandbox.send({
-          type: "trackInteraction",
+        trackInteraction({
           interactionType: "click",
-          session: sessionId,
           prompt: elementData.prompt,
           input: { x, y, action },
-          timestamp: clickTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
+          timestamp: clickTimestamp,
           duration: clickDuration,
           success: false,
           error: error.message,
@@ -646,8 +665,6 @@ const createCommands = (
           confidence: elementData.confidence ?? null,
           reasoning: elementData.reasoning ?? null,
           similarity: elementData.similarity ?? null,
-        }).catch((err) => {
-          console.warn("Failed to track click interaction:", err.message);
         });
       }
       throw error;
@@ -698,18 +715,15 @@ const createCommands = (
       await sandbox.send({ type: "moveMouse", x, y, ...elementData });
 
       // Track interaction (fire-and-forget)
-      const sessionId = sessionInstance?.get();
       const actionEndTime = Date.now();
       const actionDuration = actionEndTime - hoverStartTime;
       
-      if (sessionId && elementData.prompt) {
-        sandbox.send({
-          type: "trackInteraction",
+      if (elementData.prompt) {
+        trackInteraction({
           interactionType: "hover",
-          session: sessionId,
           prompt: elementData.prompt,
           input: { x, y },
-          timestamp: hoverTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
+          timestamp: hoverTimestamp,
           duration: actionDuration,
           success: true,
           cacheHit: elementData.cacheHit,
@@ -719,8 +733,6 @@ const createCommands = (
           reasoning: elementData.reasoning ?? null,
           similarity: elementData.similarity ?? null,
           screenshotUrl: elementData.screenshotUrl ?? null,
-        }).catch((err) => {
-          console.warn("Failed to track hover interaction:", err.message);
         });
       }
 
@@ -741,16 +753,13 @@ const createCommands = (
       return;
     } catch (error) {
       // Track interaction failure (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId && elementData.prompt) {
+      if (elementData.prompt) {
         const hoverDuration = Date.now() - hoverStartTime;
-        sandbox.send({
-          type: "trackInteraction",
+        trackInteraction({
           interactionType: "hover",
-          session: sessionId,
           prompt: elementData.prompt,
           input: { x, y },
-          timestamp: hoverTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
+          timestamp: hoverTimestamp,
           duration: hoverDuration,
           success: false,
           error: error.message,
@@ -761,8 +770,6 @@ const createCommands = (
           reasoning: elementData.reasoning ?? null,
           similarity: elementData.similarity ?? null,
           screenshotUrl: elementData.screenshotUrl ?? null,
-        }).catch((err) => {
-          console.warn("Failed to track hover interaction:", err.message);
         });
       }
       throw error;
@@ -963,23 +970,16 @@ const createCommands = (
       emitter.emit(events.log.narration, formatter.formatTypeResult(text, secret, typeActionEndTime - typeStartTime), true);
       
       // Track interaction (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId) {
-        const typeDuration = Date.now() - typeStartTime;
-        sandbox.send({
-          type: "trackInteraction",
-          interactionType: "type",
-          session: sessionId,
-          // Store masked text if secret, otherwise store actual text
-          input: { text: secret ? "****" : text, delay },
-          timestamp: typeTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-          duration: typeDuration,
-          success: true,
-          isSecret: secret, // Flag this interaction if it contains a secret
-        }).catch((err) => {
-          console.warn("Failed to track type interaction:", err.message);
-        });
-      }
+      const typeDuration = Date.now() - typeStartTime;
+      trackInteraction({
+        interactionType: "type",
+        // Store masked text if secret, otherwise store actual text
+        input: { text: secret ? "****" : text, delay },
+        timestamp: typeTimestamp,
+        duration: typeDuration,
+        success: true,
+        isSecret: secret, // Flag this interaction if it contains a secret
+      });
       
       const redrawStartTime = Date.now();
       await redraw.wait(5000, redrawOptions);
@@ -1036,21 +1036,14 @@ const createCommands = (
       );
       
       // Track interaction (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId) {
-        const pressKeysDuration = Date.now() - pressKeysStartTime;
-        sandbox.send({
-          type: "trackInteraction",
-          interactionType: "pressKeys",
-          session: sessionId,
-          input: { keys },
-          timestamp: pressKeysTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-          duration: pressKeysDuration,
-          success: true,
-        }).catch((err) => {
-          console.warn("Failed to track pressKeys interaction:", err.message);
-        });
-      }
+      const pressKeysDuration = Date.now() - pressKeysStartTime;
+      trackInteraction({
+        interactionType: "pressKeys",
+        input: { keys },
+        timestamp: pressKeysTimestamp,
+        duration: pressKeysDuration,
+        success: true,
+      });
 
       const redrawStartTime = Date.now();
       await redraw.wait(5000, redrawOptions);
@@ -1079,21 +1072,14 @@ const createCommands = (
       const result = await delay(timeout);
       
       // Track interaction (fire-and-forget)
-      const sessionId = sessionInstance?.get();
-      if (sessionId) {
-        const waitDuration = Date.now() - waitStartTime;
-        sandbox.send({
-          type: "trackInteraction",
-          interactionType: "wait",
-          session: sessionId,
-          input: { timeout },
-          timestamp: waitTimestamp, // Use dashcam elapsed time instead of absolute time
-          duration: waitDuration,
-          success: true,
-        }).catch((err) => {
-          console.warn("Failed to track wait interaction:", err.message);
-        });
-      }
+      const waitDuration = Date.now() - waitStartTime;
+      trackInteraction({
+        interactionType: "wait",
+        input: { timeout },
+        timestamp: waitTimestamp,
+        duration: waitDuration,
+        success: true,
+      });
       
       return result;
     },
@@ -1158,44 +1144,30 @@ const createCommands = (
         );
         
         // Track interaction success (fire-and-forget)
-        const sessionId = sessionInstance?.get();
-        if (sessionId) {
-          const waitForImageDuration = Date.now() - startTime;
-          sandbox.send({
-            type: "trackInteraction",
-            interactionType: "waitForImage",
-            session: sessionId,
-            prompt: description,
-            input: { timeout },
-            timestamp: waitForImageTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-            duration: waitForImageDuration,
-            success: true,
-          }).catch((err) => {
-            console.warn("Failed to track waitForImage interaction:", err.message);
-          });
-        }
+        const waitForImageDuration = Date.now() - startTime;
+        trackInteraction({
+          interactionType: "waitForImage",
+          prompt: description,
+          input: { timeout },
+          timestamp: waitForImageTimestamp,
+          duration: waitForImageDuration,
+          success: true,
+        });
         
         return;
       } else {
         // Track interaction failure (fire-and-forget)
-        const sessionId = sessionInstance?.get();
         const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for an image matching the description \"${description}\"`;
-        if (sessionId) {
-          const waitForImageDuration = Date.now() - startTime;
-          sandbox.send({
-            type: "trackInteraction",
-            interactionType: "waitForImage",
-            session: sessionId,
-            prompt: description,
-            input: { timeout },
-            timestamp: waitForImageTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-            duration: waitForImageDuration,
-            success: false,
-            error: errorMsg,
-          }).catch((err) => {
-            console.warn("Failed to track waitForImage interaction:", err.message);
-          });
-        }
+        const waitForImageDuration = Date.now() - startTime;
+        trackInteraction({
+          interactionType: "waitForImage",
+          prompt: description,
+          input: { timeout },
+          timestamp: waitForImageTimestamp,
+          duration: waitForImageDuration,
+          success: false,
+          error: errorMsg,
+        });
         
         throw new MatchError(errorMsg);
       }
@@ -1267,44 +1239,30 @@ const createCommands = (
         emitter.emit(events.log.narration, theme.dim(`"${text}" found!`), true);
         
         // Track interaction success (fire-and-forget)
-        const sessionId = sessionInstance?.get();
-        if (sessionId) {
-          const waitForTextDuration = Date.now() - startTime;
-          sandbox.send({
-            type: "trackInteraction",
-            interactionType: "waitForText",
-            session: sessionId,
-            prompt: text,
-            input: { timeout },
-            timestamp: waitForTextTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-            duration: waitForTextDuration,
-            success: true,
-          }).catch((err) => {
-            console.warn("Failed to track waitForText interaction:", err.message);
-          });
-        }
+        const waitForTextDuration = Date.now() - startTime;
+        trackInteraction({
+          interactionType: "waitForText",
+          prompt: text,
+          input: { timeout },
+          timestamp: waitForTextTimestamp,
+          duration: waitForTextDuration,
+          success: true,
+        });
         
         return;
       } else {
         // Track interaction failure (fire-and-forget)
-        const sessionId = sessionInstance?.get();
         const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for "${text}"`;
-        if (sessionId) {
-          const waitForTextDuration = Date.now() - startTime;
-          sandbox.send({
-            type: "trackInteraction",
-            interactionType: "waitForText",
-            session: sessionId,
-            prompt: text,
-            input: { timeout },
-            timestamp: waitForTextTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-            duration: waitForTextDuration,
-            success: false,
-            error: errorMsg,
-          }).catch((err) => {
-            console.warn("Failed to track waitForText interaction:", err.message);
-          });
-        }
+        const waitForTextDuration = Date.now() - startTime;
+        trackInteraction({
+          interactionType: "waitForText",
+          prompt: text,
+          input: { timeout },
+          timestamp: waitForTextTimestamp,
+          duration: waitForTextDuration,
+          success: false,
+          error: errorMsg,
+        });
         
         throw new MatchError(errorMsg);
       }
@@ -1514,45 +1472,27 @@ const createCommands = (
         });
         
         // Track interaction success
-        const sessionId = sessionInstance?.get();
-        if (sessionId) {
-          try {
-            const rememberDuration = Date.now() - rememberStartTime;
-            await sandbox.send({
-              type: "trackInteraction",
-              interactionType: "extract",
-              session: sessionId,
-              prompt: description,
-              timestamp: rememberTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-              duration: rememberDuration,
-              success: true,
-            });
-          } catch (err) {
-            console.warn("Failed to track extract interaction:", err.message);
-          }
-        }
+        const rememberDuration = Date.now() - rememberStartTime;
+        trackInteraction({
+          interactionType: "extract",
+          prompt: description,
+          timestamp: rememberTimestamp,
+          duration: rememberDuration,
+          success: true,
+        });
         
         return result.data;
       } catch (error) {
         // Track interaction failure
-        const sessionId = sessionInstance?.get();
-        if (sessionId) {
-          try {
-            const rememberDuration = Date.now() - rememberStartTime;
-            await sandbox.send({
-              type: "trackInteraction",
-              interactionType: "extract",
-              session: sessionId,
-              prompt: description,
-              timestamp: rememberTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
-              duration: rememberDuration,
-              success: false,
-              error: error.message,
-            });
-          } catch (err) {
-            console.warn("Failed to track extract interaction:", err.message);
-          }
-        }
+        const rememberDuration = Date.now() - rememberStartTime;
+        trackInteraction({
+          interactionType: "extract",
+          prompt: description,
+          timestamp: rememberTimestamp,
+          duration: rememberDuration,
+          success: false,
+          error: error.message,
+        });
         throw error;
       }
     },