testdriverai 7.0.0 → 7.1.1

package/agent/lib/commands.js

@@ -13,6 +13,21 @@ const { createRedraw } = require("./redraw.js");
 
 const { events } = require("../events.js");
 
+/**
+ * Helper to detect if arguments are using object-based API or positional API
+ * @param {Array} args - The arguments passed to a command
+ * @param {Array<string>} knownKeys - Keys that would be present in object-based call
+ * @returns {boolean} True if using object-based API
+ */
+const isObjectArgs = (args, knownKeys) => {
+  if (args.length === 0) return false;
+  if (args.length === 1 && typeof args[0] === 'object' && args[0] !== null && !Array.isArray(args[0])) {
+    // Check if it has at least one known key
+    return knownKeys.some(key => key in args[0]);
+  }
+  return false;
+};
+
 /**
  * Error When a match is not found
  * these should be recoverable by --heal
@@ -36,6 +51,28 @@ class CommandError extends Error {
   }
 }
 
+/**
+ * Extract redraw options from command options
+ * @param {Object} options - Command options that may contain redraw settings
+ * @returns {Object} Redraw options object
+ */
+const extractRedrawOptions = (options = {}) => {
+  const redrawOpts = {};
+  
+  // Support nested redraw object: { redraw: { enabled: false, diffThreshold: 0.5 } }
+  if (options.redraw && typeof options.redraw === 'object') {
+    return options.redraw;
+  }
+  
+  // Support flat options for convenience
+  if ('redrawEnabled' in options) redrawOpts.enabled = options.redrawEnabled;
+  if ('redrawScreenRedraw' in options) redrawOpts.screenRedraw = options.redrawScreenRedraw;
+  if ('redrawNetworkMonitor' in options) redrawOpts.networkMonitor = options.redrawNetworkMonitor;
+  if ('redrawDiffThreshold' in options) redrawOpts.diffThreshold = options.redrawDiffThreshold;
+  
+  return redrawOpts;
+};
+
 // Factory function that creates commands with the provided emitter
 const createCommands = (
   emitter,
@@ -44,12 +81,16 @@ const createCommands = (
   config,
   sessionInstance,
   getCurrentFilePath,
-  redrawThreshold = 0.1,
+  redrawThreshold = 0.01,
+  getDashcamElapsedTime = null,
 ) => {
   // Create SDK instance with emitter, config, and session
   const sdk = createSDK(emitter, config, sessionInstance);
-  // Create redraw instance with the system
-  const redraw = createRedraw(emitter, system, sandbox, redrawThreshold);
+  // Create redraw instance with the system - support both number and object for backward compatibility
+  const defaultRedrawOptions = typeof redrawThreshold === 'number' 
+    ? { diffThreshold: redrawThreshold }
+    : redrawThreshold;
+  const redraw = createRedraw(emitter, system, sandbox, defaultRedrawOptions);
 
   // Helper method to resolve file paths relative to the current file
   const resolveRelativePath = (relativePath) => {
@@ -198,19 +239,59 @@ const createCommands = (
 
     emitter.emit(events.log.narration, `thinking...`);
 
+    const assertStartTime = Date.now();
     let response = await sdk.req("assert", {
       expect: assertion,
       image: await system.captureScreenBase64(),
     });
+    const assertDuration = Date.now() - assertStartTime;
+    
+    // Determine if assertion passed or failed
+    const assertionPassed = response.data.indexOf("The task passed") > -1;
+    
+    // Track interaction with success/failure
+    const sessionId = sessionInstance?.get();
+    if (sessionId) {
+      try {
+        await sandbox.send({
+          type: "trackInteraction",
+          interactionType: "assert",
+          session: sessionId,
+          prompt: assertion,
+          timestamp: assertStartTime,
+          duration: assertDuration,
+          success: assertionPassed,
+          error: assertionPassed ? undefined : response.data,
+        });
+      } catch (err) {
+        console.warn("Failed to track assert interaction:", err.message);
+      }
+    }
+    
     return handleAssertResponse(response.data);
   };
-  const scroll = async (direction = "down", amount = 300) => {
+
+  /**
+   * Scroll the screen in a direction
+   * @param {string} [direction='down'] - Direction to scroll ('up', 'down', 'left', 'right')
+   * @param {Object} [options] - Additional options
+   * @param {number} [options.amount=300] - Amount to scroll in pixels
+   * @param {Object} [options.redraw] - Redraw detection options
+   * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+   * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+   * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+   * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+   */
+  const scroll = async (direction = 'down', options = {}) => {
+    let { amount = 300 } = options;
+    const redrawOptions = extractRedrawOptions(options);
+    
     emitter.emit(
       events.log.narration,
       theme.dim(`scrolling ${direction} ${amount}px...`),
     );
 
-    await redraw.start();
+    await redraw.start(redrawOptions);
 
     amount = parseInt(amount, 10);
 
@@ -222,7 +303,7 @@ const createCommands = (
           amount,
           direction,
         });
-        await redraw.wait(2500);
+        await redraw.wait(2500, redrawOptions);
         break;
       case "down":
         await sandbox.send({
@@ -230,7 +311,7 @@ const createCommands = (
           amount,
           direction,
         });
-        await redraw.wait(2500);
+        await redraw.wait(2500, redrawOptions);
         break;
       case "left":
         console.error("Not Supported");
@@ -251,91 +332,284 @@ const createCommands = (
     }
   };
 
-  // perform a mouse click
-  // click, right-click, double-click, hover
-  const click = async (x, y, action = "click") => {
-    await redraw.start();
+  /**
+   * Perform a mouse click action
+   * @param {Object|number} options - Options object or x coordinate (for backward compatibility)
+   * @param {number} options.x - X coordinate
+   * @param {number} options.y - Y coordinate
+   * @param {string} [options.action='click'] - Click action ('click', 'right-click', 'double-click', 'hover', 'mouseDown', 'mouseUp')
+   * @param {string} [options.prompt] - Prompt for tracking
+   * @param {boolean} [options.cacheHit] - Whether cache was hit
+   * @param {string} [options.selector] - Selector used
+   * @param {boolean} [options.selectorUsed] - Whether selector was used
+   * @param {Object} [options.redraw] - Redraw detection options
+   * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+   * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+   * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+   * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+   */
+  const click = async (...args) => {
+    const clickStartTime = Date.now();
+    let x, y, action, elementData, redrawOptions;
+    
+    // Handle both object and positional argument styles
+    if (isObjectArgs(args, ['x', 'y', 'action', 'prompt', 'cacheHit', 'selector'])) {
+      const { x: xPos, y: yPos, action: actionArg = 'click', redraw: redrawOpts, ...rest } = args[0];
+      x = xPos;
+      y = yPos;
+      action = actionArg;
+      elementData = rest;
+      redrawOptions = extractRedrawOptions({ redraw: redrawOpts, ...rest });
+    } else {
+      // Legacy positional: click(x, y, action, elementData)
+      [x, y, action = 'click', elementData = {}] = args;
+      redrawOptions = extractRedrawOptions(elementData);
+    }
+    
+    try {
+      await redraw.start(redrawOptions);
 
-    let button = "left";
-    let double = false;
+      let button = "left";
+      let double = false;
 
-    if (action === "right-click") {
-      button = "right";
-    }
-    if (action === "double-click") {
-      double = true;
-    }
+      if (action === "right-click") {
+        button = "right";
+      }
+      if (action === "double-click") {
+        double = true;
+      }
 
-    emitter.emit(
-      events.log.narration,
-      theme.dim(`${action} ${button} clicking at ${x}, ${y}...`),
-      true,
-    );
+      emitter.emit(
+        events.log.narration,
+        theme.dim(`${action} ${button} clicking at ${x}, ${y}...`),
+        true,
+      );
 
-    x = parseInt(x);
-    y = parseInt(y);
+      x = parseInt(x);
+      y = parseInt(y);
 
-    await sandbox.send({ type: "moveMouse", x, y });
+      // Add dashcam timestamp if available
+      if (getDashcamElapsedTime) {
+        const elapsed = getDashcamElapsedTime();
+        if (elapsed !== null) {
+          elementData.timestamp = elapsed;
+        }
+      }
 
-    emitter.emit(events.mouseMove, { x, y });
+      await sandbox.send({ type: "moveMouse", x, y, ...elementData });
 
-    await delay(2500); // wait for the mouse to move
+      emitter.emit(events.mouseMove, { x, y });
 
-    if (action !== "hover") {
-      if (action === "click" || action === "left-click") {
-        await sandbox.send({ type: "leftClick" });
-      } else if (action === "right-click") {
-        await sandbox.send({ type: "rightClick" });
-      } else if (action === "middle-click") {
-        await sandbox.send({ type: "middleClick" });
-      } else if (action === "double-click") {
-        await sandbox.send({ type: "doubleClick" });
-      } else if (action === "mouseDown") {
-        await sandbox.send({ type: "mousePress", button: "left" });
-      } else if (action === "mouseUp") {
-        await sandbox.send({
-          type: "mouseRelease",
-          button: "left",
-        });
+      await delay(2500); // wait for the mouse to move
+
+      if (action !== "hover") {
+        // Update timestamp for the actual click action
+        if (getDashcamElapsedTime) {
+          const elapsed = getDashcamElapsedTime();
+          if (elapsed !== null) {
+            elementData.timestamp = elapsed;
+          }
+        }
+
+        if (action === "click" || action === "left-click") {
+          await sandbox.send({ type: "leftClick", x, y, ...elementData });
+        } else if (action === "right-click") {
+          await sandbox.send({ type: "rightClick", x, y, ...elementData });
+        } else if (action === "middle-click") {
+          await sandbox.send({ type: "middleClick", x, y, ...elementData });
+        } else if (action === "double-click") {
+          await sandbox.send({ type: "doubleClick", x, y, ...elementData });
+        } else if (action === "mouseDown") {
+          await sandbox.send({ type: "mousePress", button: "left", x, y, ...elementData });
+        } else if (action === "mouseUp") {
+          await sandbox.send({
+            type: "mouseRelease",
+            button: "left",
+            x,
+            y,
+            ...elementData
+          });
+        }
+
+        emitter.emit(events.mouseClick, { x, y, button, click, double });
+        
+        // Track interaction
+        const sessionId = sessionInstance?.get();
+        if (sessionId && elementData.prompt) {
+          try {
+            const clickDuration = Date.now() - clickStartTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "click",
+              session: sessionId,
+              prompt: elementData.prompt,
+              input: { x, y, action },
+              timestamp: clickStartTime,
+              duration: clickDuration,
+              success: true,
+              cacheHit: elementData.cacheHit,
+              selector: elementData.selector,
+              selectorUsed: elementData.selectorUsed,
+            });
+          } catch (err) {
+            console.warn("Failed to track click interaction:", err.message);
+          }
+        }
       }
 
-      emitter.emit(events.mouseClick, { x, y, button, click, double });
+      await redraw.wait(5000, redrawOptions);
+
+      return;
+    } catch (error) {
+      // Track interaction failure
+      const sessionId = sessionInstance?.get();
+      if (sessionId && elementData.prompt) {
+        try {
+          const clickDuration = Date.now() - clickStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "click",
+            session: sessionId,
+            prompt: elementData.prompt,
+            input: { x, y, action },
+            timestamp: clickStartTime,
+            duration: clickDuration,
+            success: false,
+            error: error.message,
+            cacheHit: elementData.cacheHit,
+            selector: elementData.selector,
+            selectorUsed: elementData.selectorUsed,
+          });
+        } catch (err) {
+          console.warn("Failed to track click interaction:", err.message);
+        }
+      }
+      throw error;
     }
+  };
 
-    await redraw.wait(5000);
+  /**
+   * Hover at coordinates
+   * @param {Object|number} options - Options object or x coordinate (for backward compatibility)
+   * @param {number} options.x - X coordinate
+   * @param {number} options.y - Y coordinate
+   * @param {string} [options.prompt] - Prompt for tracking
+   * @param {boolean} [options.cacheHit] - Whether cache was hit
+   * @param {string} [options.selector] - Selector used
+   * @param {boolean} [options.selectorUsed] - Whether selector was used
+   */
+  const hover = async (...args) => {
+    const hoverStartTime = Date.now();
+    let x, y, elementData, redrawOptions;
+    
+    // Handle both object and positional argument styles
+    if (isObjectArgs(args, ['x', 'y', 'prompt', 'cacheHit', 'selector'])) {
+      const { x: xPos, y: yPos, redraw: redrawOpts, ...rest } = args[0];
+      x = xPos;
+      y = yPos;
+      elementData = rest;
+      redrawOptions = extractRedrawOptions({ redraw: redrawOpts, ...rest });
+    } else {
+      // Legacy positional: hover(x, y, elementData)
+      [x, y, elementData = {}] = args;
+      redrawOptions = extractRedrawOptions(elementData);
+    }
+    
+    try {
+      emitter.emit(events.log.narration, theme.dim(`hovering at ${x}, ${y}...`));
 
-    return;
-  };
+      await redraw.start(redrawOptions);
 
-  const hover = async (x, y) => {
-    emitter.emit(events.log.narration, theme.dim(`hovering at ${x}, ${y}...`));
+      x = parseInt(x);
+      y = parseInt(y);
 
-    await redraw.start();
+      // Add dashcam timestamp if available
+      if (getDashcamElapsedTime) {
+        const elapsed = getDashcamElapsedTime();
+        if (elapsed !== null) {
+          elementData.timestamp = elapsed;
+        }
+      }
 
-    x = parseInt(x);
-    y = parseInt(y);
+      await sandbox.send({ type: "moveMouse", x, y, ...elementData });
 
-    await sandbox.send({ type: "moveMouse", x, y });
+      // Track interaction
+      const sessionId = sessionInstance?.get();
+      if (sessionId && elementData.prompt) {
+        try {
+          const hoverDuration = Date.now() - hoverStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "hover",
+            session: sessionId,
+            prompt: elementData.prompt,
+            input: { x, y },
+            timestamp: hoverStartTime,
+            duration: hoverDuration,
+            success: true,
+            cacheHit: elementData.cacheHit,
+            selector: elementData.selector,
+            selectorUsed: elementData.selectorUsed,
+          });
+        } catch (err) {
+          console.warn("Failed to track hover interaction:", err.message);
+        }
+      }
 
-    await redraw.wait(2500);
+      await redraw.wait(2500, redrawOptions);
 
-    return;
+      return;
+    } catch (error) {
+      // Track interaction failure
+      const sessionId = sessionInstance?.get();
+      if (sessionId && elementData.prompt) {
+        try {
+          const hoverDuration = Date.now() - hoverStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "hover",
+            session: sessionId,
+            prompt: elementData.prompt,
+            input: { x, y },
+            timestamp: hoverStartTime,
+            duration: hoverDuration,
+            success: false,
+            error: error.message,
+            cacheHit: elementData.cacheHit,
+            selector: elementData.selector,
+            selectorUsed: elementData.selectorUsed,
+          });
+        } catch (err) {
+          console.warn("Failed to track hover interaction:", err.message);
+        }
+      }
+      throw error;
+    }
   };
 
   let commands = {
     scroll: scroll,
     click: click,
     hover: hover,
-    // method, levenshein, dice, or combined
-    // leven = this is turbo, all around good for text similarity
-    // dice = this is good for short strings, but not as good for long strings
-    // turbo (default) = turbo of both, with a 2x preference for levenshtein
-    "hover-text": async (
-      text,
-      description = null,
-      action = "click",
-      timeout = 5000, // we pass this to the subsequent wait-for-text block
-    ) => {
+    /**
+     * Hover over text on screen
+     * @param {Object|string} options - Options object or text (for backward compatibility)
+     * @param {string} options.text - Text to find and hover over
+     * @param {string|null} [options.description] - Optional description of the element
+     * @param {string} [options.action='click'] - Action to perform
+     * @param {number} [options.timeout=5000] - Timeout in milliseconds
+     */
+    "hover-text": async (...args) => {
+      let text, description, action, timeout;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['text', 'description', 'action', 'timeout'])) {
+        ({ text, description = null, action = 'click', timeout = 5000 } = args[0]);
+      } else {
+        // Legacy positional: hoverText(text, description, action, timeout)
+        [text, description = null, action = 'click', timeout = 5000] = args;
+      }
+      
       emitter.emit(
         events.log.narration,
         theme.dim(
@@ -346,7 +620,7 @@ const createCommands = (
       text = text ? text.toString() : null;
 
       // wait for the text to appear on screen
-      await commands["wait-for-text"](text, timeout);
+      await commands["wait-for-text"]({ text, timeout });
 
       description = description ? description.toString() : null;
 
@@ -369,15 +643,30 @@ const createCommands = (
 
       // Perform the action using the located coordinates
       if (action === "hover") {
-        await commands.hover(response.coordinates.x, response.coordinates.y);
+        await commands.hover({ x: response.coordinates.x, y: response.coordinates.y });
       } else {
-        await click(response.coordinates.x, response.coordinates.y, action);
+        await click({ x: response.coordinates.x, y: response.coordinates.y, action });
       }
 
       return response;
     },
-    // uses our api to find all images on screen
-    "hover-image": async (description, action = "click") => {
+    /**
+     * Hover over an image on screen
+     * @param {Object|string} options - Options object or description (for backward compatibility)
+     * @param {string} options.description - Description of the image to find
+     * @param {string} [options.action='click'] - Action to perform
+     */
+    "hover-image": async (...args) => {
+      let description, action;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['description', 'action'])) {
+        ({ description, action = 'click' } = args[0]);
+      } else {
+        // Legacy positional: hoverImage(description, action)
+        [description, action = 'click'] = args;
+      }
+      
       emitter.emit(
         events.log.narration,
         theme.dim(`searching for image: "${description}"...`),
@@ -394,14 +683,31 @@ const createCommands = (
 
       // Perform the action using the located coordinates
       if (action === "hover") {
-        await commands.hover(response.coordinates.x, response.coordinates.y);
+        await commands.hover({ x: response.coordinates.x, y: response.coordinates.y });
       } else {
-        await click(response.coordinates.x, response.coordinates.y, action);
+        await click({ x: response.coordinates.x, y: response.coordinates.y, action });
       }
 
       return response;
     },
-    "match-image": async (relativePath, action = "click", invert = false) => {
+    /**
+     * Match and interact with an image template
+     * @param {Object|string} options - Options object or path (for backward compatibility)
+     * @param {string} options.path - Path to the image template
+     * @param {string} [options.action='click'] - Action to perform
+     * @param {boolean} [options.invert=false] - Invert the match
+     */
+    "match-image": async (...args) => {
+      let relativePath, action, invert;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['path', 'action', 'invert'])) {
+        ({ path: relativePath, action = 'click', invert = false } = args[0]);
+      } else {
+        // Legacy positional: matchImage(relativePath, action, invert)
+        [relativePath, action = 'click', invert = false] = args;
+      }
+      
       emitter.emit(
         events.log.narration,
         theme.dim(`${action} on image template "${relativePath}"...`),
@@ -422,52 +728,173 @@ const createCommands = (
         throw new CommandError(`Image not found: ${resolvedPath}`);
       } else {
         if (action === "click") {
-          await click(result.centerX, result.centerY, action);
+          await click({ x: result.centerX, y: result.centerY, action });
         } else if (action === "hover") {
-          await hover(result.centerX, result.centerY);
+          await hover({ x: result.centerX, y: result.centerY });
         }
       }
 
       return true;
     },
-    // type a string
+    /**
+     * Type text
+     * @param {string|number} text - Text to type
+     * @param {Object} [options] - Additional options
+     * @param {number} [options.delay=250] - Delay between keystrokes in milliseconds
+     * @param {boolean} [options.secret=false] - If true, text is treated as sensitive (not logged or stored)
+     * @param {Object} [options.redraw] - Redraw detection options
+     * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+     * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+     * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+     * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+     */
+    "type": async (text, options = {}) => {
+      const typeStartTime = Date.now();
+      const { delay = 250, secret = false, redraw: redrawOpts, ...elementData } = options;
+      const redrawOptions = extractRedrawOptions({ redraw: redrawOpts, ...options });
+      
+      // Log masked version if secret, otherwise show actual text
+      if (secret) {
+        emitter.emit(events.log.narration, theme.dim(`typing secret "****"...`));
+      } else {
+        emitter.emit(events.log.narration, theme.dim(`typing "${text}"...`));
+      }
 
-    type: async (string, delay = 250) => {
-      emitter.emit(events.log.narration, theme.dim(`typing "${string}"...`));
+      await redraw.start(redrawOptions);
 
-      await redraw.start();
+      text = text.toString();
 
-      string = string.toString();
+      // Add dashcam timestamp if available
+      if (getDashcamElapsedTime) {
+        const elapsed = getDashcamElapsedTime();
+        if (elapsed !== null) {
+          elementData.timestamp = elapsed;
+        }
+      }
 
-      await sandbox.send({ type: "write", text: string, delay });
-      await redraw.wait(5000);
+      // Actually type the text in the sandbox
+      await sandbox.send({ type: "write", text, delay, ...elementData });
+      
+      // Track interaction
+      const sessionId = sessionInstance?.get();
+      if (sessionId) {
+        try {
+          const typeDuration = Date.now() - typeStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "type",
+            session: sessionId,
+            // Store masked text if secret, otherwise store actual text
+            input: { text: secret ? "****" : text, delay },
+            timestamp: typeStartTime,
+            duration: typeDuration,
+            success: true,
+            isSecret: secret, // Flag this interaction if it contains a secret
+          });
+        } catch (err) {
+          console.warn("Failed to track type interaction:", err.message);
+        }
+      }
+      
+      await redraw.wait(5000, redrawOptions);
       return;
     },
-    // press keys
-    // different than `type`, becasue it can press multiple keys at once
-    "press-keys": async (inputKeys) => {
+    /**
+     * Press keyboard keys
+     * @param {Array} keys - Array of keys to press
+     * @param {Object} [options] - Additional options
+     * @param {Object} [options.redraw] - Redraw detection options
+     * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+     * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+     * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+     * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+     */
+    "press-keys": async (keys, options = {}) => {
+      const pressKeysStartTime = Date.now();
+      const redrawOptions = extractRedrawOptions(options);
       emitter.emit(
         events.log.narration,
         theme.dim(
-          `pressing keys: ${Array.isArray(inputKeys) ? inputKeys.join(", ") : inputKeys}...`,
+          `pressing keys: ${Array.isArray(keys) ? keys.join(", ") : keys}...`,
         ),
       );
 
-      await redraw.start();
+      await redraw.start(redrawOptions);
 
       // finally, press the keys
-      await sandbox.send({ type: "press", keys: inputKeys });
+      await sandbox.send({ type: "press", keys });
+      
+      // Track interaction
+      const sessionId = sessionInstance?.get();
+      if (sessionId) {
+        try {
+          const pressKeysDuration = Date.now() - pressKeysStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "pressKeys",
+            session: sessionId,
+            input: { keys },
+            timestamp: pressKeysStartTime,
+            duration: pressKeysDuration,
+            success: true,
+          });
+        } catch (err) {
+          console.warn("Failed to track pressKeys interaction:", err.message);
+        }
+      }
 
-      await redraw.wait(5000);
+      await redraw.wait(5000, redrawOptions);
 
       return;
     },
-    // simple delay, usually to let ui render or webpage to load
-    wait: async (timeout = 3000) => {
+    /**
+     * Wait for specified time
+     * @param {number} [timeout=3000] - Time to wait in milliseconds
+     * @param {Object} [options] - Additional options (reserved for future use)
+     */
+    "wait": async (timeout = 3000, options = {}) => {
+      const waitStartTime = Date.now();
       emitter.emit(events.log.narration, theme.dim(`waiting ${timeout}ms...`));
-      return await delay(timeout);
+      const result = await delay(timeout);
+      
+      // Track interaction
+      const sessionId = sessionInstance?.get();
+      if (sessionId) {
+        try {
+          const waitDuration = Date.now() - waitStartTime;
+          await sandbox.send({
+            type: "trackInteraction",
+            interactionType: "wait",
+            session: sessionId,
+            input: { timeout },
+            timestamp: waitStartTime,
+            duration: waitDuration,
+            success: true,
+          });
+        } catch (err) {
+          console.warn("Failed to track wait interaction:", err.message);
+        }
+      }
+      
+      return result;
     },
-    "wait-for-image": async (description, timeout = 10000) => {
+    /**
+     * Wait for image to appear on screen
+     * @param {Object|string} options - Options object or description (for backward compatibility)
+     * @param {string} options.description - Description of the image
+     * @param {number} [options.timeout=10000] - Timeout in milliseconds
+     */
+    "wait-for-image": async (...args) => {
+      let description, timeout;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['description', 'timeout'])) {
+        ({ description, timeout = 10000 } = args[0]);
+      } else {
+        // Legacy positional: waitForImage(description, timeout)
+        [description, timeout = 10000] = args;
+      }
+      
       emitter.emit(
         events.log.narration,
         theme.dim(
@@ -507,15 +934,80 @@ const createCommands = (
           ),
           true,
         );
+        
+        // Track interaction success
+        const sessionId = sessionInstance?.get();
+        if (sessionId) {
+          try {
+            const waitForImageDuration = Date.now() - startTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "waitForImage",
+              session: sessionId,
+              prompt: description,
+              input: { timeout },
+              timestamp: startTime,
+              duration: waitForImageDuration,
+              success: true,
+            });
+          } catch (err) {
+            console.warn("Failed to track waitForImage interaction:", err.message);
+          }
+        }
+        
         return;
       } else {
-        throw new MatchError(
-          `Timed out (${niceSeconds(timeout)} seconds) while searching for an image matching the description "${description}"`,
-        );
+        // Track interaction failure
+        const sessionId = sessionInstance?.get();
+        const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for an image matching the description "${description}"`;
+        if (sessionId) {
+          try {
+            const waitForImageDuration = Date.now() - startTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "waitForImage",
+              session: sessionId,
+              prompt: description,
+              input: { timeout },
+              timestamp: startTime,
+              duration: waitForImageDuration,
+              success: false,
+              error: errorMsg,
+            });
+          } catch (err) {
+            console.warn("Failed to track waitForImage interaction:", err.message);
+          }
+        }
+        
+        throw new MatchError(errorMsg);
       }
     },
-    "wait-for-text": async (text, timeout = 5000) => {
-      await redraw.start();
+    /**
+     * Wait for text to appear on screen
+     * @param {Object|string} options - Options object or text (for backward compatibility)
+     * @param {string} options.text - Text to wait for
+     * @param {number} [options.timeout=5000] - Timeout in milliseconds
+     * @param {Object} [options.redraw] - Redraw detection options
+     * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+     * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+     * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+     * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+     */
+    "wait-for-text": async (...args) => {
+      let text, timeout, redrawOptions;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['text', 'timeout'])) {
+        const { redraw: redrawOpts, ...rest } = args[0];
+        ({ text, timeout = 5000 } = rest);
+        redrawOptions = extractRedrawOptions({ redraw: redrawOpts, ...rest });
+      } else {
+        // Legacy positional: waitForText(text, timeout)
+        [text, timeout = 5000] = args;
+        redrawOptions = {};
+      }
+      
+      await redraw.start(redrawOptions);
 
       emitter.emit(
         events.log.narration,
@@ -552,20 +1044,82 @@ const createCommands = (
 
       if (passed) {
         emitter.emit(events.log.narration, theme.dim(`"${text}" found!`), true);
+        
+        // Track interaction success
+        const sessionId = sessionInstance?.get();
+        if (sessionId) {
+          try {
+            const waitForTextDuration = Date.now() - startTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "waitForText",
+              session: sessionId,
+              prompt: text,
+              input: { timeout },
+              timestamp: startTime,
+              duration: waitForTextDuration,
+              success: true,
+            });
+          } catch (err) {
+            console.warn("Failed to track waitForText interaction:", err.message);
+          }
+        }
+        
         return;
       } else {
-        throw new MatchError(
-          `Timed out (${niceSeconds(timeout)} seconds) while searching for "${text}"`,
-        );
+        // Track interaction failure
+        const sessionId = sessionInstance?.get();
+        const errorMsg = `Timed out (${niceSeconds(timeout)} seconds) while searching for "${text}"`;
+        if (sessionId) {
+          try {
+            const waitForTextDuration = Date.now() - startTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "waitForText",
+              session: sessionId,
+              prompt: text,
+              input: { timeout },
+              timestamp: startTime,
+              duration: waitForTextDuration,
+              success: false,
+              error: errorMsg,
+            });
+          } catch (err) {
+            console.warn("Failed to track waitForText interaction:", err.message);
+          }
+        }
+        
+        throw new MatchError(errorMsg);
       }
     },
-    "scroll-until-text": async (
-      text,
-      direction = "down",
-      maxDistance = 10000,
-      invert = false,
-    ) => {
-      await redraw.start();
+    /**
+     * Scroll until text is found
+     * @param {Object|string} options - Options object or text (for backward compatibility)
+     * @param {string} options.text - Text to find
+     * @param {string} [options.direction='down'] - Scroll direction
+     * @param {number} [options.maxDistance=10000] - Maximum distance to scroll in pixels
+     * @param {boolean} [options.invert=false] - Invert the match
+     * @param {Object} [options.redraw] - Redraw detection options
+     * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+     * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+     * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+     * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+     */
+    "scroll-until-text": async (...args) => {
+      let text, direction, maxDistance, invert, redrawOptions;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['text', 'direction', 'maxDistance', 'invert'])) {
+        const { redraw: redrawOpts, ...rest } = args[0];
+        ({ text, direction = 'down', maxDistance = 10000, invert = false } = rest);
+        redrawOptions = extractRedrawOptions({ redraw: redrawOpts, ...rest });
+      } else {
+        // Legacy positional: scrollUntilText(text, direction, maxDistance, invert)
+        [text, direction = 'down', maxDistance = 10000, invert = false] = args;
+        redrawOptions = {};
+      }
+      
+      await redraw.start(redrawOptions);
 
       emitter.emit(
         events.log.narration,
@@ -597,7 +1151,7 @@ const createCommands = (
             ),
             true,
           );
-          await scroll(direction, incrementDistance);
+          await scroll({ direction, amount: incrementDistance });
           scrollDistance = scrollDistance + incrementDistance;
         }
       }
@@ -611,21 +1165,34 @@ const createCommands = (
         );
       }
     },
-    "scroll-until-image": async (
-      description,
-      direction = "down",
-      maxDistance = 10000,
-      method = "mouse",
-      path,
-      invert = false,
-    ) => {
-      const needle = description || path;
+    /**
+     * Scroll until image is found
+     * @param {Object|string} options - Options object or description (for backward compatibility)
+     * @param {string} [options.description] - Description of the image
+     * @param {string} [options.direction='down'] - Scroll direction
+     * @param {number} [options.maxDistance=10000] - Maximum distance to scroll in pixels
+     * @param {string} [options.method='mouse'] - Scroll method
+     * @param {string} [options.path] - Path to image template
+     * @param {boolean} [options.invert=false] - Invert the match
+     */
+    "scroll-until-image": async (...args) => {
+      let description, direction, maxDistance, method, imagePath, invert;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['description', 'direction', 'maxDistance', 'method', 'path', 'invert'])) {
+        ({ description, direction = 'down', maxDistance = 10000, method = 'mouse', path: imagePath, invert = false } = args[0]);
+      } else {
+        // Legacy positional: scrollUntilImage(description, direction, maxDistance, method, path, invert)
+        [description, direction = 'down', maxDistance = 10000, method = 'mouse', imagePath, invert = false] = args;
+      }
+      
+      const needle = description || imagePath;
 
       if (!needle) {
         throw new CommandError("No description or path provided");
       }
 
-      if (description && path) {
+      if (description && imagePath) {
         throw new CommandError(
           "Only one of description or path can be provided",
         );
@@ -651,9 +1218,9 @@ const createCommands = (
           );
         }
 
-        if (path) {
+        if (imagePath) {
           // Don't throw if not found. We only want to know if it's found or not.
-          passed = await commands["match-image"](path, null).catch(
+          passed = await commands["match-image"]({ path: imagePath }).catch(
             console.warn,
           );
         }
@@ -664,7 +1231,7 @@ const createCommands = (
             theme.dim(`scrolling ${direction} ${incrementDistance} pixels...`),
             true,
           );
-          await scroll(direction, incrementDistance, method);
+          await scroll({ direction, amount: incrementDistance });
           scrollDistance = scrollDistance + incrementDistance;
         }
       }
@@ -682,30 +1249,122 @@ const createCommands = (
         );
       }
     },
-    // run applescript to focus an application by name
-    "focus-application": async (name) => {
-      await redraw.start();
+    /**
+     * Focus an application by name
+     * @param {string} name - Application name
+     * @param {Object} [options] - Additional options
+     * @param {Object} [options.redraw] - Redraw detection options
+     * @param {boolean} [options.redraw.enabled=true] - Enable/disable redraw detection
+     * @param {boolean} [options.redraw.screenRedraw=true] - Enable/disable screen redraw detection
+     * @param {boolean} [options.redraw.networkMonitor=true] - Enable/disable network monitoring
+     * @param {number} [options.redraw.diffThreshold=0.1] - Screen diff threshold percentage
+     */
+    "focus-application": async (name, options = {}) => {
+      const redrawOptions = extractRedrawOptions(options);
+      await redraw.start(redrawOptions);
 
       await sandbox.send({
         type: "commands.focus-application",
         name,
       });
-      await redraw.wait(1000);
+      await redraw.wait(1000, redrawOptions);
       return "The application was focused.";
     },
-    remember: async (description) => {
-      let result = await sdk.req("remember", {
-        image: await system.captureScreenBase64(),
-        description,
-      });
-      return result.data;
+    /**
+     * Extract and remember information from the screen using AI
+     * @param {Object|string} options - Options object or description (for backward compatibility)
+     * @param {string} options.description - What to remember
+     */
+    "remember": async (...args) => {
+      const rememberStartTime = Date.now();
+      let description;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['description'])) {
+        ({ description } = args[0]);
+      } else {
+        // Legacy positional: remember(description)
+        [description] = args;
+      }
+      
+      try {
+        let result = await sdk.req("remember", {
+          image: await system.captureScreenBase64(),
+          description,
+        });
+        
+        // Track interaction success
+        const sessionId = sessionInstance?.get();
+        if (sessionId) {
+          try {
+            const rememberDuration = Date.now() - rememberStartTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "remember",
+              session: sessionId,
+              prompt: description,
+              timestamp: rememberStartTime,
+              duration: rememberDuration,
+              success: true,
+            });
+          } catch (err) {
+            console.warn("Failed to track remember interaction:", err.message);
+          }
+        }
+        
+        return result.data;
+      } catch (error) {
+        // Track interaction failure
+        const sessionId = sessionInstance?.get();
+        if (sessionId) {
+          try {
+            const rememberDuration = Date.now() - rememberStartTime;
+            await sandbox.send({
+              type: "trackInteraction",
+              interactionType: "remember",
+              session: sessionId,
+              prompt: description,
+              timestamp: rememberStartTime,
+              duration: rememberDuration,
+              success: false,
+              error: error.message,
+            });
+          } catch (err) {
+            console.warn("Failed to track remember interaction:", err.message);
+          }
+        }
+        throw error;
+      }
     },
-    assert: async (assertion) => {
+    /**
+     * Make an AI-powered assertion
+     * @param {string} assertion - Assertion to check
+     * @param {Object} [options] - Additional options (reserved for future use)
+     */
+    "assert": async (assertion, options = {}) => {
       let response = await assert(assertion, true);
 
       return response;
     },
-    exec: async (language = "pwsh", code, timeout, silent = false) => {
+    /**
+     * Execute code in the sandbox
+     * @param {Object|string} options - Options object or language (for backward compatibility)
+     * @param {string} [options.language='pwsh'] - Language ('js', 'pwsh', or 'sh')
+     * @param {string} options.code - Code to execute
+     * @param {number} [options.timeout] - Timeout in milliseconds
+     * @param {boolean} [options.silent=false] - Suppress output
+     */
+    "exec": async (...args) => {
+      let language, code, timeout, silent;
+      
+      // Handle both object and positional argument styles
+      if (isObjectArgs(args, ['language', 'code', 'timeout', 'silent'])) {
+        ({ language = 'pwsh', code, timeout, silent = false } = args[0]);
+      } else {
+        // Legacy positional: exec(language, code, timeout, silent)
+        [language = 'pwsh', code, timeout, silent = false] = args;
+      }
+      
       emitter.emit(events.log.narration, theme.dim(`calling exec...`), true);
 
       emitter.emit(events.log.log, code);