testdriverai 7.3.11 → 7.3.13

package/lib/init-project.js

@@ -1,6 +1,96 @@
 const fs = require("fs");
 const path = require("path");
-const { execSync } = require("child_process");
+const { execSync, spawn } = require("child_process");
+
+/**
+ * Run an npm install command with an animated progress bar
+ * @param {string} cmd - The command to run (e.g. "npm")
+ * @param {string[]} args - Command arguments
+ * @param {string} cwd - Working directory
+ * @param {string} label - Label to show (e.g. "vitest testdriverai")
+ * @returns {Promise<void>}
+ */
+function runInstall(cmd, args, cwd, label) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      shell: process.platform === "win32",
+    });
+
+    const spinnerFrames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+    const barWidth = 20;
+    let frame = 0;
+    let status = "resolving";
+    let filled = 0;
+
+    // Parse npm stderr for progress hints
+    const handleData = (data) => {
+      const text = data.toString();
+      if (text.includes("idealTree")) {
+        status = "resolving packages";
+        filled = Math.max(filled, 3);
+      } else if (text.includes("reify:")) {
+        status = "installing";
+        filled = Math.max(filled, 8);
+        // Try to extract package name from reify output
+        const match = text.match(/reify:([^\s:]+)/);
+        if (match) {
+          status = `installing ${match[1]}`;
+        }
+      } else if (text.includes("timing")) {
+        filled = Math.max(filled, 14);
+        status = "finalizing";
+      } else if (text.includes("added")) {
+        filled = barWidth;
+        status = "done";
+      }
+      // Slowly increment to show activity
+      if (filled < barWidth - 2) {
+        filled = Math.min(filled + 1, barWidth - 2);
+      }
+    };
+
+    child.stdout.on("data", handleData);
+    child.stderr.on("data", handleData);
+
+    const isTTY = process.stderr.isTTY;
+
+    const interval = setInterval(() => {
+      frame = (frame + 1) % spinnerFrames.length;
+      const spinner = spinnerFrames[frame];
+      const bar = "█".repeat(filled) + "░".repeat(barWidth - filled);
+      const line = `  ${spinner} ${label} [${bar}] ${status}`;
+      if (isTTY) {
+        process.stderr.clearLine(0);
+        process.stderr.cursorTo(0);
+        process.stderr.write(line);
+      }
+    }, 80);
+
+    child.on("close", (code) => {
+      clearInterval(interval);
+      if (isTTY) {
+        process.stderr.clearLine(0);
+        process.stderr.cursorTo(0);
+      }
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`${cmd} ${args.join(" ")} exited with code ${code}`));
+      }
+    });
+
+    child.on("error", (err) => {
+      clearInterval(interval);
+      if (isTTY) {
+        process.stderr.clearLine(0);
+        process.stderr.cursorTo(0);
+      }
+      reject(err);
+    });
+  });
+}
 
 /**
  * Initialize a TestDriver project with all necessary files and configuration
@@ -378,6 +468,16 @@ jobs:
       if (copiedCount > 0) {
         progress(`✓ Copied ${copiedCount} agent(s) to .github/agents/`);
       }
+
+      // Also set testdriver.md as copilot-instructions.md if it doesn't already exist
+      const copilotInstructionsPath = path.join(targetDir, ".github", "copilot-instructions.md");
+      const testdriverAgentSource = path.join(agentsSourceDir, "testdriver.md");
+      if (!fs.existsSync(copilotInstructionsPath) && fs.existsSync(testdriverAgentSource)) {
+        fs.copyFileSync(testdriverAgentSource, copilotInstructionsPath);
+        progress("✓ Created .github/copilot-instructions.md");
+      } else if (fs.existsSync(copilotInstructionsPath)) {
+        progress("⊘ copilot-instructions.md already exists, skipping");
+      }
     } else {
       progress("⚠ Agents directory not found (will be available after npm install)");
     }
@@ -409,11 +509,10 @@ jobs:
     if (!options.skipInstall) {
       progress("\n📦 Installing dependencies...");
       try {
-        execSync("npm install -D vitest testdriverai && npm install dotenv", {
-          cwd: targetDir,
-          stdio: "pipe",
-        });
-        progress("✓ Dependencies installed successfully");
+        await runInstall("npm", ["install", "-D", "vitest", "testdriverai"], targetDir, "vitest testdriverai");
+        progress("✓ Installed vitest, testdriverai");
+        await runInstall("npm", ["install", "dotenv"], targetDir, "dotenv");
+        progress("✓ Installed dotenv");
       } catch (error) {
         errors.push("Failed to install dependencies. Run manually:");
         errors.push("  npm install -D vitest testdriverai");

package/lib/vitest/hooks.mjs

@@ -324,11 +324,8 @@ export function TestDriver(context, options = {}) {
       // Add testdriver log to dashcam tracking
       await testdriver.dashcam.addFileLog(logPath, "TestDriver Log");
 
-      // Add web log tracking before starting dashcam
-      await testdriver.dashcam.addWebLog("**", "Web Logs");
-
-      // Start dashcam recording
-      await testdriver.dashcam.start();
+      // Web log tracking and dashcam.start() are handled by provision.chrome()
+      // This ensures addWebLog is called with the domain pattern BEFORE dashcam.start()
     }
   })();
 

package/lib/vitest/setup-disable-defender.mjs

@@ -0,0 +1,52 @@
+/**
+ * Post-spawn hook to disable Windows Defender
+ * 
+ * Usage in vitest.config.mjs:
+ * ```js
+ * setupFiles: [
+ *   'testdriverai/vitest/setup',
+ *   'testdriverai/vitest/setup-aws',
+ *   'testdriverai/vitest/setup-disable-defender'
+ * ]
+ * ```
+ */
+
+import { execSync } from 'child_process';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { beforeEach } from 'vitest';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+beforeEach(async (context) => {
+  // Only run if we have an instance IP (self-hosted mode)
+  if (!context.ip) return;
+
+  // Get instance ID from global state set by setup-aws
+  const instanceInfo = globalThis.__testdriverAWS?.instances?.get(context.task.id);
+  if (!instanceInfo?.instanceId) {
+    console.warn('[TestDriver] No instance ID found, skipping Defender disable');
+    return;
+  }
+
+  const { instanceId, awsRegion } = instanceInfo;
+  const scriptPath = join(__dirname, '../../setup/aws/disable-defender.sh');
+
+  console.log(`[TestDriver] Disabling Windows Defender on ${instanceId}...`);
+
+  try {
+    execSync(`bash ${scriptPath}`, {
+      encoding: 'utf-8',
+      env: {
+        ...process.env,
+        AWS_REGION: awsRegion,
+        INSTANCE_ID: instanceId,
+      },
+      stdio: 'inherit',
+    });
+  } catch (error) {
+    console.warn('[TestDriver] Failed to disable Defender:', error.message);
+    // Don't throw - this is optional optimization
+  }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.3.11",
+  "version": "7.3.13",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",
@@ -23,6 +23,7 @@
     },
     "./vitest/setup": "./lib/vitest/setup.mjs",
     "./vitest/setup-aws": "./lib/vitest/setup-aws.mjs",
+    "./vitest/setup-disable-defender": "./lib/vitest/setup-disable-defender.mjs",
     "./vitest/hooks": {
       "types": "./lib/vitest/hooks.d.ts",
       "default": "./lib/vitest/hooks.mjs"

package/sdk-log-formatter.js

@@ -376,6 +376,15 @@ class SDKLogFormatter {
     }
     if (meta.cacheHit) {
       metaParts.push(chalk.bold.yellow("⚡ cached"));
+      if (meta.validated) {
+        const confStr = meta.validationConfidence !== null && meta.validationConfidence !== undefined
+          ? ` ${(meta.validationConfidence * 100).toFixed(1)}%`
+          : '';
+        metaParts.push(chalk.green(`✅ validated${confStr}`));
+        if (meta.coordsUpdated) {
+          metaParts.push(chalk.dim.yellow(`↗ coords shifted`));
+        }
+      }
     }
     if (meta.confidence !== undefined && meta.confidence !== null) {
       metaParts.push(chalk.dim.gray(`confidence: ${meta.confidence}`));
@@ -470,6 +479,46 @@ class SDKLogFormatter {
     return parts.join(" ");
   }
 
+  /**
+   * Format a single-line findAll message (combines finding + result) 🔎
+   * @param {string} description - Element description
+   * @param {number} count - Number of elements found
+   * @param {Object} meta - Metadata (duration, cache hit)
+   * @returns {string} Formatted message
+   */
+  formatFindAllSingleLine(description, count, meta = {}) {
+    const parts = [];
+    this.addTimestamp(parts);
+    parts.push(this.getPrefix("findAll"));
+    parts.push(chalk.bold.magenta("Finding All"));
+    parts.push(chalk.cyan(`"${description}"`));
+    
+    const metaParts = [];
+    
+    // Add count with appropriate coloring
+    if (count > 0) {
+      metaParts.push(chalk.green(`found ${count}`));
+    } else {
+      metaParts.push(chalk.yellow("found 0"));
+    }
+    
+    // Add cache hit indicator
+    if (meta.cacheHit) {
+      metaParts.push(chalk.bold.yellow("⚡ cached"));
+    }
+    
+    // Add duration
+    if (meta.duration) {
+      metaParts.push(this.formatDurationColored(meta.duration));
+    }
+    
+    if (metaParts.length > 0) {
+      parts.push(this.joinMetaParts(metaParts));
+    }
+    
+    return parts.join(" ");
+  }
+
   /**
    * Format an asserting message (when assertion starts) ✓
    * @param {string} assertion - What is being asserted
@@ -932,6 +981,47 @@ class SDKLogFormatter {
     
     return parts.join(" ");
   }
+
+  /**
+   * Format act() start message - provides visual scope boundary
+   * @param {string} task - The task being executed
+   * @returns {string} Formatted act start message
+   */
+  formatActStart(task) {
+    const parts = [];
+    this.addTimestamp(parts);
+    parts.push(this.getPrefix("action"));
+    parts.push(chalk.bold.cyan("Act"));
+    parts.push(chalk.cyan(`"${task}"`));
+    return parts.join(" ");
+  }
+
+  /**
+   * Format act() completion message - provides visual scope boundary
+   * @param {number} durationMs - Duration in milliseconds
+   * @param {boolean} success - Whether the act completed successfully
+   * @param {string} [error] - Error message if failed
+   * @returns {string} Formatted act complete message
+   */
+  formatActComplete(durationMs, success, error = null) {
+    const parts = [];
+    this.addTimestamp(parts);
+    parts.push(this.getResultPrefix());
+    
+    if (success) {
+      parts.push(chalk.green("complete"));
+    } else {
+      parts.push(chalk.red("failed"));
+      if (error) {
+        parts.push(chalk.dim("·"));
+        parts.push(chalk.red(error));
+      }
+    }
+    
+    parts.push(this.formatDurationColored(durationMs, "default"));
+    
+    return parts.join(" ");
+  }
 }
 
 // Export singleton instance

package/sdk.d.ts

@@ -228,11 +228,25 @@ export interface TestDriverOptions {
   analytics?: boolean;
   /** Enable console logging output (default: true) */
   logging?: boolean;
-  /** Enable/disable cache (default: true). Set to false to force regeneration on all find operations */
-  cache?: boolean;
-  /** Global AI sampling configuration. Can be overridden per find() or assert() call. */
+  /** Enable/disable cache, or configure with thresholds
+   * @example { cache: { enabled: true, thresholds: { find: { screen: 0.05, element: 0.8 }, assert: 0.05 } } }
+   */
+  cache?: boolean | {
+    enabled?: boolean;
+    thresholds?: {
+      /** Thresholds for find operations */
+      find?: {
+        /** Pixel diff threshold for screen comparison (0-1, default 0.05 = 5% diff allowed) */
+        screen?: number;
+        /** OpenCV template match threshold for element matching (0-1, default 0.8 = 80% correlation) */
+        element?: number;
+      };
+      /** Pixel diff threshold for assert operations (0-1, default 0.05 = 5% diff allowed) */
+      assert?: number;
+    };
+  };
   ai?: AIConfig;
-  /** Cache threshold configuration for different methods */
+  /** @deprecated Use cache.thresholds instead */
   cacheThreshold?: {
     /** Threshold for find operations (default: 0.05 = 5% difference, 95% similarity) */
     find?: number;
@@ -272,20 +286,29 @@ export interface TestDriverOptions {
    * Example: 001-click-before-L42-submit-button.png
    */
   autoScreenshots?: boolean;
-  /** Redraw configuration for screen change detection */
+  /** Redraw configuration for screen change detection
+   * @example { redraw: { enabled: true, thresholds: { screen: 0.05, network: true } } }
+   */
   redraw?:
     | boolean
     | {
         /** Enable redraw detection (default: true) */
         enabled?: boolean;
-        /** Pixel difference threshold for redraw detection */
+        /** Threshold configuration */
+        thresholds?: {
+          /** Screen diff threshold (0-1). Set to false to disable screen redraw detection. Default: 0.05 */
+          screen?: number | false;
+          /** Enable/disable network activity monitoring (default: false) */
+          network?: boolean;
+        };
+        /** @deprecated Use thresholds.screen instead */
         diffThreshold?: number;
-        /** Enable screen redraw detection */
+        /** @deprecated Use thresholds.screen !== false instead */
         screenRedraw?: boolean;
-        /** Enable network activity monitoring */
+        /** @deprecated Use thresholds.network instead */
         networkMonitor?: boolean;
       };
-  /** @deprecated Use redraw.diffThreshold instead */
+  /** @deprecated Use redraw option instead */
   redrawThreshold?: number | object;
   /** Additional environment variables */
   environment?: Record<string, any>;
@@ -366,8 +389,8 @@ export interface HoverResult {
   [key: string]: any;
 }
 
-/** Bounding box for an OCR word */
-export interface OCRBoundingBox {
+/** Bounding box for a parsed element (pixel coordinates) */
+export interface ParsedElementBBox {
   /** Left edge X coordinate */
   x0: number;
   /** Top edge Y coordinate */
@@ -378,24 +401,36 @@ export interface OCRBoundingBox {
   y1: number;
 }
 
-/** Individual word extracted by OCR */
-export interface OCRWord {
-  /** The text content of the word */
+/** Bounding box as {left, top, width, height} */
+export interface ParsedElementBoundingBox {
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+}
+
+/** Individual element detected by OmniParser */
+export interface ParsedElement {
+  /** Element index */
+  index: number;
+  /** Element type (e.g. "text", "icon", "button") */
+  type: string;
+  /** Text content or description */
   content: string;
-  /** Confidence score for this word (0-100) */
-  confidence: number;
-  /** Bounding box coordinates */
-  bbox: OCRBoundingBox;
+  /** Interactivity level (e.g. "clickable", "non-interactive") */
+  interactivity: string;
+  /** Bounding box in pixel coordinates */
+  bbox: ParsedElementBBox;
+  /** Bounding box as {left, top, width, height} */
+  boundingBox: ParsedElementBoundingBox;
 }
 
-/** Result from OCR text extraction */
-export interface OCRResult {
-  /** Array of extracted words with positions */
-  words: OCRWord[];
-  /** All text concatenated with spaces */
-  fullText: string;
-  /** Overall OCR confidence (0-100) */
-  confidence: number;
+/** Result from OmniParser screen analysis */
+export interface ParseResult {
+  /** Array of detected UI elements */
+  elements: ParsedElement[];
+  /** URL of the annotated screenshot */
+  annotatedImageUrl: string;
   /** Width of the analyzed screenshot */
   imageWidth: number;
   /** Height of the analyzed screenshot */
@@ -711,7 +746,7 @@ export class Element {
   /**
    * Find the element on screen
    * @param newDescription - Optional new description to search for
-   * @param cacheThreshold - Cache threshold for this specific find (overrides global setting)
+   * @param options - Cache options: number for threshold, or object with cache.thresholds
    */
   find(newDescription?: string, cacheThreshold?: number): Promise<Element>;
 
@@ -1021,7 +1056,7 @@ export default class TestDriverSDK {
    * Automatically locates the element and returns it
    *
    * @param description - Description of the element to find
-   * @param options - Cache threshold (number) or options object
+   * @param options - Cache threshold (number) or options object with cache.thresholds
    * @returns Chainable promise that resolves to Element instance
    *
    * @example
@@ -1034,8 +1069,10 @@ export default class TestDriverSDK {
    * await element.click();
    *
    * @example
-   * // Find with custom cache threshold
-   * const element = await client.find('login button', 0.01);
+   * // Find with custom cache thresholds
+   * const element = await client.find('login button', {
+   *   cache: { thresholds: { screen: 0.05, element: 0.9 } }
+   * });
    *
    * @example
    * // Poll for element with timeout (retries every 5 seconds)
@@ -1045,7 +1082,7 @@ export default class TestDriverSDK {
   find(description: string, cacheThreshold?: number): ChainableElementPromise;
   find(
     description: string,
-    options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; ai?: AIConfig },
+    options?: { cacheThreshold?: number; cacheKey?: string; timeout?: number; ai?: AIConfig; cache?: { thresholds?: { screen?: number; element?: number } } },
   ): ChainableElementPromise;
 
   /**
@@ -1060,9 +1097,13 @@ export default class TestDriverSDK {
    *
    * @example
    * // Find with custom cache threshold
-   * const items = await client.findAll('list item', 0.01);
+   * const items = await client.findAll('list item', 0.05);
    */
   findAll(description: string, cacheThreshold?: number): Promise<Element[]>;
+  findAll(
+    description: string,
+    options?: { cacheThreshold?: number; cacheKey?: string; cache?: { thresholds?: { screen?: number } } },
+  ): Promise<Element[]>;
 
   // Text Interaction Methods
 
@@ -1282,7 +1323,7 @@ export default class TestDriverSDK {
    *
    * @example
    * // With custom threshold
-   * await client.assert('the page loaded', { threshold: 0.01, cacheKey: 'login-test' });
+   * await client.assert('the page loaded', { threshold: 0.05, cacheKey: 'login-test' });
    */
   assert(assertion: string, options?: { threshold?: number; cacheKey?: string; os?: string; resolution?: string; ai?: AIConfig }): Promise<boolean>;
 
@@ -1343,32 +1384,28 @@ export default class TestDriverSDK {
   screenshot(filename?: string): Promise<string>;
 
   /**
-   * Extract all visible text from the current screen using OCR (Tesseract)
-   * Returns structured data with text content, bounding boxes, and confidence scores
+   * Parse the current screen using OmniParser v2 to detect all UI elements
+   * Returns structured data with element types, bounding boxes, and content
+   * Requires enterprise or self-hosted plan.
    *
-   * @returns OCR extraction result with words, positions, and confidence
+   * @returns Parsed screen elements with positions and types
    *
    * @example
-   * // Get all text on screen
-   * const result = await testdriver.ocr();
-   * console.log(result.fullText);
+   * // Get all elements on screen
+   * const result = await testdriver.parse();
+   * console.log(`Found ${result.elements.length} elements`);
    *
    * @example
-   * // Find and click text
-   * const result = await testdriver.ocr();
-   * const submit = result.words.find(w => w.content === 'Submit');
-   * if (submit) {
-   *   const x = (submit.bbox.x0 + submit.bbox.x1) / 2;
-   *   const y = (submit.bbox.y0 + submit.bbox.y1) / 2;
-   *   await testdriver.click({ x, y });
-   * }
+   * // Find clickable elements
+   * const result = await testdriver.parse();
+   * const clickable = result.elements.filter(e => e.interactivity === 'clickable');
    *
    * @example
-   * // Check if text exists
-   * const result = await testdriver.ocr();
-   * const hasError = result.words.some(w => w.content.toLowerCase().includes('error'));
+   * // Find text content
+   * const result = await testdriver.parse();
+   * const textElements = result.elements.filter(e => e.type === 'text');
    */
-  ocr(): Promise<OCRResult>;
+  parse(): Promise<ParseResult>;
 
   /**
    * Wait for specified time